1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
22 use List
::Util
qw(first);
25 use Storable
qw(dclone);
26 use Time
::HiRes
qw(gettimeofday usleep);
30 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
33 use PVE
::DataCenterConfig
;
34 use PVE
::Exception
qw(raise raise_param_exc);
35 use PVE
::Format
qw(render_duration render_bytes);
36 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
37 use PVE
::Mapping
::PCI
;
38 use PVE
::Mapping
::USB
;
40 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
43 use PVE
::RESTEnvironment
qw(log_warn);
44 use PVE
::RPCEnvironment
;
48 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
52 use PVE
::QemuServer
::Helpers
qw(config_aware_timeout min_version windows_version);
53 use PVE
::QemuServer
::Cloudinit
;
54 use PVE
::QemuServer
::CGroup
;
55 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
56 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
57 use PVE
::QemuServer
::Machine
;
58 use PVE
::QemuServer
::Memory
qw(get_current_memory);
59 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
60 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
61 use PVE
::QemuServer
::QMPHelpers
qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
62 use PVE
::QemuServer
::USB
;
66 require PVE
::Network
::SDN
::Zones
;
67 require PVE
::Network
::SDN
::Vnets
;
71 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
75 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
76 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
79 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
80 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
83 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
84 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
87 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
88 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
90 # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
91 # anymore. how can we deperacate this sanely without breaking existing instances, or using
92 # older backups and snapshot?
94 "$EDK2_FW_BASE/OVMF_CODE.fd",
95 "$EDK2_FW_BASE/OVMF_VARS.fd",
100 "$EDK2_FW_BASE/AAVMF_CODE.fd",
101 "$EDK2_FW_BASE/AAVMF_VARS.fd",
106 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
108 # Note about locking: we use flock on the config file protect against concurent actions.
109 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
110 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
111 # But you can ignore this kind of lock with the --skiplock flag.
119 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
120 description
=> "Some command save/restore state from this location.",
126 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
127 description
=> "Specifies the QEMU machine type.",
129 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
134 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
137 $nodename_cache //= PVE
::INotify
::nodename
();
138 return $nodename_cache;
145 enum
=> [qw(i6300esb ib700)],
146 description
=> "Watchdog type to emulate.",
147 default => 'i6300esb',
152 enum
=> [qw(reset shutdown poweroff pause debug none)],
153 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
157 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
161 description
=> "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
166 fstrim_cloned_disks
=> {
167 description
=> "Run fstrim after moving a disk or migrating the VM.",
172 'freeze-fs-on-backup' => {
173 description
=> "Freeze/thaw guest filesystems on backup for consistency.",
179 description
=> "Select the agent type",
183 enum
=> [qw(virtio isa)],
189 description
=> "Select the VGA type.",
194 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
197 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
204 description
=> 'Enable a specific clipboard. If not set, depending on'
205 .' the display type the SPICE one will be added.',
216 description
=> "The size of the file in MB.",
220 pattern
=> '[a-zA-Z0-9\-]+',
222 format_description
=> 'string',
223 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
230 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
231 description
=> "Configure an audio device."
235 enum
=> ['spice', 'none'],
238 description
=> "Driver backend for the audio device."
242 my $spice_enhancements_fmt = {
247 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
251 enum
=> ['off', 'all', 'filter'],
254 description
=> "Enable video streaming. Uses compression for detected video streams."
261 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
263 description
=> "The file on the host to gather entropy from. In most cases '/dev/urandom'"
264 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
265 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
266 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
267 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
268 ." a hardware RNG from the host.",
272 description
=> "Maximum bytes of entropy allowed to get injected into the guest every"
273 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
274 ." `0` to disable limiting (potentially dangerous!).",
277 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
278 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
279 # reading from /dev/urandom
284 description
=> "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
285 ." the guest to retrieve another 'max_bytes' of entropy.",
291 my $meta_info_fmt = {
294 description
=> "The guest creation timestamp as UNIX epoch time",
300 description
=> "The QEMU (machine) version from the time this VM was created.",
301 pattern
=> '\d+(\.\d+)+',
310 description
=> "Specifies whether a VM will be started during system bootup.",
316 description
=> "Automatic restart after crash (currently ignored).",
321 type
=> 'string', format
=> 'pve-hotplug-features',
322 description
=> "Selectively enable hotplug features. This is a comma separated list of"
323 ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
324 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
325 ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
327 default => 'network,disk,usb',
332 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
338 description
=> "Lock/unlock the VM.",
339 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
344 description
=> "Limit of CPU usage.",
345 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
346 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
354 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
355 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
356 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
357 ." weights of all the other running VMs.",
360 default => 'cgroup v1: 1024, cgroup v2: 100',
365 description
=> "Memory properties.",
366 format
=> $PVE::QemuServer
::Memory
::memory_fmt
371 description
=> "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
377 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the"
378 ." more memory this VM gets. Number is relative to weights of all other running VMs."
379 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
387 description
=> "Keyboard layout for VNC server. This option is generally not required and"
388 ." is often better handled from within the guest OS.",
389 enum
=> PVE
::Tools
::kvmkeymaplist
(),
394 type
=> 'string', format
=> 'dns-name',
395 description
=> "Set a name for the VM. Only used on the configuration web interface.",
400 description
=> "SCSI controller model",
401 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
407 description
=> "Description for the VM. Shown in the web-interface VM's summary."
408 ." This is saved as comment inside the configuration file.",
409 maxLength
=> 1024 * 8,
414 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
415 description
=> "Specify guest operating system.",
416 verbose_description
=> <<EODESC,
417 Specify guest operating system. This is used to enable special
418 optimization/features for specific operating systems:
421 other;; unspecified OS
422 wxp;; Microsoft Windows XP
423 w2k;; Microsoft Windows 2000
424 w2k3;; Microsoft Windows 2003
425 w2k8;; Microsoft Windows 2008
426 wvista;; Microsoft Windows Vista
427 win7;; Microsoft Windows 7
428 win8;; Microsoft Windows 8/2012/2012r2
429 win10;; Microsoft Windows 10/2016/2019
430 win11;; Microsoft Windows 11/2022
431 l24;; Linux 2.4 Kernel
432 l26;; Linux 2.6 - 6.X Kernel
433 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
438 type
=> 'string', format
=> 'pve-qm-boot',
439 description
=> "Specify guest boot order. Use the 'order=' sub-property as usage with no"
440 ." key or 'legacy=' is deprecated.",
444 type
=> 'string', format
=> 'pve-qm-bootdisk',
445 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
446 pattern
=> '(ide|sata|scsi|virtio)\d+',
451 description
=> "The number of CPUs. Please use option -sockets instead.",
458 description
=> "The number of CPU sockets.",
465 description
=> "The number of cores per socket.",
472 description
=> "Enable/disable NUMA.",
478 description
=> "Enable/disable hugepages memory.",
479 enum
=> [qw(any 2 1024)],
485 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
486 ." after VM shutdown and can be used for subsequent starts.",
491 description
=> "Number of hotplugged vcpus.",
498 description
=> "Enable/disable ACPI.",
503 description
=> "Enable/disable communication with the QEMU Guest Agent and its properties.",
505 format
=> $agent_fmt,
510 description
=> "Enable/disable KVM hardware virtualization.",
516 description
=> "Enable/disable time drift fix.",
522 description
=> "Set the real time clock (RTC) to local time. This is enabled by default if"
523 ." the `ostype` indicates a Microsoft Windows OS.",
528 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
532 type
=> 'string', format
=> $vga_fmt,
533 description
=> "Configure the VGA hardware.",
534 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
535 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
536 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
537 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
538 ." display server. For win* OS you can select how many independent displays you want,"
539 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
540 ." using a serial device as terminal.",
544 type
=> 'string', format
=> 'pve-qm-watchdog',
545 description
=> "Create a virtual hardware watchdog device.",
546 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
547 ." action), the watchdog must be periodically polled by an agent inside the guest or"
548 ." else the watchdog will reset the guest (or execute the respective action specified)",
553 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
554 description
=> "Set the initial date of the real time clock. Valid format for date are:"
555 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
556 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
559 startup
=> get_standard_option
('pve-startup-order'),
563 description
=> "Enable/disable Template.",
569 description
=> "Arbitrary arguments passed to kvm.",
570 verbose_description
=> <<EODESCR,
571 Arbitrary arguments passed to kvm, for example:
573 args: -no-reboot -smbios 'type=0,vendor=FOO'
575 NOTE: this option is for experts only.
582 description
=> "Enable/disable the USB tablet device.",
583 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
584 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
585 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
586 ." may consider disabling this to save some context switches. This is turned off by"
587 ." default if you use spice (`qm set <vmid> --vga qxl`).",
592 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
596 migrate_downtime
=> {
599 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
605 type
=> 'string', format
=> 'pve-qm-ide',
606 typetext
=> '<volume>',
607 description
=> "This is an alias for option -ide2",
611 description
=> "Emulated CPU type.",
613 format
=> 'pve-vm-cpu-conf',
615 parent
=> get_standard_option
('pve-snapshot-name', {
617 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
621 description
=> "Timestamp for snapshots.",
627 type
=> 'string', format
=> 'pve-volume-id',
628 description
=> "Reference to a volume which stores the VM state. This is used internally"
631 vmstatestorage
=> get_standard_option
('pve-storage-id', {
632 description
=> "Default storage for VM state volumes/files.",
635 runningmachine
=> get_standard_option
('pve-qemu-machine', {
636 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
640 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
641 ." internally for snapshots.",
644 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
645 format_description
=> 'QEMU -cpu parameter'
647 machine
=> get_standard_option
('pve-qemu-machine'),
649 description
=> "Virtual processor architecture. Defaults to the host.",
652 enum
=> [qw(x86_64 aarch64)],
655 description
=> "Specify SMBIOS type 1 fields.",
656 type
=> 'string', format
=> 'pve-qm-smbios1',
663 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
664 ." remove disk operations.",
670 enum
=> [ qw(seabios ovmf) ],
671 description
=> "Select BIOS implementation.",
672 default => 'seabios',
676 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
677 format_description
=> 'UUID',
678 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
679 ." to disable explicitly.",
680 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
681 ." value identifier to the guest OS. This allows to notify the guest operating system"
682 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
683 ." execution or creation from a template). The guest operating system notices the"
684 ." change, and is then able to react as appropriate by marking its copies of"
685 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
686 ."Note that auto-creation only works when done through API/CLI create or update methods"
687 .", but not when manually editing the config file.",
688 default => "1 (autogenerated)",
693 format
=> 'pve-volume-id',
695 description
=> "Script that will be executed during various steps in the vms lifetime.",
699 format
=> $ivshmem_fmt,
700 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
706 format
=> $audio_fmt,
707 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
710 spice_enhancements
=> {
712 format
=> $spice_enhancements_fmt,
713 description
=> "Configure additional enhancements for SPICE.",
717 type
=> 'string', format
=> 'pve-tag-list',
718 description
=> 'Tags of the VM. This is only meta information.',
724 description
=> "Configure a VirtIO-based Random Number Generator.",
729 format
=> $meta_info_fmt,
730 description
=> "Some (read-only) meta-information about this guest.",
734 type
=> 'string', format
=> 'pve-cpuset',
735 description
=> "List of host cores used to execute guest processes, for example: 0,5,8-11",
744 description
=> 'Specify a custom file containing all meta data passed to the VM via"
745 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
746 format
=> 'pve-volume-id',
747 format_description
=> 'volume',
752 description
=> 'To pass a custom file containing all network data to the VM via cloud-init.',
753 format
=> 'pve-volume-id',
754 format_description
=> 'volume',
759 description
=> 'To pass a custom file containing all user data to the VM via cloud-init.',
760 format
=> 'pve-volume-id',
761 format_description
=> 'volume',
766 description
=> 'To pass a custom file containing all vendor data to the VM via cloud-init.',
767 format
=> 'pve-volume-id',
768 format_description
=> 'volume',
771 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
773 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
774 my $confdesc_cloudinit = {
778 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
779 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
780 .' and `configdrive2` for windows.',
781 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
786 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
787 ." image's configured default user.",
792 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
793 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
794 .' support hashed passwords.',
799 description
=> 'cloud-init: do an automatic package upgrade after the first boot.',
805 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
807 format
=> 'pve-qm-cicustom',
812 description
=> 'cloud-init: Sets DNS search domains for a container. Create will'
813 .' automatically use the setting from the host if neither searchdomain nor nameserver'
818 type
=> 'string', format
=> 'address-list',
819 description
=> 'cloud-init: Sets DNS server IP address for a container. Create will'
820 .' automatically use the setting from the host if neither searchdomain nor nameserver'
826 format
=> 'urlencoded',
827 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
831 # what about other qemu settings ?
833 #machine => 'string',
846 ##soundhw => 'string',
848 while (my ($k, $v) = each %$confdesc) {
849 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
853 my $MAX_SERIAL_PORTS = 4;
854 my $MAX_PARALLEL_PORTS = 3;
856 for (my $i = 0; $i < $PVE::QemuServer
::Memory
::MAX_NUMA
; $i++) {
857 $confdesc->{"numa$i"} = $PVE::QemuServer
::Memory
::numadesc
;
860 my $nic_model_list = [
876 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
878 my $net_fmt_bridge_descr = <<__EOD__;
879 Bridge to attach the network device to. The Proxmox VE standard bridge
882 If you do not specify a bridge, we create a kvm user (NATed) network
883 device, which provides DHCP and DNS services. The following addresses
890 The DHCP server assign addresses to the guest starting from 10.0.2.15.
894 macaddr
=> get_standard_option
('mac-addr', {
895 description
=> "MAC address. That address must be unique withing your network. This is"
896 ." automatically generated if not specified.",
900 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
901 ." very low CPU overhead. If your guest does not support this driver, it is usually"
902 ." best to use 'e1000'.",
903 enum
=> $nic_model_list,
906 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
907 bridge
=> get_standard_option
('pve-bridge-id', {
908 description
=> $net_fmt_bridge_descr,
913 minimum
=> 0, maximum
=> 64,
914 description
=> 'Number of packet queues to be used on the device.',
920 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
925 minimum
=> 1, maximum
=> 4094,
926 description
=> 'VLAN tag to apply to packets on this interface.',
931 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
932 description
=> 'VLAN trunks to pass through this interface.',
933 format_description
=> 'vlanid[;vlanid...]',
938 description
=> 'Whether this interface should be protected by the firewall.',
943 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
948 minimum
=> 1, maximum
=> 65520,
949 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
956 type
=> 'string', format
=> $net_fmt,
957 description
=> "Specify network devices.",
960 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
965 format
=> 'pve-ipv4-config',
966 format_description
=> 'IPv4Format/CIDR',
967 description
=> 'IPv4 address in CIDR format.',
974 format_description
=> 'GatewayIPv4',
975 description
=> 'Default gateway for IPv4 traffic.',
981 format
=> 'pve-ipv6-config',
982 format_description
=> 'IPv6Format/CIDR',
983 description
=> 'IPv6 address in CIDR format.',
990 format_description
=> 'GatewayIPv6',
991 description
=> 'Default gateway for IPv6 traffic.',
996 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
999 type
=> 'string', format
=> 'pve-qm-ipconfig',
1000 description
=> <<'EODESCR',
1001 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1003 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1005 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1006 gateway should be provided.
1007 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1008 cloud-init 19.4 or newer.
1010 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1014 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
1016 for (my $i = 0; $i < $MAX_NETS; $i++) {
1017 $confdesc->{"net$i"} = $netdesc;
1018 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1021 foreach my $key (keys %$confdesc_cloudinit) {
1022 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1025 PVE
::JSONSchema
::register_format
('pve-cpuset', \
&pve_verify_cpuset
);
1026 sub pve_verify_cpuset
{
1027 my ($set_text, $noerr) = @_;
1029 my ($count, $members) = eval { PVE
::CpuSet
::parse_cpuset
($set_text) };
1033 die "unable to parse cpuset option\n";
1036 return PVE
::CpuSet-
>new($members)->short_string();
1039 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1040 sub verify_volume_id_or_qm_path
{
1041 my ($volid, $noerr) = @_;
1043 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1045 return verify_volume_id_or_absolute_path
($volid, $noerr);
1048 PVE
::JSONSchema
::register_format
('pve-volume-id-or-absolute-path', \
&verify_volume_id_or_absolute_path
);
1049 sub verify_volume_id_or_absolute_path
{
1050 my ($volid, $noerr) = @_;
1052 return $volid if $volid =~ m
|^/|;
1054 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1065 pattern
=> '(/dev/.+|socket)',
1066 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1067 verbose_description
=> <<EODESCR,
1068 Create a serial device inside the VM (n is 0 to 3), and pass through a
1069 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1070 host side (use 'qm terminal' to open a terminal connection).
1072 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1073 use with special care.
1075 CAUTION: Experimental! User reported problems with this option.
1082 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1083 description
=> "Map host parallel devices (n is 0 to 2).",
1084 verbose_description
=> <<EODESCR,
1085 Map host parallel devices (n is 0 to 2).
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1090 CAUTION: Experimental! User reported problems with this option.
1094 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1095 $confdesc->{"parallel$i"} = $paralleldesc;
1098 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1099 $confdesc->{"serial$i"} = $serialdesc;
1102 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1103 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1106 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1107 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1110 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
1111 $confdesc->{"usb$i"} = $PVE::QemuServer
::USB
::usbdesc
;
1119 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1120 . " Deprecated, use 'order=' instead.",
1121 pattern
=> '[acdn]{1,4}',
1122 format_description
=> "[acdn]{1,4}",
1124 # note: this is also the fallback if boot: is not given at all
1130 format
=> 'pve-qm-bootdev-list',
1131 format_description
=> "device[;device...]",
1132 description
=> <<EODESC,
1133 The guest will attempt to boot from devices in the order they appear here.
1135 Disks, optical drives and passed-through storage USB devices will be directly
1136 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1137 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1139 Note that only devices in this list will be marked as bootable and thus loaded
1140 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1141 (e.g. software-raid), you need to specify all of them here.
1143 Overrides the deprecated 'legacy=[acdn]*' value when given.
1147 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1149 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1150 sub verify_bootdev
{
1151 my ($dev, $noerr) = @_;
1153 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1154 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && !$special;
1158 return 0 if $dev !~ m/^$base\d+$/;
1159 return 0 if !$confdesc->{$dev};
1163 return $dev if $check->("net");
1164 return $dev if $check->("usb");
1165 return $dev if $check->("hostpci");
1168 die "invalid boot device '$dev'\n";
1171 sub print_bootorder
{
1173 return "" if !@$devs;
1174 my $data = { order
=> join(';', @$devs) };
1175 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1178 my $kvm_api_version = 0;
1181 return $kvm_api_version if $kvm_api_version;
1183 open my $fh, '<', '/dev/kvm' or return;
1185 # 0xae00 => KVM_GET_API_VERSION
1186 $kvm_api_version = ioctl($fh, 0xae00, 0);
1189 return $kvm_api_version;
1192 my $kvm_user_version = {};
1195 sub kvm_user_version
{
1198 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1199 my $st = stat($binary);
1201 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1202 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1203 $cachedmtime == $st->mtime;
1205 $kvm_user_version->{$binary} = 'unknown';
1206 $kvm_mtime->{$binary} = $st->mtime;
1210 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1211 $kvm_user_version->{$binary} = $2;
1215 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1218 return $kvm_user_version->{$binary};
1221 my sub extract_version
{
1222 my ($machine_type, $version) = @_;
1223 $version = kvm_user_version
() if !defined($version);
1224 return PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1227 sub kernel_has_vhost_net
{
1228 return -c
'/dev/vhost-net';
1233 return defined($confdesc->{$key});
1237 sub get_cdrom_path
{
1239 return $cdrom_path if defined($cdrom_path);
1241 $cdrom_path = first
{ -l
$_ } map { "/dev/cdrom$_" } ('', '1', '2');
1243 if (!defined($cdrom_path)) {
1244 log_warn
("no physical CD-ROM available, ignoring");
1252 my ($storecfg, $vmid, $cdrom) = @_;
1254 if ($cdrom eq 'cdrom') {
1255 return get_cdrom_path
();
1256 } elsif ($cdrom eq 'none') {
1258 } elsif ($cdrom =~ m
|^/|) {
1261 return PVE
::Storage
::path
($storecfg, $cdrom);
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id
{
1267 my ($vmid, $file, $media) = @_;
1269 if (!($file eq 'none' || $file eq 'cdrom' ||
1270 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1272 return if $file =~ m
|/|;
1274 if ($media && $media eq 'cdrom') {
1275 $file = "local:iso/$file";
1277 $file = "local:$vmid/$file";
1284 sub verify_media_type
{
1285 my ($opt, $vtype, $media) = @_;
1290 if ($media eq 'disk') {
1292 } elsif ($media eq 'cdrom') {
1295 die "internal error";
1298 return if ($vtype eq $etype);
1300 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1303 sub cleanup_drive_path
{
1304 my ($opt, $storecfg, $drive) = @_;
1306 # try to convert filesystem paths to volume IDs
1308 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1309 ($drive->{file
} !~ m
|^/dev/.+|) &&
1310 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1311 ($drive->{file
} !~ m/^\d+$/)) {
1312 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1313 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1315 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1316 verify_media_type
($opt, $vtype, $drive->{media
});
1317 $drive->{file
} = $volid;
1320 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1323 sub parse_hotplug_features
{
1328 return $res if $data eq '0';
1330 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1332 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1333 if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1336 die "invalid hotplug feature '$feature'\n";
1342 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1343 sub pve_verify_hotplug_features
{
1344 my ($value, $noerr) = @_;
1346 return $value if parse_hotplug_features
($value);
1350 die "unable to parse hotplug option\n";
1353 sub assert_clipboard_config
{
1356 my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1360 && $vga->{'clipboard'} eq 'vnc'
1362 && $vga->{type
} !~ $clipboard_regex
1364 die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1369 my($fh, $noerr) = @_;
1372 my $SG_GET_VERSION_NUM = 0x2282;
1374 my $versionbuf = "\x00" x
8;
1375 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1377 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1380 my $version = unpack("I", $versionbuf);
1381 if ($version < 30000) {
1382 die "scsi generic interface too old\n" if !$noerr;
1386 my $buf = "\x00" x
36;
1387 my $sensebuf = "\x00" x
8;
1388 my $cmd = pack("C x3 C x1", 0x12, 36);
1390 # see /usr/include/scsi/sg.h
1391 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1394 $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1397 $ret = ioctl($fh, $SG_IO, $packet);
1399 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1403 my @res = unpack($sg_io_hdr_t, $packet);
1404 if ($res[17] || $res[18]) {
1405 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1410 $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1412 $res->{removable
} = $res->{removable
} & 128 ?
1 : 0;
1413 $res->{type
} &= 0x1F;
1421 my $fh = IO
::File-
>new("+<$path") || return;
1422 my $res = scsi_inquiry
($fh, 1);
1428 sub print_tabletdevice_full
{
1429 my ($conf, $arch) = @_;
1431 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1433 # we use uhci for old VMs because tablet driver was buggy in older qemu
1435 if ($q35 || $arch eq 'aarch64') {
1441 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1444 sub print_keyboarddevice_full
{
1445 my ($conf, $arch) = @_;
1447 return if $arch ne 'aarch64';
1449 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1452 my sub get_drive_id
{
1454 return "$drive->{interface}$drive->{index}";
1457 sub print_drivedevice_full
{
1458 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1463 my $drive_id = get_drive_id
($drive);
1464 if ($drive->{interface
} eq 'virtio') {
1465 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1466 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1467 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1468 } elsif ($drive->{interface
} eq 'scsi') {
1470 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1471 my $unit = $drive->{index} % $maxdev;
1472 my $devicetype = 'hd';
1474 if (drive_is_cdrom
($drive)) {
1477 if ($drive->{file
} =~ m
|^/|) {
1478 $path = $drive->{file
};
1479 if (my $info = path_is_scsi
($path)) {
1480 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1481 $devicetype = 'block';
1482 } elsif ($info->{type
} == 1) { # tape
1483 $devicetype = 'generic';
1487 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1490 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1491 my $version = extract_version
($machine_type, kvm_user_version
());
1492 if ($path =~ m/^iscsi\:\/\
// &&
1493 !min_version
($version, 4, 1)) {
1494 $devicetype = 'generic';
1498 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1499 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1501 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1502 .",lun=$drive->{index}";
1504 $device .= ",drive=drive-$drive_id,id=$drive_id";
1506 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1507 $device .= ",rotation_rate=1";
1509 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1511 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1512 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1513 my $controller = int($drive->{index} / $maxdev);
1514 my $unit = $drive->{index} % $maxdev;
1516 # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1517 # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1518 # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1519 # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1520 # were before. Move odd ones up by 2 where they don't clash.
1521 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) && $drive->{interface
} eq 'ide') {
1522 $controller += 2 * ($unit % 2);
1526 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1528 $device = "ide-$devicetype";
1529 if ($drive->{interface
} eq 'ide') {
1530 $device .= ",bus=ide.$controller,unit=$unit";
1532 $device .= ",bus=ahci$controller.$unit";
1534 $device .= ",drive=drive-$drive_id,id=$drive_id";
1536 if ($devicetype eq 'hd') {
1537 if (my $model = $drive->{model
}) {
1538 $model = URI
::Escape
::uri_unescape
($model);
1539 $device .= ",model=$model";
1541 if ($drive->{ssd
}) {
1542 $device .= ",rotation_rate=1";
1545 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1546 } elsif ($drive->{interface
} eq 'usb') {
1548 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1550 die "unsupported interface type";
1553 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1555 if (my $serial = $drive->{serial
}) {
1556 $serial = URI
::Escape
::uri_unescape
($serial);
1557 $device .= ",serial=$serial";
1564 sub get_initiator_name
{
1567 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1568 while (defined(my $line = <$fh>)) {
1569 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1578 my sub storage_allows_io_uring_default
{
1579 my ($scfg, $cache_direct) = @_;
1581 # io_uring with cache mode writeback or writethrough on krbd will hang...
1582 return if $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1584 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1585 # sometimes, just plain disable...
1586 return if $scfg && $scfg->{type
} eq 'lvm';
1588 # io_uring causes problems when used with CIFS since kernel 5.15
1589 # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1590 return if $scfg && $scfg->{type
} eq 'cifs';
1595 my sub drive_uses_cache_direct
{
1596 my ($drive, $scfg) = @_;
1598 my $cache_direct = 0;
1600 if (my $cache = $drive->{cache
}) {
1601 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1602 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1606 return $cache_direct;
1609 sub print_drive_commandline_full
{
1610 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1613 my $volid = $drive->{file
};
1614 my $format = $drive->{format
};
1615 my $drive_id = get_drive_id
($drive);
1617 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1618 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1620 if (drive_is_cdrom
($drive)) {
1621 $path = get_iso_path
($storecfg, $vmid, $volid);
1622 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1625 $path = PVE
::Storage
::path
($storecfg, $volid);
1626 $format //= qemu_img_format
($scfg, $volname);
1633 my $is_rbd = $path =~ m/^rbd:/;
1636 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1637 foreach my $o (@qemu_drive_options) {
1638 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1641 # snapshot only accepts on|off
1642 if (defined($drive->{snapshot
})) {
1643 my $v = $drive->{snapshot
} ?
'on' : 'off';
1644 $opts .= ",snapshot=$v";
1647 if (defined($drive->{ro
})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1648 $opts .= ",readonly=" . ($drive->{ro
} ?
'on' : 'off');
1651 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1652 my ($dir, $qmpname) = @$type;
1653 if (my $v = $drive->{"mbps$dir"}) {
1654 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1656 if (my $v = $drive->{"mbps${dir}_max"}) {
1657 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1659 if (my $v = $drive->{"bps${dir}_max_length"}) {
1660 $opts .= ",throttling.bps$qmpname-max-length=$v";
1662 if (my $v = $drive->{"iops${dir}"}) {
1663 $opts .= ",throttling.iops$qmpname=$v";
1665 if (my $v = $drive->{"iops${dir}_max"}) {
1666 $opts .= ",throttling.iops$qmpname-max=$v";
1668 if (my $v = $drive->{"iops${dir}_max_length"}) {
1669 $opts .= ",throttling.iops$qmpname-max-length=$v";
1674 $format = "rbd" if $is_rbd;
1675 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1677 $opts .= ",format=alloc-track,file.driver=$format";
1679 $opts .= ",format=$format";
1682 my $cache_direct = drive_uses_cache_direct
($drive, $scfg);
1684 $opts .= ",cache=none" if !$drive->{cache
} && $cache_direct;
1686 if (!$drive->{aio
}) {
1687 if ($io_uring && storage_allows_io_uring_default
($scfg, $cache_direct)) {
1688 # io_uring supports all cache modes
1689 $opts .= ",aio=io_uring";
1691 # aio native works only with O_DIRECT
1693 $opts .= ",aio=native";
1695 $opts .= ",aio=threads";
1700 if (!drive_is_cdrom
($drive)) {
1702 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1703 $detectzeroes = 'off';
1704 } elsif ($drive->{discard
}) {
1705 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1707 # This used to be our default with discard not being specified:
1708 $detectzeroes = 'on';
1711 # note: 'detect-zeroes' works per blockdev and we want it to persist
1712 # after the alloc-track is removed, so put it on 'file' directly
1713 my $dz_param = $pbs_name ?
"file.detect-zeroes" : "detect-zeroes";
1714 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1718 $opts .= ",backing=$pbs_name";
1719 $opts .= ",auto-remove=on";
1722 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1723 my $file_param = "file";
1725 # non-rbd drivers require the underlying file to be a seperate block
1726 # node, so add a second .file indirection
1727 $file_param .= ".file" if !$is_rbd;
1728 $file_param .= ".filename";
1730 my $pathinfo = $path ?
"$file_param=$path," : '';
1732 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1735 sub print_pbs_blockdev
{
1736 my ($pbs_conf, $pbs_name) = @_;
1737 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1738 $blockdev .= ",repository=$pbs_conf->{repository}";
1739 $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace
};
1740 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1741 $blockdev .= ",archive=$pbs_conf->{archive}";
1742 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1746 sub print_netdevice_full
{
1747 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1749 my $device = $net->{model
};
1750 if ($net->{model
} eq 'virtio') {
1751 $device = 'virtio-net-pci';
1754 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1755 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1756 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1757 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1758 # and out of each queue plus one config interrupt and control vector queue
1759 my $vectors = $net->{queues
} * 2 + 2;
1760 $tmpstr .= ",vectors=$vectors,mq=on";
1761 if (min_version
($machine_version, 7, 1)) {
1762 $tmpstr .= ",packed=on";
1766 if (min_version
($machine_version, 7, 1) && $net->{model
} eq 'virtio'){
1767 $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1770 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1772 if (my $mtu = $net->{mtu
}) {
1773 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1774 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1777 } elsif ($mtu < 576) {
1778 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1779 } elsif ($mtu > $bridge_mtu) {
1780 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1782 $tmpstr .= ",host_mtu=$mtu";
1784 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1788 if ($use_old_bios_files) {
1790 if ($device eq 'virtio-net-pci') {
1791 $romfile = 'pxe-virtio.rom';
1792 } elsif ($device eq 'e1000') {
1793 $romfile = 'pxe-e1000.rom';
1794 } elsif ($device eq 'e1000e') {
1795 $romfile = 'pxe-e1000e.rom';
1796 } elsif ($device eq 'ne2k') {
1797 $romfile = 'pxe-ne2k_pci.rom';
1798 } elsif ($device eq 'pcnet') {
1799 $romfile = 'pxe-pcnet.rom';
1800 } elsif ($device eq 'rtl8139') {
1801 $romfile = 'pxe-rtl8139.rom';
1803 $tmpstr .= ",romfile=$romfile" if $romfile;
1809 sub print_netdev_full
{
1810 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1813 if ($netid =~ m/^net(\d+)$/) {
1817 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1819 my $ifname = "tap${vmid}i$i";
1821 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1822 die "interface name '$ifname' is too long (max 15 character)\n"
1823 if length($ifname) >= 16;
1825 my $vhostparam = '';
1826 if (is_native
($arch)) {
1827 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1830 my $vmname = $conf->{name
} || "vm$vmid";
1833 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1835 if ($net->{bridge
}) {
1836 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1837 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1839 $netdev = "type=user,id=$netid,hostname=$vmname";
1842 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1848 'cirrus' => 'cirrus-vga',
1850 'vmware' => 'vmware-svga',
1851 'virtio' => 'virtio-vga',
1852 'virtio-gl' => 'virtio-vga-gl',
1855 sub print_vga_device
{
1856 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1858 my $type = $vga_map->{$vga->{type
}};
1859 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1860 $type = 'virtio-gpu';
1862 my $vgamem_mb = $vga->{memory
};
1864 my $max_outputs = '';
1866 $type = $id ?
'qxl' : 'qxl-vga';
1868 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1869 # set max outputs so linux can have up to 4 qxl displays with one device
1870 if (min_version
($machine_version, 4, 1)) {
1871 $max_outputs = ",max_outputs=4";
1876 die "no devicetype for $vga->{type}\n" if !$type;
1880 if ($vga->{type
} =~ /^virtio/) {
1881 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1882 $memory = ",max_hostmem=$bytes";
1884 # from https://www.spice-space.org/multiple-monitors.html
1885 $memory = ",vgamem_mb=$vga->{memory}";
1886 my $ram = $vgamem_mb * 4;
1887 my $vram = $vgamem_mb * 2;
1888 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1890 $memory = ",vgamem_mb=$vga->{memory}";
1892 } elsif ($qxlnum && $id) {
1893 $memory = ",ram_size=67108864,vram_size=33554432";
1897 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1898 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1901 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1902 my $vgaid = "vga" . ($id // '');
1904 if ($q35 && $vgaid eq 'vga') {
1905 # the first display uses pcie.0 bus on q35 machines
1906 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1908 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1911 if ($vga->{type
} eq 'virtio-gl') {
1912 my $base = '/usr/lib/x86_64-linux-gnu/lib';
1913 die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1914 if !-e
"${base}EGL.so.1" || !-e
"${base}GL.so.1";
1916 die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1917 if !PVE
::Tools
::dir_glob_regex
('/dev/dri/', "renderD.*");
1920 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1923 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1925 my ($data, $disable_mac_autogen) = @_;
1927 my $res = eval { parse_property_string
($net_fmt, $data) };
1932 if (!defined($res->{macaddr
}) && !$disable_mac_autogen) {
1933 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1934 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1939 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1940 sub parse_ipconfig
{
1943 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1949 if ($res->{gw
} && !$res->{ip
}) {
1950 warn 'gateway specified without specifying an IP address';
1953 if ($res->{gw6
} && !$res->{ip6
}) {
1954 warn 'IPv6 gateway specified without specifying an IPv6 address';
1957 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1958 warn 'gateway specified together with DHCP';
1961 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1963 warn "IPv6 gateway specified together with $res->{ip6} address";
1967 if (!$res->{ip
} && !$res->{ip6
}) {
1968 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1977 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1980 sub add_random_macs
{
1981 my ($settings) = @_;
1983 foreach my $opt (keys %$settings) {
1984 next if $opt !~ m/^net(\d+)$/;
1985 my $net = parse_net
($settings->{$opt});
1987 $settings->{$opt} = print_net
($net);
1991 sub vm_is_volid_owner
{
1992 my ($storecfg, $vmid, $volid) = @_;
1994 if ($volid !~ m
|^/|) {
1996 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1997 if ($owner && ($owner == $vmid)) {
2005 sub vmconfig_register_unused_drive
{
2006 my ($storecfg, $vmid, $conf, $drive) = @_;
2008 if (drive_is_cloudinit
($drive)) {
2009 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2011 delete $conf->{cloudinit
};
2012 } elsif (!drive_is_cdrom
($drive)) {
2013 my $volid = $drive->{file
};
2014 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2015 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2020 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2024 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2025 format_description
=> 'UUID',
2026 description
=> "Set SMBIOS1 UUID.",
2031 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2032 format_description
=> 'Base64 encoded string',
2033 description
=> "Set SMBIOS1 version.",
2038 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2039 format_description
=> 'Base64 encoded string',
2040 description
=> "Set SMBIOS1 serial number.",
2045 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2046 format_description
=> 'Base64 encoded string',
2047 description
=> "Set SMBIOS1 manufacturer.",
2052 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2053 format_description
=> 'Base64 encoded string',
2054 description
=> "Set SMBIOS1 product ID.",
2059 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2060 format_description
=> 'Base64 encoded string',
2061 description
=> "Set SMBIOS1 SKU string.",
2066 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2067 format_description
=> 'Base64 encoded string',
2068 description
=> "Set SMBIOS1 family string.",
2073 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2081 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2088 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2091 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2093 sub parse_watchdog
{
2098 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2103 sub parse_guest_agent
{
2106 return {} if !defined($conf->{agent
});
2108 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2111 # if the agent is disabled ignore the other potentially set properties
2112 return {} if !$res->{enabled
};
2117 my ($conf, $key) = @_;
2118 return undef if !defined($conf->{agent
});
2120 my $agent = parse_guest_agent
($conf);
2121 return $agent->{$key};
2127 return {} if !$value;
2128 my $res = eval { parse_property_string
($vga_fmt, $value) };
2138 my $res = eval { parse_property_string
($rng_fmt, $value) };
2143 sub parse_meta_info
{
2148 my $res = eval { parse_property_string
($meta_info_fmt, $value) };
2153 sub new_meta_info_string
{
2154 my () = @_; # for now do not allow to override any value
2156 return PVE
::JSONSchema
::print_property_string
(
2158 'creation-qemu' => kvm_user_version
(),
2159 ctime
=> "". int(time()),
2165 sub qemu_created_version_fixups
{
2166 my ($conf, $forcemachine, $kvmver) = @_;
2168 my $meta = parse_meta_info
($conf->{meta
}) // {};
2169 my $forced_vers = PVE
::QemuServer
::Machine
::extract_version
($forcemachine);
2171 # check if we need to apply some handling for VMs that always use the latest machine version but
2172 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2173 # would be required (we do not want to pin machine version for non-windows OS type)
2175 (!defined($conf->{machine
}) || $conf->{machine
} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2176 && (!defined($meta->{'creation-qemu'}) || !min_version
($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2177 && (!$forced_vers || min_version
($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2178 && min_version
($kvmver, 6, 1) # only need to apply the change since 6.1
2180 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2181 if ($q35 && $conf->{ostype
} && $conf->{ostype
} eq 'l26') {
2182 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2183 # and thus with the predictable interface naming of systemd
2184 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2190 # add JSON properties for create and set function
2191 sub json_config_properties
{
2192 my ($prop, $with_disk_alloc) = @_;
2194 my $skip_json_config_opts = {
2198 runningmachine
=> 1,
2203 foreach my $opt (keys %$confdesc) {
2204 next if $skip_json_config_opts->{$opt};
2206 if ($with_disk_alloc && is_valid_drivename
($opt)) {
2207 $prop->{$opt} = $PVE::QemuServer
::Drive
::drivedesc_hash_with_alloc-
>{$opt};
2209 $prop->{$opt} = $confdesc->{$opt};
2216 # Properties that we can read from an OVF file
2217 sub json_ovf_properties
{
2220 for my $device (PVE
::QemuServer
::Drive
::valid_drive_names
()) {
2221 $prop->{$device} = {
2223 format
=> 'pve-volume-id-or-absolute-path',
2224 description
=> "Disk image that gets imported to $device",
2231 description
=> "The number of CPU cores.",
2236 description
=> "Amount of RAM for the VM in MB.",
2241 description
=> "Name of the VM.",
2248 # return copy of $confdesc_cloudinit to generate documentation
2249 sub cloudinit_config_properties
{
2251 return dclone
($confdesc_cloudinit);
2254 sub cloudinit_pending_properties
{
2256 map { $_ => 1 } keys $confdesc_cloudinit->%*,
2259 $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2264 my ($key, $value) = @_;
2266 die "unknown setting '$key'\n" if !$confdesc->{$key};
2268 my $type = $confdesc->{$key}->{type
};
2270 if (!defined($value)) {
2271 die "got undefined value\n";
2274 if ($value =~ m/[\n\r]/) {
2275 die "property contains a line feed\n";
2278 if ($type eq 'boolean') {
2279 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2280 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2281 die "type check ('boolean') failed - got '$value'\n";
2282 } elsif ($type eq 'integer') {
2283 return int($1) if $value =~ m/^(\d+)$/;
2284 die "type check ('integer') failed - got '$value'\n";
2285 } elsif ($type eq 'number') {
2286 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2287 die "type check ('number') failed - got '$value'\n";
2288 } elsif ($type eq 'string') {
2289 if (my $fmt = $confdesc->{$key}->{format
}) {
2290 PVE
::JSONSchema
::check_format
($fmt, $value);
2293 $value =~ s/^\"(.*)\"$/$1/;
2296 die "internal error"
2301 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2303 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2305 if (!$skiplock && !PVE
::QemuConfig-
>has_lock($conf, 'suspended')) {
2306 PVE
::QemuConfig-
>check_lock($conf);
2309 if ($conf->{template
}) {
2310 # check if any base image is still used by a linked clone
2311 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2312 my ($ds, $drive) = @_;
2313 return if drive_is_cdrom
($drive);
2315 my $volid = $drive->{file
};
2316 return if !$volid || $volid =~ m
|^/|;
2318 die "base volume '$volid' is still in use by linked cloned\n"
2319 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2325 my $remove_owned_drive = sub {
2326 my ($ds, $drive) = @_;
2327 return if drive_is_cdrom
($drive, 1);
2329 my $volid = $drive->{file
};
2330 return if !$volid || $volid =~ m
|^/|;
2331 return if $volids->{$volid};
2333 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2334 return if !$path || !$owner || ($owner != $vmid);
2336 $volids->{$volid} = 1;
2337 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2338 warn "Could not remove disk '$volid', check manually: $@" if $@;
2341 # only remove disks owned by this VM (referenced in the config)
2342 my $include_opts = {
2343 include_unused
=> 1,
2344 extra_keys
=> ['vmstate'],
2346 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2348 for my $snap (values %{$conf->{snapshots
}}) {
2349 next if !defined($snap->{vmstate
});
2350 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2351 next if !defined($drive);
2352 $remove_owned_drive->('vmstate', $drive);
2355 PVE
::QemuConfig-
>foreach_volume_full($conf->{pending
}, $include_opts, $remove_owned_drive);
2357 if ($purge_unreferenced) { # also remove unreferenced disk
2358 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2359 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2360 my ($volid, $sid, $volname, $d) = @_;
2361 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2366 eval { delete_ifaces_ipams_ips
($conf, $vmid)};
2369 if (defined $replacement_conf) {
2370 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2372 PVE
::QemuConfig-
>destroy_config($vmid);
2376 sub parse_vm_config
{
2377 my ($filename, $raw, $strict) = @_;
2379 return if !defined($raw);
2382 digest
=> Digest
::SHA
::sha1_hex
($raw),
2388 my $handle_error = sub {
2398 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2399 || die "got strange filename '$filename'";
2405 my $finish_description = sub {
2406 if (defined($descr)) {
2408 $conf->{description
} = $descr;
2414 my @lines = split(/\n/, $raw);
2415 foreach my $line (@lines) {
2416 next if $line =~ m/^\s*$/;
2418 if ($line =~ m/^\[PENDING\]\s*$/i) {
2419 $section = 'pending';
2420 $finish_description->();
2421 $conf = $res->{$section} = {};
2423 } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2424 $section = 'cloudinit';
2425 $finish_description->();
2426 $conf = $res->{$section} = {};
2429 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2431 $finish_description->();
2432 $conf = $res->{snapshots
}->{$section} = {};
2436 if ($line =~ m/^\#(.*)$/) {
2437 $descr = '' if !defined($descr);
2438 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2442 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2443 $descr = '' if !defined($descr);
2444 $descr .= PVE
::Tools
::decode_text
($2);
2445 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2446 $conf->{snapstate
} = $1;
2447 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2450 $conf->{$key} = $value;
2451 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2453 if ($section eq 'pending') {
2454 $conf->{delete} = $value; # we parse this later
2456 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2458 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2461 if ($section eq 'cloudinit') {
2462 # ignore validation only used for informative purpose
2463 $conf->{$key} = $value;
2466 eval { $value = check_type
($key, $value); };
2468 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2470 $key = 'ide2' if $key eq 'cdrom';
2471 my $fmt = $confdesc->{$key}->{format
};
2472 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2473 my $v = parse_drive
($key, $value);
2474 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2475 $v->{file
} = $volid;
2476 $value = print_drive
($v);
2478 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2483 $conf->{$key} = $value;
2486 $handle_error->("vm $vmid - unable to parse config: $line\n");
2490 $finish_description->();
2491 delete $res->{snapstate
}; # just to be sure
2496 sub write_vm_config
{
2497 my ($filename, $conf) = @_;
2499 delete $conf->{snapstate
}; # just to be sure
2501 if ($conf->{cdrom
}) {
2502 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2503 $conf->{ide2
} = $conf->{cdrom
};
2504 delete $conf->{cdrom
};
2507 # we do not use 'smp' any longer
2508 if ($conf->{sockets
}) {
2509 delete $conf->{smp
};
2510 } elsif ($conf->{smp
}) {
2511 $conf->{sockets
} = $conf->{smp
};
2512 delete $conf->{cores
};
2513 delete $conf->{smp
};
2516 my $used_volids = {};
2518 my $cleanup_config = sub {
2519 my ($cref, $pending, $snapname) = @_;
2521 foreach my $key (keys %$cref) {
2522 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2523 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2524 my $value = $cref->{$key};
2525 if ($key eq 'delete') {
2526 die "propertry 'delete' is only allowed in [PENDING]\n"
2528 # fixme: check syntax?
2531 eval { $value = check_type
($key, $value); };
2532 die "unable to parse value of '$key' - $@" if $@;
2534 $cref->{$key} = $value;
2536 if (!$snapname && is_valid_drivename
($key)) {
2537 my $drive = parse_drive
($key, $value);
2538 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2543 &$cleanup_config($conf);
2545 &$cleanup_config($conf->{pending
}, 1);
2547 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2548 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2549 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2552 # remove 'unusedX' settings if we re-add a volume
2553 foreach my $key (keys %$conf) {
2554 my $value = $conf->{$key};
2555 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2556 delete $conf->{$key};
2560 my $generate_raw_config = sub {
2561 my ($conf, $pending) = @_;
2565 # add description as comment to top of file
2566 if (defined(my $descr = $conf->{description
})) {
2568 foreach my $cl (split(/\n/, $descr)) {
2569 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2572 $raw .= "#\n" if $pending;
2576 foreach my $key (sort keys %$conf) {
2577 next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2578 $raw .= "$key: $conf->{$key}\n";
2583 my $raw = &$generate_raw_config($conf);
2585 if (scalar(keys %{$conf->{pending
}})){
2586 $raw .= "\n[PENDING]\n";
2587 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2590 if (scalar(keys %{$conf->{cloudinit
}}) && PVE
::QemuConfig-
>has_cloudinit($conf)){
2591 $raw .= "\n[special:cloudinit]\n";
2592 $raw .= &$generate_raw_config($conf->{cloudinit
});
2595 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2596 $raw .= "\n[$snapname]\n";
2597 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2607 # we use static defaults from our JSON schema configuration
2608 foreach my $key (keys %$confdesc) {
2609 if (defined(my $default = $confdesc->{$key}->{default})) {
2610 $res->{$key} = $default;
2618 my $vmlist = PVE
::Cluster
::get_vmlist
();
2620 return $res if !$vmlist || !$vmlist->{ids
};
2621 my $ids = $vmlist->{ids
};
2622 my $nodename = nodename
();
2624 foreach my $vmid (keys %$ids) {
2625 my $d = $ids->{$vmid};
2626 next if !$d->{node
} || $d->{node
} ne $nodename;
2627 next if !$d->{type
} || $d->{type
} ne 'qemu';
2628 $res->{$vmid}->{exists} = 1;
2633 # test if VM uses local resources (to prevent migration)
2634 sub check_local_resources
{
2635 my ($conf, $noerr) = @_;
2638 my $mapped_res = [];
2640 my $nodelist = PVE
::Cluster
::get_nodelist
();
2641 my $pci_map = PVE
::Mapping
::PCI
::config
();
2642 my $usb_map = PVE
::Mapping
::USB
::config
();
2644 my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2646 my $add_missing_mapping = sub {
2647 my ($type, $key, $id) = @_;
2648 for my $node (@$nodelist) {
2650 if ($type eq 'pci') {
2651 $entry = PVE
::Mapping
::PCI
::get_node_mapping
($pci_map, $id, $node);
2652 } elsif ($type eq 'usb') {
2653 $entry = PVE
::Mapping
::USB
::get_node_mapping
($usb_map, $id, $node);
2655 if (!scalar($entry->@*)) {
2656 push @{$missing_mappings_by_node->{$node}}, $key;
2661 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2662 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2664 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2666 foreach my $k (keys %$conf) {
2667 if ($k =~ m/^usb/) {
2668 my $entry = parse_property_string
('pve-qm-usb', $conf->{$k});
2669 next if $entry->{host
} =~ m/^spice$/i;
2670 if ($entry->{mapping
}) {
2671 $add_missing_mapping->('usb', $k, $entry->{mapping
});
2672 push @$mapped_res, $k;
2675 if ($k =~ m/^hostpci/) {
2676 my $entry = parse_property_string
('pve-qm-hostpci', $conf->{$k});
2677 if ($entry->{mapping
}) {
2678 $add_missing_mapping->('pci', $k, $entry->{mapping
});
2679 push @$mapped_res, $k;
2682 # sockets are safe: they will recreated be on the target side post-migrate
2683 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2684 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2687 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2689 return wantarray ?
(\
@loc_res, $mapped_res, $missing_mappings_by_node) : \
@loc_res;
2692 # check if used storages are available on all nodes (use by migrate)
2693 sub check_storage_availability
{
2694 my ($storecfg, $conf, $node) = @_;
2696 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2697 my ($ds, $drive) = @_;
2699 my $volid = $drive->{file
};
2702 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2705 # check if storage is available on both nodes
2706 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2707 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2709 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2711 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2712 if !$scfg->{content
}->{$vtype};
2716 # list nodes where all VM images are available (used by has_feature API)
2718 my ($conf, $storecfg) = @_;
2720 my $nodelist = PVE
::Cluster
::get_nodelist
();
2721 my $nodehash = { map { $_ => 1 } @$nodelist };
2722 my $nodename = nodename
();
2724 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2725 my ($ds, $drive) = @_;
2727 my $volid = $drive->{file
};
2730 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2732 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2733 if ($scfg->{disable
}) {
2735 } elsif (my $avail = $scfg->{nodes
}) {
2736 foreach my $node (keys %$nodehash) {
2737 delete $nodehash->{$node} if !$avail->{$node};
2739 } elsif (!$scfg->{shared
}) {
2740 foreach my $node (keys %$nodehash) {
2741 delete $nodehash->{$node} if $node ne $nodename
2750 sub check_local_storage_availability
{
2751 my ($conf, $storecfg) = @_;
2753 my $nodelist = PVE
::Cluster
::get_nodelist
();
2754 my $nodehash = { map { $_ => {} } @$nodelist };
2756 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2757 my ($ds, $drive) = @_;
2759 my $volid = $drive->{file
};
2762 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2764 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2766 if ($scfg->{disable
}) {
2767 foreach my $node (keys %$nodehash) {
2768 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2770 } elsif (my $avail = $scfg->{nodes
}) {
2771 foreach my $node (keys %$nodehash) {
2772 if (!$avail->{$node}) {
2773 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2780 foreach my $node (values %$nodehash) {
2781 if (my $unavail = $node->{unavailable_storages
}) {
2782 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2789 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2791 my ($vmid, $nocheck, $node) = @_;
2793 # $nocheck is set when called during a migration, in which case the config
2794 # file might still or already reside on the *other* node
2795 # - because rename has already happened, and current node is source
2796 # - because rename hasn't happened yet, and current node is target
2797 # - because rename has happened, current node is target, but hasn't yet
2799 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2800 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2805 my $vzlist = config_list
();
2807 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2809 while (defined(my $de = $fd->read)) {
2810 next if $de !~ m/^(\d+)\.pid$/;
2812 next if !defined($vzlist->{$vmid});
2813 if (my $pid = check_running
($vmid)) {
2814 $vzlist->{$vmid}->{pid
} = $pid;
2821 our $vmstatus_return_properties = {
2822 vmid
=> get_standard_option
('pve-vmid'),
2824 description
=> "QEMU process status.",
2826 enum
=> ['stopped', 'running'],
2829 description
=> "Maximum memory in bytes.",
2832 renderer
=> 'bytes',
2835 description
=> "Root disk size in bytes.",
2838 renderer
=> 'bytes',
2841 description
=> "VM name.",
2846 description
=> "VM run state from the 'query-status' QMP monitor command.",
2851 description
=> "PID of running qemu process.",
2856 description
=> "Uptime.",
2859 renderer
=> 'duration',
2862 description
=> "Maximum usable CPUs.",
2867 description
=> "The current config lock, if any.",
2872 description
=> "The current configured tags, if any",
2876 'running-machine' => {
2877 description
=> "The currently running machine type (if running).",
2882 description
=> "The currently running QEMU version (if running).",
2888 my $last_proc_pid_stat;
2890 # get VM status information
2891 # This must be fast and should not block ($full == false)
2892 # We only query KVM using QMP if $full == true (this can be slow)
2894 my ($opt_vmid, $full) = @_;
2898 my $storecfg = PVE
::Storage
::config
();
2900 my $list = vzlist
();
2901 my $defaults = load_defaults
();
2903 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2905 my $cpucount = $cpuinfo->{cpus
} || 1;
2907 foreach my $vmid (keys %$list) {
2908 next if $opt_vmid && ($vmid ne $opt_vmid);
2910 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2912 my $d = { vmid
=> int($vmid) };
2913 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2915 # fixme: better status?
2916 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2918 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2919 if (defined($size)) {
2920 $d->{disk
} = 0; # no info available
2921 $d->{maxdisk
} = $size;
2927 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2928 * ($conf->{cores
} || $defaults->{cores
});
2929 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2930 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2932 $d->{name
} = $conf->{name
} || "VM $vmid";
2933 $d->{maxmem
} = get_current_memory
($conf->{memory
})*(1024*1024);
2935 if ($conf->{balloon
}) {
2936 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2937 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2938 : $defaults->{shares
};
2949 $d->{diskwrite
} = 0;
2951 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2953 $d->{serial
} = 1 if conf_has_serial
($conf);
2954 $d->{lock} = $conf->{lock} if $conf->{lock};
2955 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2960 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2961 foreach my $dev (keys %$netdev) {
2962 next if $dev !~ m/^tap([1-9]\d*)i/;
2964 my $d = $res->{$vmid};
2967 $d->{netout
} += $netdev->{$dev}->{receive
};
2968 $d->{netin
} += $netdev->{$dev}->{transmit
};
2971 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2972 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2977 my $ctime = gettimeofday
;
2979 foreach my $vmid (keys %$list) {
2981 my $d = $res->{$vmid};
2982 my $pid = $d->{pid
};
2985 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2986 next if !$pstat; # not running
2988 my $used = $pstat->{utime} + $pstat->{stime
};
2990 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2992 if ($pstat->{vsize
}) {
2993 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2996 my $old = $last_proc_pid_stat->{$pid};
2998 $last_proc_pid_stat->{$pid} = {
3006 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
3008 if ($dtime > 1000) {
3009 my $dutime = $used - $old->{used
};
3011 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
3012 $last_proc_pid_stat->{$pid} = {
3018 $d->{cpu
} = $old->{cpu
};
3022 return $res if !$full;
3024 my $qmpclient = PVE
::QMPClient-
>new();
3026 my $ballooncb = sub {
3027 my ($vmid, $resp) = @_;
3029 my $info = $resp->{'return'};
3030 return if !$info->{max_mem
};
3032 my $d = $res->{$vmid};
3034 # use memory assigned to VM
3035 $d->{maxmem
} = $info->{max_mem
};
3036 $d->{balloon
} = $info->{actual
};
3038 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
3039 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
3040 $d->{freemem
} = $info->{free_mem
};
3043 $d->{ballooninfo
} = $info;
3046 my $blockstatscb = sub {
3047 my ($vmid, $resp) = @_;
3048 my $data = $resp->{'return'} || [];
3049 my $totalrdbytes = 0;
3050 my $totalwrbytes = 0;
3052 for my $blockstat (@$data) {
3053 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
3054 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
3056 $blockstat->{device
} =~ s/drive-//;
3057 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
3059 $res->{$vmid}->{diskread
} = $totalrdbytes;
3060 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
3063 my $machinecb = sub {
3064 my ($vmid, $resp) = @_;
3065 my $data = $resp->{'return'} || [];
3067 $res->{$vmid}->{'running-machine'} =
3068 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
3071 my $versioncb = sub {
3072 my ($vmid, $resp) = @_;
3073 my $data = $resp->{'return'} // {};
3074 my $version = 'unknown';
3076 if (my $v = $data->{qemu
}) {
3077 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
3080 $res->{$vmid}->{'running-qemu'} = $version;
3083 my $statuscb = sub {
3084 my ($vmid, $resp) = @_;
3086 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3087 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3088 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3089 # this fails if ballon driver is not loaded, so this must be
3090 # the last commnand (following command are aborted if this fails).
3091 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3093 my $status = 'unknown';
3094 if (!defined($status = $resp->{'return'}->{status
})) {
3095 warn "unable to get VM status\n";
3099 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3102 foreach my $vmid (keys %$list) {
3103 next if $opt_vmid && ($vmid ne $opt_vmid);
3104 next if !$res->{$vmid}->{pid
}; # not running
3105 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3108 $qmpclient->queue_execute(undef, 2);
3110 foreach my $vmid (keys %$list) {
3111 next if $opt_vmid && ($vmid ne $opt_vmid);
3112 next if !$res->{$vmid}->{pid
}; #not running
3114 # we can't use the $qmpclient since it might have already aborted on
3115 # 'query-balloon', but this might also fail for older versions...
3116 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
3117 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3120 foreach my $vmid (keys %$list) {
3121 next if $opt_vmid && ($vmid ne $opt_vmid);
3122 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3128 sub conf_has_serial
{
3131 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3132 if ($conf->{"serial$i"}) {
3140 sub conf_has_audio
{
3141 my ($conf, $id) = @_;
3144 my $audio = $conf->{"audio$id"};
3145 return if !defined($audio);
3147 my $audioproperties = parse_property_string
($audio_fmt, $audio);
3148 my $audiodriver = $audioproperties->{driver
} // 'spice';
3151 dev
=> $audioproperties->{device
},
3152 dev_id
=> "audiodev$id",
3153 backend
=> $audiodriver,
3154 backend_id
=> "$audiodriver-backend${id}",
3159 my ($audio, $audiopciaddr, $machine_version) = @_;
3163 my $id = $audio->{dev_id
};
3165 if (min_version
($machine_version, 4, 2)) {
3166 $audiodev = ",audiodev=$audio->{backend_id}";
3169 if ($audio->{dev
} eq 'AC97') {
3170 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3171 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3172 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3173 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3174 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3176 die "unkown audio device '$audio->{dev}', implement me!";
3179 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3187 socket => "/var/run/qemu-server/$vmid.swtpm",
3188 pid
=> "/var/run/qemu-server/$vmid.swtpm.pid",
3192 sub add_tpm_device
{
3193 my ($vmid, $devices, $conf) = @_;
3195 return if !$conf->{tpmstate0
};
3197 my $paths = get_tpm_paths
($vmid);
3199 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3200 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3201 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3205 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3207 return if !$tpmdrive;
3210 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
3211 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
3213 $state = PVE
::Storage
::map_volume
($storecfg, $tpm->{file
});
3215 $state = $tpm->{file
};
3218 my $paths = get_tpm_paths
($vmid);
3220 # during migration, we will get state from remote
3223 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3230 "--create-platform-cert",
3233 "/etc/swtpm_setup.conf", # do not use XDG configs
3235 "0", # force creation as root, error if not possible
3236 "--not-overwrite", # ignore existing state, do not modify
3239 push @$setup_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3240 # TPM 2.0 supports ECC crypto, use if possible
3241 push @$setup_cmd, "--ecc" if $tpm->{version
} eq 'v2.0';
3243 run_command
($setup_cmd, outfunc
=> sub {
3244 print "swtpm_setup: $1\n";
3248 # Used to distinguish different invocations in the log.
3249 my $log_prefix = "[id=" . int(time()) . "] ";
3251 my $emulator_cmd = [
3255 "backend-uri=file://$state,mode=0600",
3257 "type=unixio,path=$paths->{socket},mode=0600",
3259 "file=$paths->{pid}",
3260 "--terminate", # terminate on QEMU disconnect
3263 "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3265 push @$emulator_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3266 run_command
($emulator_cmd, outfunc
=> sub { print $1; });
3268 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3269 while (! -e
$paths->{pid
}) {
3270 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3274 # return untainted PID of swtpm daemon so it can be killed on error
3275 file_read_firstline
($paths->{pid
}) =~ m/(\d+)/;
3279 sub vga_conf_has_spice
{
3282 my $vgaconf = parse_vga
($vga);
3283 my $vgatype = $vgaconf->{type
};
3284 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3291 return get_host_arch
() eq $arch;
3296 return $conf->{arch
} // get_host_arch
();
3299 my $default_machines = {
3304 sub get_installed_machine_version
{
3305 my ($kvmversion) = @_;
3306 $kvmversion = kvm_user_version
() if !defined($kvmversion);
3307 $kvmversion =~ m/^(\d+\.\d+)/;
3311 sub windows_get_pinned_machine_version
{
3312 my ($machine, $base_version, $kvmversion) = @_;
3314 my $pin_version = $base_version;
3315 if (!defined($base_version) ||
3316 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3318 $pin_version = get_installed_machine_version
($kvmversion);
3320 if (!$machine || $machine eq 'pc') {
3321 $machine = "pc-i440fx-$pin_version";
3322 } elsif ($machine eq 'q35') {
3323 $machine = "pc-q35-$pin_version";
3324 } elsif ($machine eq 'virt') {
3325 $machine = "virt-$pin_version";
3327 warn "unknown machine type '$machine', not touching that!\n";
3333 sub get_vm_machine
{
3334 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3336 my $machine = $forcemachine || $conf->{machine
};
3338 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3339 $kvmversion //= kvm_user_version
();
3340 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3341 # layout which confuses windows quite a bit and may result in various regressions..
3342 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3343 if (windows_version
($conf->{ostype
})) {
3344 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3347 $machine ||= $default_machines->{$arch};
3348 if ($add_pve_version) {
3349 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3350 $machine .= "+pve$pvever";
3354 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3355 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3356 $machine = $1 if $is_pxe;
3358 # for version-pinned machines that do not include a pve-version (e.g.
3359 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3360 $machine .= '+pve0';
3362 $machine .= '.pxe' if $is_pxe;
3368 sub get_ovmf_files
($$$) {
3369 my ($arch, $efidisk, $smm) = @_;
3371 my $types = $OVMF->{$arch}
3372 or die "no OVMF images known for architecture '$arch'\n";
3374 my $type = 'default';
3375 if ($arch eq 'x86_64') {
3376 if (defined($efidisk->{efitype
}) && $efidisk->{efitype
} eq '4m') {
3377 $type = $smm ?
"4m" : "4m-no-smm";
3378 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3380 # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3384 my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3385 die "EFI base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3386 die "EFI vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
3388 return ($ovmf_code, $ovmf_vars);
3392 aarch64
=> '/usr/bin/qemu-system-aarch64',
3393 x86_64
=> '/usr/bin/qemu-system-x86_64',
3395 sub get_command_for_arch
($) {
3397 return '/usr/bin/kvm' if is_native
($arch);
3399 my $cmd = $Arch2Qemu->{$arch}
3400 or die "don't know how to emulate architecture '$arch'\n";
3404 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3405 # to use in a QEMU command line (-cpu element), first array_intersect the result
3406 # of query_supported_ with query_understood_. This is necessary because:
3408 # a) query_understood_ returns flags the host cannot use and
3409 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3410 # flags, but CPU settings - with most of them being flags. Those settings
3411 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3413 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3414 # expensive. If you need the value returned from this, you can get it much
3415 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3416 # $accel being 'kvm' or 'tcg'.
3418 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3419 # changes, automatically populating pmxcfs.
3421 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3422 # since kvm and tcg machines support different flags
3424 sub query_supported_cpu_flags
{
3427 $arch //= get_host_arch
();
3428 my $default_machine = $default_machines->{$arch};
3432 # FIXME: Once this is merged, the code below should work for ARM as well:
3433 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3434 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3437 my $kvm_supported = defined(kvm_version
());
3438 my $qemu_cmd = get_command_for_arch
($arch);
3440 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3442 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3443 my $query_supported_run_qemu = sub {
3449 '-machine', $default_machine,
3451 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3452 '-mon', 'chardev=qmp,mode=control',
3453 '-pidfile', $pidfile,
3458 push @$cmd, '-accel', 'tcg';
3461 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3462 die "QEMU flag querying VM exited with code " . $rc if $rc;
3465 my $cmd_result = mon_cmd
(
3467 'query-cpu-model-expansion',
3469 model
=> { name
=> 'host' }
3472 my $props = $cmd_result->{model
}->{props
};
3473 foreach my $prop (keys %$props) {
3474 next if $props->{$prop} ne '1';
3475 # QEMU returns some flags multiple times, with '_', '.' or '-'
3476 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3477 # We only keep those with underscores, to match /proc/cpuinfo
3478 $prop =~ s/\.|-/_/g;
3479 $flags->{$prop} = 1;
3484 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3485 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3489 return [ sort keys %$flags ];
3492 # We need to query QEMU twice, since KVM and TCG have different supported flags
3493 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3494 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3495 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3497 if ($kvm_supported) {
3498 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3499 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3506 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3507 my $understood_cpu_flag_dir = "/usr/share/kvm";
3508 sub query_understood_cpu_flags
{
3509 my $arch = get_host_arch
();
3510 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3512 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3515 my $raw = file_get_contents
($filepath);
3516 $raw =~ s/^\s+|\s+$//g;
3517 my @flags = split(/\s+/, $raw);
3522 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3523 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3524 my sub should_disable_smm
{
3525 my ($conf, $vga, $machine) = @_;
3527 return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3529 return (!defined($conf->{bios
}) || $conf->{bios
} eq 'seabios') &&
3530 $vga->{type
} && $vga->{type
} =~ m/^(serial\d+|none)$/;
3533 my sub print_ovmf_drive_commandlines
{
3534 my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3536 my $d = $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
3538 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch, $d, $q35);
3540 my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3542 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3543 my ($path, $format) = $d->@{'file', 'format'};
3545 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3546 if (!defined($format)) {
3547 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3548 $format = qemu_img_format
($scfg, $volname);
3550 } elsif (!defined($format)) {
3551 die "efidisk format must be specified\n";
3553 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3554 if ($path =~ m/^rbd:/) {
3555 $var_drive_str .= ',cache=writeback';
3556 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3558 $var_drive_str .= ",format=$format,file=$path";
3560 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3561 $var_drive_str .= ',readonly=on' if drive_is_read_only
($conf, $d);
3563 log_warn
("no efidisk configured! Using temporary efivars disk.");
3564 my $path = "/tmp/$vmid-ovmf.fd";
3565 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3566 $var_drive_str .= ",format=raw,file=$path";
3567 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $version_guard->(4, 1, 2);
3570 return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3573 sub config_to_command
{
3574 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3577 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3580 my $ostype = $conf->{ostype
};
3581 my $winversion = windows_version
($ostype);
3582 my $kvm = $conf->{kvm
};
3583 my $nodename = nodename
();
3585 my $arch = get_vm_arch
($conf);
3586 my $kvm_binary = get_command_for_arch
($arch);
3587 my $kvmver = kvm_user_version
($kvm_binary);
3589 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3590 $kvmver //= "undefined";
3591 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3594 my $add_pve_version = min_version
($kvmver, 4, 1);
3596 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3597 my $machine_version = extract_version
($machine_type, $kvmver);
3598 $kvm //= 1 if is_native
($arch);
3600 $machine_version =~ m/(\d+)\.(\d+)/;
3601 my ($machine_major, $machine_minor) = ($1, $2);
3603 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3604 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3605 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3606 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3607 ." please upgrade node '$nodename'\n"
3608 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3609 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3610 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3611 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3612 ." node '$nodename'\n";
3615 # if a specific +pve version is required for a feature, use $version_guard
3616 # instead of min_version to allow machines to be run with the minimum
3618 my $required_pve_version = 0;
3619 my $version_guard = sub {
3620 my ($major, $minor, $pve) = @_;
3621 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3622 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3623 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3624 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3628 if ($kvm && !defined kvm_version
()) {
3629 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3630 ." or enable in BIOS.\n";
3633 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3634 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3635 my $use_old_bios_files = undef;
3636 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3639 if ($conf->{affinity
}) {
3640 push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity
};
3643 push @$cmd, $kvm_binary;
3645 push @$cmd, '-id', $vmid;
3647 my $vmname = $conf->{name
} || "vm$vmid";
3649 push @$cmd, '-name', "$vmname,debug-threads=on";
3651 push @$cmd, '-no-shutdown';
3655 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3656 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3657 push @$cmd, '-mon', "chardev=qmp,mode=control";
3659 if (min_version
($machine_version, 2, 12)) {
3660 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3661 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3664 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3666 push @$cmd, '-daemonize';
3668 if ($conf->{smbios1
}) {
3669 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3670 if ($smbios_conf->{base64
}) {
3671 # Do not pass base64 flag to qemu
3672 delete $smbios_conf->{base64
};
3673 my $smbios_string = "";
3674 foreach my $key (keys %$smbios_conf) {
3676 if ($key eq "uuid") {
3677 $value = $smbios_conf->{uuid
}
3679 $value = decode_base64
($smbios_conf->{$key});
3681 # qemu accepts any binary data, only commas need escaping by double comma
3683 $smbios_string .= "," . $key . "=" . $value if $value;
3685 push @$cmd, '-smbios', "type=1" . $smbios_string;
3687 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3691 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3692 my ($code_drive_str, $var_drive_str) =
3693 print_ovmf_drive_commandlines
($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3694 push $cmd->@*, '-drive', $code_drive_str;
3695 push $cmd->@*, '-drive', $var_drive_str;
3698 if ($q35) { # tell QEMU to load q35 config early
3699 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3700 if (min_version
($machine_version, 4, 0)) {
3701 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3703 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3707 if (defined(my $fixups = qemu_created_version_fixups
($conf, $forcemachine, $kvmver))) {
3708 push @$cmd, $fixups->@*;
3711 if ($conf->{vmgenid
}) {
3712 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3715 # add usb controllers
3716 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3717 $conf, $bridges, $arch, $machine_type, $machine_version);
3718 push @$devices, @usbcontrollers if @usbcontrollers;
3719 my $vga = parse_vga
($conf->{vga
});
3721 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3722 $vga->{type
} = 'qxl' if $qxlnum;
3724 if (!$vga->{type
}) {
3725 if ($arch eq 'aarch64') {
3726 $vga->{type
} = 'virtio';
3727 } elsif (min_version
($machine_version, 2, 9)) {
3728 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3730 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3734 # enable absolute mouse coordinates (needed by vnc)
3735 my $tablet = $conf->{tablet
};
3736 if (!defined($tablet)) {
3737 $tablet = $defaults->{tablet
};
3738 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3739 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3743 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3744 my $kbd = print_keyboarddevice_full
($conf, $arch);
3745 push @$devices, '-device', $kbd if defined($kbd);
3748 my $bootorder = device_bootorder
($conf);
3750 # host pci device passthrough
3751 my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3752 $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3755 my $usb_dev_features = {};
3756 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3758 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3759 $conf, $usb_dev_features, $bootorder, $machine_version);
3760 push @$devices, @usbdevices if @usbdevices;
3763 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3764 my $path = $conf->{"serial$i"} or next;
3765 if ($path eq 'socket') {
3766 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3767 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3768 # On aarch64, serial0 is the UART device. QEMU only allows
3769 # connecting UART devices via the '-serial' command line, as
3770 # the device has a fixed slot on the hardware...
3771 if ($arch eq 'aarch64' && $i == 0) {
3772 push @$devices, '-serial', "chardev:serial$i";
3774 push @$devices, '-device', "isa-serial,chardev=serial$i";
3777 die "no such serial device\n" if ! -c
$path;
3778 push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3779 push @$devices, '-device', "isa-serial,chardev=serial$i";
3784 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3785 if (my $path = $conf->{"parallel$i"}) {
3786 die "no such parallel device\n" if ! -c
$path;
3787 my $devtype = $path =~ m!^/dev/usb/lp! ?
'serial' : 'parallel';
3788 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3789 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3793 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3794 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3795 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3796 push @$devices, @$audio_devs;
3799 # Add a TPM only if the VM is not a template,
3800 # to support backing up template VMs even if the TPM disk is write-protected.
3801 add_tpm_device
($vmid, $devices, $conf) if (!PVE
::QemuConfig-
>is_template($conf));
3804 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3805 $sockets = $conf->{sockets
} if $conf->{sockets
};
3807 my $cores = $conf->{cores
} || 1;
3809 my $maxcpus = $sockets * $cores;
3811 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3813 my $allowed_vcpus = $cpuinfo->{cpus
};
3815 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3817 if ($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3818 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3819 for (my $i = 2; $i <= $vcpus; $i++) {
3820 my $cpustr = print_cpu_device
($conf,$i);
3821 push @$cmd, '-device', $cpustr;
3826 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3828 push @$cmd, '-nodefaults';
3830 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3832 push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3834 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3836 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3837 push @$devices, '-device', print_vga_device
(
3838 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3840 push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type
} eq 'virtio-gl'; # VIRGL
3842 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3843 push @$cmd, '-vnc', "unix:$socket,password=on";
3845 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3846 push @$cmd, '-nographic';
3850 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3851 my $useLocaltime = $conf->{localtime};
3853 if ($winversion >= 5) { # windows
3854 $useLocaltime = 1 if !defined($conf->{localtime});
3856 # use time drift fix when acpi is enabled
3857 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3858 $tdf = 1 if !defined($conf->{tdf
});
3862 if ($winversion >= 6) {
3863 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3864 push @$machineFlags, 'hpet=off';
3867 push @$rtcFlags, 'driftfix=slew' if $tdf;
3869 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3870 push @$rtcFlags, "base=$conf->{startdate}";
3871 } elsif ($useLocaltime) {
3872 push @$rtcFlags, 'base=localtime';
3876 push @$cmd, '-cpu', $forcecpu;
3878 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3881 PVE
::QemuServer
::Memory
::config
(
3882 $conf, $vmid, $sockets, $cores, $hotplug_features->{memory
}, $cmd);
3884 push @$cmd, '-S' if $conf->{freeze
};
3886 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3888 my $guest_agent = parse_guest_agent
($conf);
3890 if ($guest_agent->{enabled
}) {
3891 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3892 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3894 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3895 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3896 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3897 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3898 } elsif ($guest_agent->{type
} eq 'isa') {
3899 push @$devices, '-device', "isa-serial,chardev=qga0";
3903 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3904 if ($rng && $version_guard->(4, 1, 2)) {
3905 check_rng_source
($rng->{source
});
3907 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3908 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3909 my $limiter_str = "";
3911 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3914 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3915 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3916 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3921 assert_clipboard_config
($vga);
3922 my $is_spice = $qxlnum || $vga->{type
} =~ /^virtio/;
3924 if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3927 for (my $i = 1; $i < $qxlnum; $i++){
3928 push @$devices, '-device', print_vga_device
(
3929 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3932 # assume other OS works like Linux
3933 my ($ram, $vram) = ("134217728", "67108864");
3934 if ($vga->{memory
}) {
3935 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3936 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3938 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3939 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3943 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3945 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3946 if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3947 push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3949 push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3951 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3954 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3955 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3956 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3958 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3959 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3961 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3962 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3963 if ($spice_enhancement->{foldersharing
}) {
3964 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3965 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3968 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3969 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3970 if $spice_enhancement->{videostreaming
};
3971 push @$devices, '-spice', "$spice_opts";
3975 # enable balloon by default, unless explicitly disabled
3976 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3977 my $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3978 my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3979 $ballooncmd .= ",free-page-reporting=on" if min_version
($machine_version, 6, 2);
3980 push @$devices, '-device', $ballooncmd;
3983 if ($conf->{watchdog
}) {
3984 my $wdopts = parse_watchdog
($conf->{watchdog
});
3985 my $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3986 my $watchdog = $wdopts->{model
} || 'i6300esb';
3987 push @$devices, '-device', "$watchdog$pciaddr";
3988 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3992 my $scsicontroller = {};
3993 my $ahcicontroller = {};
3994 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3996 # Add iscsi initiator name if available
3997 if (my $initiator = get_initiator_name
()) {
3998 push @$devices, '-iscsi', "initiator-name=$initiator";
4001 PVE
::QemuConfig-
>foreach_volume($conf, sub {
4002 my ($ds, $drive) = @_;
4004 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
4005 check_volume_storage_type
($storecfg, $drive->{file
});
4006 push @$vollist, $drive->{file
};
4009 # ignore efidisk here, already added in bios/fw handling code above
4010 return if $drive->{interface
} eq 'efidisk';
4012 return if $drive->{interface
} eq 'tpmstate';
4014 $use_virtio = 1 if $ds =~ m/^virtio/;
4016 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
4018 if ($drive->{interface
} eq 'virtio'){
4019 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
4022 if ($drive->{interface
} eq 'scsi') {
4024 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
4026 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4027 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4029 my $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
4030 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
4033 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
4034 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4035 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4036 } elsif ($drive->{iothread
}) {
4038 "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4043 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
4044 $queues = ",num_queues=$drive->{queues}";
4047 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4048 if !$scsicontroller->{$controller};
4049 $scsicontroller->{$controller}=1;
4052 if ($drive->{interface
} eq 'sata') {
4053 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
4054 my $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
4055 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4056 if !$ahcicontroller->{$controller};
4057 $ahcicontroller->{$controller}=1;
4060 my $pbs_conf = $pbs_backing->{$ds};
4061 my $pbs_name = undef;
4063 $pbs_name = "drive-$ds-pbs";
4064 push @$devices, '-blockdev', print_pbs_blockdev
($pbs_conf, $pbs_name);
4067 my $drive_cmd = print_drive_commandline_full
(
4068 $storecfg, $vmid, $drive, $pbs_name, min_version
($kvmver, 6, 0));
4070 # extra protection for templates, but SATA and IDE don't support it..
4071 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
4073 push @$devices, '-drive',$drive_cmd;
4074 push @$devices, '-device', print_drivedevice_full
(
4075 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4078 for (my $i = 0; $i < $MAX_NETS; $i++) {
4079 my $netname = "net$i";
4081 next if !$conf->{$netname};
4082 my $d = parse_net
($conf->{$netname});
4084 # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4086 $use_virtio = 1 if $d->{model
} eq 'virtio';
4088 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
4090 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
4091 push @$devices, '-netdev', $netdevfull;
4093 my $netdevicefull = print_netdevice_full
(
4094 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4096 push @$devices, '-device', $netdevicefull;
4099 if ($conf->{ivshmem
}) {
4100 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4104 $bus = print_pcie_addr
("ivshmem");
4106 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4109 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4110 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4112 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4113 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4114 .",size=$ivshmem->{size}M";
4117 # pci.4 is nested in pci.1
4118 $bridges->{1} = 1 if $bridges->{4};
4120 if (!$q35) { # add pci bridges
4121 if (min_version
($machine_version, 2, 3)) {
4125 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4128 for my $k (sort {$b cmp $a} keys %$bridges) {
4129 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4132 if ($k == 2 && $legacy_igd) {
4135 my $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
4136 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4138 if ($q35) { # add after -readconfig pve-q35.cfg
4139 splice @$devices, 2, 0, '-device', $devstr;
4141 unshift @$devices, '-device', $devstr if $k > 0;
4146 push @$machineFlags, 'accel=tcg';
4149 push @$machineFlags, 'smm=off' if should_disable_smm
($conf, $vga, $machine_type);
4151 my $machine_type_min = $machine_type;
4152 if ($add_pve_version) {
4153 $machine_type_min =~ s/\+pve\d+$//;
4154 $machine_type_min .= "+pve$required_pve_version";
4156 push @$machineFlags, "type=${machine_type_min}";
4158 push @$cmd, @$devices;
4159 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4160 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4161 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4163 if (my $vmstate = $conf->{vmstate
}) {
4164 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4165 push @$vollist, $vmstate;
4166 push @$cmd, '-loadstate', $statepath;
4167 print "activating and using '$vmstate' as vmstate\n";
4170 if (PVE
::QemuConfig-
>is_template($conf)) {
4171 # needed to workaround base volumes being read-only
4172 push @$cmd, '-snapshot';
4176 if ($conf->{args
}) {
4177 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4181 return wantarray ?
($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4184 sub check_rng_source
{
4187 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4188 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4191 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4192 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
4193 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4194 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4195 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4196 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4204 my $res = mon_cmd
($vmid, 'query-spice');
4206 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4209 sub vm_devices_list
{
4212 my $res = mon_cmd
($vmid, 'query-pci');
4213 my $devices_to_check = [];
4215 foreach my $pcibus (@$res) {
4216 push @$devices_to_check, @{$pcibus->{devices
}},
4219 while (@$devices_to_check) {
4221 for my $d (@$devices_to_check) {
4222 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4223 next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices
};
4225 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4226 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4228 $devices_to_check = $to_check;
4231 my $resblock = mon_cmd
($vmid, 'query-block');
4232 foreach my $block (@$resblock) {
4233 if($block->{device
} =~ m/^drive-(\S+)/){
4238 my $resmice = mon_cmd
($vmid, 'query-mice');
4239 foreach my $mice (@$resmice) {
4240 if ($mice->{name
} eq 'QEMU HID Tablet') {
4241 $devices->{tablet
} = 1;
4246 # for usb devices there is no query-usb
4247 # but we can iterate over the entries in
4248 # qom-list path=/machine/peripheral
4249 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4250 foreach my $per (@$resperipheral) {
4251 if ($per->{name
} =~ m/^usb(?:redirdev)?\d+$/) {
4252 $devices->{$per->{name
}} = 1;
4260 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4262 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4264 my $devices_list = vm_devices_list
($vmid);
4265 return 1 if defined($devices_list->{$deviceid});
4267 # add PCI bridge if we need it for the device
4268 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4270 if ($deviceid eq 'tablet') {
4271 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4272 } elsif ($deviceid eq 'keyboard') {
4273 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4274 } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4276 qemu_spice_usbredir_chardev_add
($vmid, "usbredirchardev$id");
4277 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_spice_usbdevice
($id, "xhci", $id + 1));
4278 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4279 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device, {}, $1 + 1));
4280 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4281 qemu_iothread_add
($vmid, $deviceid, $device);
4283 qemu_driveadd
($storecfg, $vmid, $device);
4284 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4286 qemu_deviceadd
($vmid, $devicefull);
4287 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4289 eval { qemu_drivedel
($vmid, $deviceid); };
4293 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4294 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4295 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4296 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4298 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4300 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4301 qemu_iothread_add
($vmid, $deviceid, $device);
4302 $devicefull .= ",iothread=iothread-$deviceid";
4305 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4306 $devicefull .= ",num_queues=$device->{queues}";
4309 qemu_deviceadd
($vmid, $devicefull);
4310 qemu_deviceaddverify
($vmid, $deviceid);
4311 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4312 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4313 qemu_driveadd
($storecfg, $vmid, $device);
4315 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4316 eval { qemu_deviceadd
($vmid, $devicefull); };
4318 eval { qemu_drivedel
($vmid, $deviceid); };
4322 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4323 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4325 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4326 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type);
4327 my $use_old_bios_files = undef;
4328 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4330 my $netdevicefull = print_netdevice_full
(
4331 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4332 qemu_deviceadd
($vmid, $netdevicefull);
4334 qemu_deviceaddverify
($vmid, $deviceid);
4335 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4338 eval { qemu_netdevdel
($vmid, $deviceid); };
4342 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4344 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4345 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4347 qemu_deviceadd
($vmid, $devicefull);
4348 qemu_deviceaddverify
($vmid, $deviceid);
4350 die "can't hotplug device '$deviceid'\n";
4356 # fixme: this should raise exceptions on error!
4357 sub vm_deviceunplug
{
4358 my ($vmid, $conf, $deviceid) = @_;
4360 my $devices_list = vm_devices_list
($vmid);
4361 return 1 if !defined($devices_list->{$deviceid});
4363 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4364 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4366 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4367 qemu_devicedel
($vmid, $deviceid);
4368 } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4369 qemu_devicedel
($vmid, $deviceid);
4370 qemu_devicedelverify
($vmid, $deviceid);
4371 } elsif ($deviceid =~ m/^usb\d+$/) {
4372 qemu_devicedel
($vmid, $deviceid);
4373 qemu_devicedelverify
($vmid, $deviceid);
4374 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4375 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4377 qemu_devicedel
($vmid, $deviceid);
4378 qemu_devicedelverify
($vmid, $deviceid);
4379 qemu_drivedel
($vmid, $deviceid);
4380 qemu_iothread_del
($vmid, $deviceid, $device);
4381 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4382 qemu_devicedel
($vmid, $deviceid);
4383 qemu_devicedelverify
($vmid, $deviceid);
4384 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4385 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4387 qemu_devicedel
($vmid, $deviceid);
4388 qemu_devicedelverify
($vmid, $deviceid);
4389 qemu_drivedel
($vmid, $deviceid);
4390 qemu_deletescsihw
($conf, $vmid, $deviceid);
4392 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4393 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4394 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4395 qemu_devicedel
($vmid, $deviceid);
4396 qemu_devicedelverify
($vmid, $deviceid);
4397 qemu_netdevdel
($vmid, $deviceid);
4399 die "can't unplug device '$deviceid'\n";
4405 sub qemu_spice_usbredir_chardev_add
{
4406 my ($vmid, $id) = @_;
4408 mon_cmd
($vmid, "chardev-add" , (
4419 sub qemu_iothread_add
{
4420 my ($vmid, $deviceid, $device) = @_;
4422 if ($device->{iothread
}) {
4423 my $iothreads = vm_iothreads_list
($vmid);
4424 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4428 sub qemu_iothread_del
{
4429 my ($vmid, $deviceid, $device) = @_;
4431 if ($device->{iothread
}) {
4432 my $iothreads = vm_iothreads_list
($vmid);
4433 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4438 my ($storecfg, $vmid, $device) = @_;
4440 my $kvmver = get_running_qemu_version
($vmid);
4441 my $io_uring = min_version
($kvmver, 6, 0);
4442 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4443 $drive =~ s/\\/\\\\/g;
4444 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4446 # If the command succeeds qemu prints: "OK
"
4447 return 1 if $ret =~ m/OK/s;
4449 die "adding drive failed
: $ret\n";
4453 my ($vmid, $deviceid) = @_;
4455 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4458 return 1 if $ret eq "";
4460 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4461 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4463 die "deleting drive
$deviceid failed
: $ret\n";
4466 sub qemu_deviceaddverify {
4467 my ($vmid, $deviceid) = @_;
4469 for (my $i = 0; $i <= 5; $i++) {
4470 my $devices_list = vm_devices_list($vmid);
4471 return 1 if defined($devices_list->{$deviceid});
4475 die "error on hotplug device
'$deviceid'\n";
4479 sub qemu_devicedelverify {
4480 my ($vmid, $deviceid) = @_;
4482 # need to verify that the device is correctly removed as device_del
4483 # is async and empty return is not reliable
4485 for (my $i = 0; $i <= 5; $i++) {
4486 my $devices_list = vm_devices_list($vmid);
4487 return 1 if !defined($devices_list->{$deviceid});
4491 die "error on hot-unplugging device
'$deviceid'\n";
4494 sub qemu_findorcreatescsihw {
4495 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4497 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4499 my $scsihwid="$controller_prefix$controller";
4500 my $devices_list = vm_devices_list($vmid);
4502 if (!defined($devices_list->{$scsihwid})) {
4503 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4509 sub qemu_deletescsihw {
4510 my ($conf, $vmid, $opt) = @_;
4512 my $device = parse_drive($opt, $conf->{$opt});
4514 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4515 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4519 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4521 my $devices_list = vm_devices_list($vmid);
4522 foreach my $opt (keys %{$devices_list}) {
4523 if (is_valid_drivename($opt)) {
4524 my $drive = parse_drive($opt, $conf->{$opt});
4525 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4531 my $scsihwid="scsihw
$controller";
4533 vm_deviceunplug($vmid, $conf, $scsihwid);
4538 sub qemu_add_pci_bridge {
4539 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4545 print_pci_addr($device, $bridges, $arch, $machine_type);
4547 while (my ($k, $v) = each %$bridges) {
4550 return 1 if !defined($bridgeid) || $bridgeid < 1;
4552 my $bridge = "pci
.$bridgeid";
4553 my $devices_list = vm_devices_list($vmid);
4555 if (!defined($devices_list->{$bridge})) {
4556 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4562 sub qemu_set_link_status {
4563 my ($vmid, $device, $up) = @_;
4565 mon_cmd($vmid, "set_link
", name => $device,
4566 up => $up ? JSON::true : JSON::false);
4569 sub qemu_netdevadd {
4570 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4572 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4573 my %options = split(/[=,]/, $netdev);
4575 if (defined(my $vhost = $options{vhost})) {
4576 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4579 if (defined(my $queues = $options{queues})) {
4580 $options{queues} = $queues + 0;
4583 mon_cmd($vmid, "netdev_add
", %options);
4587 sub qemu_netdevdel {
4588 my ($vmid, $deviceid) = @_;
4590 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4593 sub qemu_usb_hotplug {
4594 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4598 # remove the old one first
4599 vm_deviceunplug($vmid, $conf, $deviceid);
4601 # check if xhci controller is necessary and available
4602 my $devicelist = vm_devices_list($vmid);
4604 if (!$devicelist->{xhci}) {
4605 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4606 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4610 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4613 sub qemu_cpu_hotplug {
4614 my ($vmid, $conf, $vcpus) = @_;
4616 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4619 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4620 $sockets = $conf->{sockets} if $conf->{sockets};
4621 my $cores = $conf->{cores} || 1;
4622 my $maxcpus = $sockets * $cores;
4624 $vcpus = $maxcpus if !$vcpus;
4626 die "you can
't add more vcpus than maxcpus\n"
4627 if $vcpus > $maxcpus;
4629 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4631 if ($vcpus < $currentvcpus) {
4633 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4635 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4636 qemu_devicedel($vmid, "cpu$i");
4638 my $currentrunningvcpus = undef;
4640 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4641 last if scalar(@{$currentrunningvcpus}) == $i-1;
4642 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4646 #update conf after each succesfull cpu unplug
4647 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4648 PVE::QemuConfig->write_config($vmid, $conf);
4651 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4657 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4658 die "vcpus in running vm does not match its configuration\n"
4659 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4661 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4663 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4664 my $cpustr = print_cpu_device($conf, $i);
4665 qemu_deviceadd($vmid, $cpustr);
4668 my $currentrunningvcpus = undef;
4670 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4671 last if scalar(@{$currentrunningvcpus}) == $i;
4672 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4676 #update conf after each succesfull cpu hotplug
4677 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4678 PVE::QemuConfig->write_config($vmid, $conf);
4682 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4683 mon_cmd($vmid, "cpu-add", id => int($i));
4688 sub qemu_block_set_io_throttle {
4689 my ($vmid, $deviceid,
4690 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4691 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4692 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4693 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4695 return if !check_running($vmid) ;
4697 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4699 bps_rd => int($bps_rd),
4700 bps_wr => int($bps_wr),
4702 iops_rd => int($iops_rd),
4703 iops_wr => int($iops_wr),
4704 bps_max => int($bps_max),
4705 bps_rd_max => int($bps_rd_max),
4706 bps_wr_max => int($bps_wr_max),
4707 iops_max => int($iops_max),
4708 iops_rd_max => int($iops_rd_max),
4709 iops_wr_max => int($iops_wr_max),
4710 bps_max_length => int($bps_max_length),
4711 bps_rd_max_length => int($bps_rd_max_length),
4712 bps_wr_max_length => int($bps_wr_max_length),
4713 iops_max_length => int($iops_max_length),
4714 iops_rd_max_length => int($iops_rd_max_length),
4715 iops_wr_max_length => int($iops_wr_max_length),
4720 sub qemu_block_resize {
4721 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4723 my $running = check_running($vmid);
4725 PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4727 return if !$running;
4729 my $padding = (1024 - $size % 1024) % 1024;
4730 $size = $size + $padding;
4735 device => $deviceid,
4741 sub qemu_volume_snapshot {
4742 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4744 my $running = check_running($vmid);
4746 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4747 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4749 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4753 sub qemu_volume_snapshot_delete {
4754 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4756 my $running = check_running($vmid);
4761 my $conf = PVE::QemuConfig->load_config($vmid);
4762 PVE::QemuConfig->foreach_volume($conf, sub {
4763 my ($ds, $drive) = @_;
4764 $running = 1 if $drive->{file} eq $volid;
4768 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4769 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4771 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4775 sub set_migration_caps {
4776 my ($vmid, $savevm) = @_;
4778 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4780 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4781 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4786 "auto-converge" => 1,
4788 "x-rdma-pin-all" => 0,
4791 "dirty-bitmaps" => $dirty_bitmaps,
4794 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4796 for my $supported_capability (@$supported_capabilities) {
4798 capability => $supported_capability->{capability},
4799 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4803 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4807 my ($conf, $func, @param) = @_;
4811 my $test_volid = sub {
4812 my ($key, $drive, $snapname, $pending) = @_;
4814 my $volid = $drive->{file};
4817 $volhash->{$volid}->{cdrom} //= 1;
4818 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4820 my $replicate = $drive->{replicate} // 1;
4821 $volhash->{$volid}->{replicate} //= 0;
4822 $volhash->{$volid}->{replicate} = 1 if $replicate;
4824 $volhash->{$volid}->{shared} //= 0;
4825 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4827 $volhash->{$volid}->{is_unused} //= 0;
4828 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4830 $volhash->{$volid}->{is_attached} //= 0;
4831 $volhash->{$volid}->{is_attached} = 1
4832 if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4834 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4835 if defined($snapname);
4837 $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4839 my $size = $drive->{size};
4840 $volhash->{$volid}->{size} //= $size if $size;
4842 $volhash->{$volid}->{is_vmstate} //= 0;
4843 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4845 $volhash->{$volid}->{is_tpmstate} //= 0;
4846 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0
';
4848 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4851 my $include_opts = {
4852 extra_keys => ['vmstate
'],
4853 include_unused => 1,
4856 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4858 PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4859 if defined($conf->{pending}) && $conf->{pending}->%*;
4861 foreach my $snapname (keys %{$conf->{snapshots}}) {
4862 my $snap = $conf->{snapshots}->{$snapname};
4863 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4866 foreach my $volid (keys %$volhash) {
4867 &$func($volid, $volhash->{$volid}, @param);
4871 my $fast_plug_option = {
4875 'migrate_downtime
' => 1,
4876 'migrate_speed
' => 1,
4883 'vmstatestorage
' => 1,
4886 for my $opt (keys %$confdesc_cloudinit) {
4887 $fast_plug_option->{$opt} = 1;
4890 # hotplug changes in [PENDING]
4891 # $selection hash can be used to only apply specified options, for
4892 # example: { cores => 1 } (only apply changed 'cores
')
4893 # $errors ref is used to return error messages
4894 sub vmconfig_hotplug_pending {
4895 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4897 my $defaults = load_defaults();
4898 my $arch = get_vm_arch($conf);
4899 my $machine_type = get_vm_machine($conf, undef, $arch);
4901 # commit values which do not have any impact on running VM first
4902 # Note: those option cannot raise errors, we we do not care about
4903 # $selection and always apply them.
4905 my $add_error = sub {
4906 my ($opt, $msg) = @_;
4907 $errors->{$opt} = "hotplug problem - $msg";
4910 my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4912 my $cloudinit_record_changed = sub {
4913 my ($conf, $opt, $old, $new) = @_;
4914 return if !$cloudinit_pending_properties->{$opt};
4916 my $ci = ($conf->{cloudinit} //= {});
4918 my $recorded = $ci->{$opt};
4919 my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4921 if (defined($new)) {
4922 if (defined($old)) {
4923 # an existing value is being modified
4924 if (defined($recorded)) {
4925 # the value was already not in sync
4926 if ($new eq $recorded) {
4927 # a value is being reverted to the cloud-init state:
4929 delete $added{$opt};
4931 # the value was changed multiple times, do nothing
4933 } elsif ($added{$opt}) {
4934 # the value had been marked as added and is being changed, do nothing
4936 # the value is new, record it:
4940 # a new value is being added
4941 if (defined($recorded)) {
4942 # it was already not in sync
4943 if ($new eq $recorded) {
4944 # a value is being reverted to the cloud-init state:
4946 delete $added{$opt};
4948 # the value had temporarily been removed, do nothing
4950 } elsif ($added{$opt}) {
4951 # the value had been marked as added already, do nothing
4953 # the value is new, add it
4957 } elsif (!defined($old)) {
4958 # a non-existent value is being removed? ignore...
4960 # a value is being deleted
4961 if (defined($recorded)) {
4962 # a value was already recorded, just keep it
4963 } elsif ($added{$opt}) {
4964 # the value was marked as added, remove it
4965 delete $added{$opt};
4967 # a previously unrecorded value is being removed, record the old value:
4972 my $added = join(',', sort keys %added);
4973 $ci->{added} = $added if length($added);
4977 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4978 if ($fast_plug_option->{$opt}) {
4979 my $new = delete $conf->{pending}->{$opt};
4980 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4981 $conf->{$opt} = $new;
4987 PVE::QemuConfig->write_config($vmid, $conf);
4990 my $ostype = $conf->{ostype};
4991 my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4992 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4993 my $usb_hotplug = $hotplug_features->{usb}
4994 && min_version($version, 7, 1)
4995 && defined($ostype) && ($ostype eq 'l26
' || windows_version($ostype) > 7);
4997 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4998 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5000 foreach my $opt (sort keys %$pending_delete_hash) {
5001 next if $selection && !$selection->{$opt};
5002 my $force = $pending_delete_hash->{$opt}->{force};
5004 if ($opt eq 'hotplug
') {
5005 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
5006 } elsif ($opt eq 'tablet
') {
5007 die "skip\n" if !$hotplug_features->{usb};
5008 if ($defaults->{tablet}) {
5009 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
5010 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
5011 if $arch eq 'aarch64
';
5013 vm_deviceunplug($vmid, $conf, 'tablet
');
5014 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
5016 } elsif ($opt =~ m/^usb(\d+)$/) {
5018 die "skip\n" if !$usb_hotplug;
5019 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5020 vm_deviceunplug
($vmid, $conf, $opt);
5021 } elsif ($opt eq 'vcpus') {
5022 die "skip\n" if !$hotplug_features->{cpu
};
5023 qemu_cpu_hotplug
($vmid, $conf, undef);
5024 } elsif ($opt eq 'balloon') {
5025 # enable balloon device is not hotpluggable
5026 die "skip\n" if defined($conf->{balloon
}) && $conf->{balloon
} == 0;
5027 # here we reset the ballooning value to memory
5028 my $balloon = get_current_memory
($conf->{memory
});
5029 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5030 } elsif ($fast_plug_option->{$opt}) {
5032 } elsif ($opt =~ m/^net(\d+)$/) {
5033 die "skip\n" if !$hotplug_features->{network
};
5034 vm_deviceunplug
($vmid, $conf, $opt);
5036 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
5037 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
});
5039 } elsif (is_valid_drivename
($opt)) {
5040 die "skip\n" if !$hotplug_features->{disk
} || $opt =~ m/(ide|sata)(\d+)/;
5041 vm_deviceunplug
($vmid, $conf, $opt);
5042 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5043 } elsif ($opt =~ m/^memory$/) {
5044 die "skip\n" if !$hotplug_features->{memory
};
5045 PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf);
5046 } elsif ($opt eq 'cpuunits') {
5047 $cgroup->change_cpu_shares(undef);
5048 } elsif ($opt eq 'cpulimit') {
5049 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5055 &$add_error($opt, $err) if $err ne "skip\n";
5057 my $old = delete $conf->{$opt};
5058 $cloudinit_record_changed->($conf, $opt, $old, undef);
5059 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5064 foreach my $opt (keys %{$conf->{pending
}}) {
5065 next if $selection && !$selection->{$opt};
5066 my $value = $conf->{pending
}->{$opt};
5068 if ($opt eq 'hotplug') {
5069 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug
} =~ /memory/);
5070 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug
} =~ /cpu/);
5071 } elsif ($opt eq 'tablet') {
5072 die "skip\n" if !$hotplug_features->{usb
};
5074 vm_deviceplug
($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5075 vm_deviceplug
($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5076 if $arch eq 'aarch64';
5077 } elsif ($value == 0) {
5078 vm_deviceunplug
($vmid, $conf, 'tablet');
5079 vm_deviceunplug
($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5081 } elsif ($opt =~ m/^usb(\d+)$/) {
5083 die "skip\n" if !$usb_hotplug;
5084 my $d = eval { parse_property_string
('pve-qm-usb', $value) };
5086 if ($d->{host
} =~ m/^spice$/i) {
5087 $id = "usbredirdev$index";
5089 qemu_usb_hotplug
($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5090 } elsif ($opt eq 'vcpus') {
5091 die "skip\n" if !$hotplug_features->{cpu
};
5092 qemu_cpu_hotplug
($vmid, $conf, $value);
5093 } elsif ($opt eq 'balloon') {
5094 # enable/disable balloning device is not hotpluggable
5095 my $old_balloon_enabled = !!(!defined($conf->{balloon
}) || $conf->{balloon
});
5096 my $new_balloon_enabled = !!(!defined($conf->{pending
}->{balloon
}) || $conf->{pending
}->{balloon
});
5097 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5099 # allow manual ballooning if shares is set to zero
5100 if ((defined($conf->{shares
}) && ($conf->{shares
} == 0))) {
5101 my $memory = get_current_memory
($conf->{memory
});
5102 my $balloon = $conf->{pending
}->{balloon
} || $memory;
5103 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5105 } elsif ($opt =~ m/^net(\d+)$/) {
5106 # some changes can be done without hotplug
5107 vmconfig_update_net
($storecfg, $conf, $hotplug_features->{network
},
5108 $vmid, $opt, $value, $arch, $machine_type);
5109 } elsif (is_valid_drivename
($opt)) {
5110 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5111 # some changes can be done without hotplug
5112 my $drive = parse_drive
($opt, $value);
5113 if (drive_is_cloudinit
($drive)) {
5114 $cloudinit_opt = [$opt, $drive];
5115 # apply all the other changes first, then generate the cloudinit disk
5118 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5119 $vmid, $opt, $value, $arch, $machine_type);
5120 } elsif ($opt =~ m/^memory$/) { #dimms
5121 die "skip\n" if !$hotplug_features->{memory
};
5122 $value = PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $value);
5123 } elsif ($opt eq 'cpuunits') {
5124 my $new_cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{pending
}->{$opt}); #clamp
5125 $cgroup->change_cpu_shares($new_cpuunits);
5126 } elsif ($opt eq 'cpulimit') {
5127 my $cpulimit = $conf->{pending
}->{$opt} == 0 ?
-1 : int($conf->{pending
}->{$opt} * 100000);
5128 $cgroup->change_cpu_quota($cpulimit, 100000);
5129 } elsif ($opt eq 'agent') {
5130 vmconfig_update_agent
($conf, $opt, $value);
5132 die "skip\n"; # skip non-hot-pluggable options
5136 &$add_error($opt, $err) if $err ne "skip\n";
5138 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5139 $conf->{$opt} = $value;
5140 delete $conf->{pending
}->{$opt};
5144 if (defined($cloudinit_opt)) {
5145 my ($opt, $drive) = @$cloudinit_opt;
5146 my $value = $conf->{pending
}->{$opt};
5148 my $temp = {%$conf, $opt => $value};
5149 PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($temp, $vmid);
5150 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5151 $vmid, $opt, $value, $arch, $machine_type);
5154 &$add_error($opt, $err) if $err ne "skip\n";
5156 $conf->{$opt} = $value;
5157 delete $conf->{pending
}->{$opt};
5161 # unplug xhci controller if no usb device is left
5164 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
5165 next if !defined($conf->{"usb$i"});
5170 vm_deviceunplug
($vmid, $conf, 'xhci');
5174 PVE
::QemuConfig-
>write_config($vmid, $conf);
5176 if ($hotplug_features->{cloudinit
} && PVE
::QemuServer
::Cloudinit
::has_changes
($conf)) {
5177 PVE
::QemuServer
::vmconfig_update_cloudinit_drive
($storecfg, $conf, $vmid);
5181 sub try_deallocate_drive
{
5182 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5184 if (($force || $key =~ /^unused/) && !drive_is_cdrom
($drive, 1)) {
5185 my $volid = $drive->{file
};
5186 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
5187 my $sid = PVE
::Storage
::parse_volume_id
($volid);
5188 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5190 # check if the disk is really unused
5191 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5192 if PVE
::QemuServer
::Drive
::is_volume_in_use
($storecfg, $conf, $key, $volid);
5193 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5196 # If vm is not owner of this disk remove from config
5204 sub vmconfig_delete_or_detach_drive
{
5205 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5207 my $drive = parse_drive
($opt, $conf->{$opt});
5209 my $rpcenv = PVE
::RPCEnvironment
::get
();
5210 my $authuser = $rpcenv->get_user();
5213 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5214 try_deallocate_drive
($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5216 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $drive);
5222 sub vmconfig_apply_pending
{
5223 my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5225 return if !scalar(keys %{$conf->{pending
}});
5227 my $add_apply_error = sub {
5228 my ($opt, $msg) = @_;
5229 my $err_msg = "unable to apply pending change $opt : $msg";
5230 $errors->{$opt} = $err_msg;
5236 my $pending_delete_hash = PVE
::QemuConfig-
>parse_pending_delete($conf->{pending
}->{delete});
5237 foreach my $opt (sort keys %$pending_delete_hash) {
5238 my $force = $pending_delete_hash->{$opt}->{force
};
5240 if ($opt =~ m/^unused/) {
5241 die "internal error";
5242 } elsif (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5243 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5244 } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5246 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
5247 eval { PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
}) };
5253 $add_apply_error->($opt, $err);
5255 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5256 delete $conf->{$opt};
5260 PVE
::QemuConfig-
>cleanup_pending($conf);
5262 my $generate_cloudinit = $skip_cloud_init ?
0 : undef;
5264 foreach my $opt (keys %{$conf->{pending
}}) { # add/change
5265 next if $opt eq 'delete'; # just to be sure
5267 if (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5268 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, parse_drive
($opt, $conf->{$opt}))
5269 } elsif (defined($conf->{pending
}->{$opt}) && $opt =~ m/^net\d+$/) {
5271 my $new_net = PVE
::QemuServer
::parse_net
($conf->{pending
}->{$opt});
5273 my $old_net = PVE
::QemuServer
::parse_net
($conf->{$opt});
5275 if ($old_net->{bridge
} ne $new_net->{bridge
} ||
5276 $old_net->{macaddr
} ne $new_net->{macaddr
}) {
5277 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($old_net->{bridge
}, $old_net->{macaddr
}, $conf->{name
});
5280 #fixme: reuse ip if mac change && same bridge
5281 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($new_net->{bridge
}, $conf->{name
}, $new_net->{macaddr
}, $vmid, undef, 1);
5286 $add_apply_error->($opt, $err);
5289 if (is_valid_drivename
($opt)) {
5290 my $drive = parse_drive
($opt, $conf->{pending
}->{$opt});
5291 $generate_cloudinit //= 1 if drive_is_cloudinit
($drive);
5294 $conf->{$opt} = delete $conf->{pending
}->{$opt};
5298 # write all changes at once to avoid unnecessary i/o
5299 PVE
::QemuConfig-
>write_config($vmid, $conf);
5300 if ($generate_cloudinit) {
5301 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5302 # After successful generation and if there were changes to be applied, update the
5303 # config to drop the {cloudinit} entry.
5304 PVE
::QemuConfig-
>write_config($vmid, $conf);
5309 sub vmconfig_update_net
{
5310 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5312 my $newnet = parse_net
($value);
5314 if ($conf->{$opt}) {
5315 my $oldnet = parse_net
($conf->{$opt});
5317 if (safe_string_ne
($oldnet->{model
}, $newnet->{model
}) ||
5318 safe_string_ne
($oldnet->{macaddr
}, $newnet->{macaddr
}) ||
5319 safe_num_ne
($oldnet->{queues
}, $newnet->{queues
}) ||
5320 safe_num_ne
($oldnet->{mtu
}, $newnet->{mtu
}) ||
5321 !($newnet->{bridge
} && $oldnet->{bridge
})) { # bridge/nat mode change
5323 # for non online change, we try to hot-unplug
5324 die "skip\n" if !$hotplug;
5325 vm_deviceunplug
($vmid, $conf, $opt);
5328 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($oldnet->{bridge
}, $oldnet->{macaddr
}, $conf->{name
});
5333 die "internal error" if $opt !~ m/net(\d+)/;
5334 my $iface = "tap${vmid}i$1";
5336 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5337 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
}) ||
5338 safe_string_ne
($oldnet->{trunks
}, $newnet->{trunks
}) ||
5339 safe_num_ne
($oldnet->{firewall
}, $newnet->{firewall
})) {
5340 PVE
::Network
::tap_unplug
($iface);
5342 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
})) {
5344 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($oldnet->{bridge
}, $oldnet->{macaddr
}, $conf->{name
});
5345 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($newnet->{bridge
}, $conf->{name
}, $newnet->{macaddr
}, $vmid, undef, 1);
5350 PVE
::Network
::SDN
::Zones
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5352 PVE
::Network
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5354 } elsif (safe_num_ne
($oldnet->{rate
}, $newnet->{rate
})) {
5355 # Rate can be applied on its own but any change above needs to
5356 # include the rate in tap_plug since OVS resets everything.
5357 PVE
::Network
::tap_rate_limit
($iface, $newnet->{rate
});
5360 if (safe_string_ne
($oldnet->{link_down
}, $newnet->{link_down
})) {
5361 qemu_set_link_status
($vmid, $opt, !$newnet->{link_down
});
5370 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($newnet->{bridge
}, $conf->{name
}, $newnet->{macaddr
}, $vmid, undef, 1);
5372 PVE
::Network
::SDN
::Vnets
::add_dhcp_mapping
($newnet->{bridge
}, $newnet->{macaddr
});
5373 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5379 sub vmconfig_update_agent
{
5380 my ($conf, $opt, $value) = @_;
5382 die "skip\n" if !$conf->{$opt};
5384 my $hotplug_options = { fstrim_cloned_disks
=> 1 };
5386 my $old_agent = parse_guest_agent
($conf);
5387 my $agent = parse_guest_agent
({$opt => $value});
5389 for my $option (keys %$agent) { # added/changed options
5390 next if defined($hotplug_options->{$option});
5391 die "skip\n" if safe_string_ne
($agent->{$option}, $old_agent->{$option});
5394 for my $option (keys %$old_agent) { # removed options
5395 next if defined($hotplug_options->{$option});
5396 die "skip\n" if safe_string_ne
($old_agent->{$option}, $agent->{$option});
5399 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5402 sub vmconfig_update_disk
{
5403 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5405 my $drive = parse_drive
($opt, $value);
5407 if ($conf->{$opt} && (my $old_drive = parse_drive
($opt, $conf->{$opt}))) {
5408 my $media = $drive->{media
} || 'disk';
5409 my $oldmedia = $old_drive->{media
} || 'disk';
5410 die "unable to change media type\n" if $media ne $oldmedia;
5412 if (!drive_is_cdrom
($old_drive)) {
5414 if ($drive->{file
} ne $old_drive->{file
}) {
5416 die "skip\n" if !$hotplug;
5418 # unplug and register as unused
5419 vm_deviceunplug
($vmid, $conf, $opt);
5420 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive)
5423 # update existing disk
5425 # skip non hotpluggable value
5426 if (safe_string_ne
($drive->{aio
}, $old_drive->{aio
}) ||
5427 safe_string_ne
($drive->{discard
}, $old_drive->{discard
}) ||
5428 safe_string_ne
($drive->{iothread
}, $old_drive->{iothread
}) ||
5429 safe_string_ne
($drive->{queues
}, $old_drive->{queues
}) ||
5430 safe_string_ne
($drive->{cache
}, $old_drive->{cache
}) ||
5431 safe_string_ne
($drive->{ssd
}, $old_drive->{ssd
}) ||
5432 safe_string_ne
($drive->{ro
}, $old_drive->{ro
})) {
5437 if (safe_num_ne
($drive->{mbps
}, $old_drive->{mbps
}) ||
5438 safe_num_ne
($drive->{mbps_rd
}, $old_drive->{mbps_rd
}) ||
5439 safe_num_ne
($drive->{mbps_wr
}, $old_drive->{mbps_wr
}) ||
5440 safe_num_ne
($drive->{iops
}, $old_drive->{iops
}) ||
5441 safe_num_ne
($drive->{iops_rd
}, $old_drive->{iops_rd
}) ||
5442 safe_num_ne
($drive->{iops_wr
}, $old_drive->{iops_wr
}) ||
5443 safe_num_ne
($drive->{mbps_max
}, $old_drive->{mbps_max
}) ||
5444 safe_num_ne
($drive->{mbps_rd_max
}, $old_drive->{mbps_rd_max
}) ||
5445 safe_num_ne
($drive->{mbps_wr_max
}, $old_drive->{mbps_wr_max
}) ||
5446 safe_num_ne
($drive->{iops_max
}, $old_drive->{iops_max
}) ||
5447 safe_num_ne
($drive->{iops_rd_max
}, $old_drive->{iops_rd_max
}) ||
5448 safe_num_ne
($drive->{iops_wr_max
}, $old_drive->{iops_wr_max
}) ||
5449 safe_num_ne
($drive->{bps_max_length
}, $old_drive->{bps_max_length
}) ||
5450 safe_num_ne
($drive->{bps_rd_max_length
}, $old_drive->{bps_rd_max_length
}) ||
5451 safe_num_ne
($drive->{bps_wr_max_length
}, $old_drive->{bps_wr_max_length
}) ||
5452 safe_num_ne
($drive->{iops_max_length
}, $old_drive->{iops_max_length
}) ||
5453 safe_num_ne
($drive->{iops_rd_max_length
}, $old_drive->{iops_rd_max_length
}) ||
5454 safe_num_ne
($drive->{iops_wr_max_length
}, $old_drive->{iops_wr_max_length
})) {
5456 qemu_block_set_io_throttle
(
5458 ($drive->{mbps
} || 0)*1024*1024,
5459 ($drive->{mbps_rd
} || 0)*1024*1024,
5460 ($drive->{mbps_wr
} || 0)*1024*1024,
5461 $drive->{iops
} || 0,
5462 $drive->{iops_rd
} || 0,
5463 $drive->{iops_wr
} || 0,
5464 ($drive->{mbps_max
} || 0)*1024*1024,
5465 ($drive->{mbps_rd_max
} || 0)*1024*1024,
5466 ($drive->{mbps_wr_max
} || 0)*1024*1024,
5467 $drive->{iops_max
} || 0,
5468 $drive->{iops_rd_max
} || 0,
5469 $drive->{iops_wr_max
} || 0,
5470 $drive->{bps_max_length
} || 1,
5471 $drive->{bps_rd_max_length
} || 1,
5472 $drive->{bps_wr_max_length
} || 1,
5473 $drive->{iops_max_length
} || 1,
5474 $drive->{iops_rd_max_length
} || 1,
5475 $drive->{iops_wr_max_length
} || 1,
5485 if ($drive->{file
} eq 'none') {
5486 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5487 if (drive_is_cloudinit
($old_drive)) {
5488 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive);
5491 my $path = get_iso_path
($storecfg, $vmid, $drive->{file
});
5493 # force eject if locked
5494 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5497 mon_cmd
($vmid, "blockdev-change-medium",
5498 id
=> "$opt", filename
=> "$path");
5506 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5508 PVE
::Storage
::activate_volumes
($storecfg, [$drive->{file
}]) if $drive->{file
} !~ m
|^/dev/.+|;
5509 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5512 sub vmconfig_update_cloudinit_drive
{
5513 my ($storecfg, $conf, $vmid) = @_;
5515 my $cloudinit_ds = undef;
5516 my $cloudinit_drive = undef;
5518 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5519 my ($ds, $drive) = @_;
5520 if (PVE
::QemuServer
::drive_is_cloudinit
($drive)) {
5521 $cloudinit_ds = $ds;
5522 $cloudinit_drive = $drive;
5526 return if !$cloudinit_drive;
5528 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5529 PVE
::QemuConfig-
>write_config($vmid, $conf);
5532 my $running = PVE
::QemuServer
::check_running
($vmid);
5535 my $path = PVE
::Storage
::path
($storecfg, $cloudinit_drive->{file
});
5537 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$cloudinit_ds");
5538 mon_cmd
($vmid, "blockdev-change-medium", id
=> "$cloudinit_ds", filename
=> "$path");
5543 # called in locked context by incoming migration
5544 sub vm_migrate_get_nbd_disks
{
5545 my ($storecfg, $conf, $replicated_volumes) = @_;
5547 my $local_volumes = {};
5548 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5549 my ($ds, $drive) = @_;
5551 return if drive_is_cdrom
($drive);
5552 return if $ds eq 'tpmstate0';
5554 my $volid = $drive->{file
};
5558 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid);
5560 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5561 return if $scfg->{shared
};
5563 my $format = qemu_img_format
($scfg, $volname);
5565 # replicated disks re-use existing state via bitmap
5566 my $use_existing = $replicated_volumes->{$volid} ?
1 : 0;
5567 $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5569 return $local_volumes;
5572 # called in locked context by incoming migration
5573 sub vm_migrate_alloc_nbd_disks
{
5574 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5577 foreach my $opt (sort keys %$source_volumes) {
5578 my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5580 if ($use_existing) {
5581 $nbd->{$opt}->{drivestr
} = print_drive
($drive);
5582 $nbd->{$opt}->{volid
} = $volid;
5583 $nbd->{$opt}->{replicated
} = 1;
5587 $storeid = PVE
::JSONSchema
::map_id
($storagemap, $storeid);
5589 # order of precedence, filtered by whether storage supports it:
5590 # 1. explicit requested format
5591 # 2. default format of storage
5592 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5593 $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5595 my $size = $drive->{size
} / 1024;
5596 my $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $format, undef, $size);
5597 my $newdrive = $drive;
5598 $newdrive->{format
} = $format;
5599 $newdrive->{file
} = $newvolid;
5600 my $drivestr = print_drive
($newdrive);
5601 $nbd->{$opt}->{drivestr
} = $drivestr;
5602 $nbd->{$opt}->{volid
} = $newvolid;
5608 # see vm_start_nolock for parameters, additionally:
5610 # storagemap = parsed storage map for allocating NBD disks
5612 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5614 return PVE
::QemuConfig-
>lock_config($vmid, sub {
5615 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migrate_opts->{migratedfrom
});
5617 die "you can't start a vm if it's a template\n"
5618 if !$params->{skiptemplate
} && PVE
::QemuConfig-
>is_template($conf);
5620 my $has_suspended_lock = PVE
::QemuConfig-
>has_lock($conf, 'suspended');
5621 my $has_backup_lock = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5623 my $running = check_running
($vmid, undef, $migrate_opts->{migratedfrom
});
5625 if ($has_backup_lock && $running) {
5626 # a backup is currently running, attempt to start the guest in the
5627 # existing QEMU instance
5628 return vm_resume
($vmid);
5631 PVE
::QemuConfig-
>check_lock($conf)
5632 if !($params->{skiplock
} || $has_suspended_lock);
5634 $params->{resume
} = $has_suspended_lock || defined($conf->{vmstate
});
5636 die "VM $vmid already running\n" if $running;
5638 if (my $storagemap = $migrate_opts->{storagemap
}) {
5639 my $replicated = $migrate_opts->{replicated_volumes
};
5640 my $disks = vm_migrate_get_nbd_disks
($storecfg, $conf, $replicated);
5641 $migrate_opts->{nbd
} = vm_migrate_alloc_nbd_disks
($storecfg, $vmid, $disks, $storagemap);
5643 foreach my $opt (keys %{$migrate_opts->{nbd
}}) {
5644 $conf->{$opt} = $migrate_opts->{nbd
}->{$opt}->{drivestr
};
5648 return vm_start_nolock
($storecfg, $vmid, $conf, $params, $migrate_opts);
5654 # statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5655 # skiplock => 0/1, skip checking for config lock
5656 # skiptemplate => 0/1, skip checking whether VM is template
5657 # forcemachine => to force QEMU machine (rollback/migration)
5658 # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5659 # timeout => in seconds
5660 # paused => start VM in paused state (backup)
5661 # resume => resume from hibernation
5672 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5673 # migratedfrom => source node
5674 # spice_ticket => used for spice migration, passed via tunnel/stdin
5675 # network => CIDR of migration network
5676 # type => secure/insecure - tunnel over encrypted connection or plain-text
5677 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5678 # replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5679 # offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5680 # contained in config
5681 sub vm_start_nolock
{
5682 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5684 my $statefile = $params->{statefile
};
5685 my $resume = $params->{resume
};
5687 my $migratedfrom = $migrate_opts->{migratedfrom
};
5688 my $migration_type = $migrate_opts->{type
};
5692 # clean up leftover reboot request files
5693 eval { clear_reboot_request
($vmid); };
5696 if (!$statefile && scalar(keys %{$conf->{pending
}})) {
5697 vmconfig_apply_pending
($vmid, $conf, $storecfg);
5698 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5701 # don't regenerate the ISO if the VM is started as part of a live migration
5702 # this way we can reuse the old ISO with the correct config
5703 if (!$migratedfrom) {
5704 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5705 # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5706 # $conf->{cloudinit}, so we could just not do this?
5707 # But we do it above, so for now let's be consistent.
5708 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5712 # override offline migrated volumes, conf is out of date still
5713 if (my $offline_volumes = $migrate_opts->{offline_volumes
}) {
5714 for my $key (sort keys $offline_volumes->%*) {
5715 my $parsed = parse_drive
($key, $conf->{$key});
5716 $parsed->{file
} = $offline_volumes->{$key};
5717 $conf->{$key} = print_drive
($parsed);
5721 my $defaults = load_defaults
();
5723 # set environment variable useful inside network script
5724 # for remote migration the config is available on the target node!
5725 if (!$migrate_opts->{remote_node
}) {
5726 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom;
5729 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5731 my $forcemachine = $params->{forcemachine
};
5732 my $forcecpu = $params->{forcecpu
};
5734 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5735 $forcemachine = $conf->{runningmachine
};
5736 $forcecpu = $conf->{runningcpu
};
5737 print "Resuming suspended VM\n";
5740 my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command
($storecfg, $vmid,
5741 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5744 my $get_migration_ip = sub {
5745 my ($nodename) = @_;
5747 return $migration_ip if defined($migration_ip);
5749 my $cidr = $migrate_opts->{network
};
5751 if (!defined($cidr)) {
5752 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5753 $cidr = $dc_conf->{migration
}->{network
};
5756 if (defined($cidr)) {
5757 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5759 die "could not get IP: no address configured on local " .
5760 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5762 die "could not get IP: multiple addresses configured on local " .
5763 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5765 $migration_ip = @$ips[0];
5768 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5769 if !defined($migration_ip);
5771 return $migration_ip;
5775 if ($statefile eq 'tcp') {
5776 my $migrate = $res->{migrate
} = { proto
=> 'tcp' };
5777 $migrate->{addr
} = "localhost";
5778 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5779 my $nodename = nodename
();
5781 if (!defined($migration_type)) {
5782 if (defined($datacenterconf->{migration
}->{type
})) {
5783 $migration_type = $datacenterconf->{migration
}->{type
};
5785 $migration_type = 'secure';
5789 if ($migration_type eq 'insecure') {
5790 $migrate->{addr
} = $get_migration_ip->($nodename);
5791 $migrate->{addr
} = "[$migrate->{addr}]" if Net
::IP
::ip_is_ipv6
($migrate->{addr
});
5794 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5795 $migrate->{port
} = PVE
::Tools
::next_migrate_port
($pfamily);
5796 $migrate->{uri
} = "tcp:$migrate->{addr}:$migrate->{port}";
5797 push @$cmd, '-incoming', $migrate->{uri
};
5800 } elsif ($statefile eq 'unix') {
5801 # should be default for secure migrations as a ssh TCP forward
5802 # tunnel is not deterministic reliable ready and fails regurarly
5803 # to set up in time, so use UNIX socket forwards
5804 my $migrate = $res->{migrate
} = { proto
=> 'unix' };
5805 $migrate->{addr
} = "/run/qemu-server/$vmid.migrate";
5806 unlink $migrate->{addr
};
5808 $migrate->{uri
} = "unix:$migrate->{addr}";
5809 push @$cmd, '-incoming', $migrate->{uri
};
5812 } elsif (-e
$statefile) {
5813 push @$cmd, '-loadstate', $statefile;
5815 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5816 push @$vollist, $statefile;
5817 push @$cmd, '-loadstate', $statepath;
5819 } elsif ($params->{paused
}) {
5823 my $memory = get_current_memory
($conf->{memory
});
5824 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $memory, $resume);
5826 my $pci_reserve_list = [];
5827 for my $device (values $pci_devices->%*) {
5828 next if $device->{mdev
}; # we don't reserve for mdev devices
5829 push $pci_reserve_list->@*, map { $_->{id
} } $device->{ids
}->@*;
5832 # reserve all PCI IDs before actually doing anything with them
5833 PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, $start_timeout);
5837 for my $id (sort keys %$pci_devices) {
5838 my $d = $pci_devices->{$id};
5839 my ($index) = ($id =~ m/^hostpci(\d+)$/);
5842 for my $dev ($d->{ids
}->@*) {
5843 my $info = eval { PVE
::QemuServer
::PCI
::prepare_pci_device
($vmid, $dev->{id
}, $index, $d->{mdev
}) };
5846 $chosen_mdev = $info;
5847 last if $chosen_mdev; # if successful, we're done
5853 next if !$d->{mdev
};
5854 die "could not create mediated device\n" if !defined($chosen_mdev);
5856 # nvidia grid needs the uuid of the mdev as qemu parameter
5857 if (!defined($uuid) && $chosen_mdev->{vendor
} =~ m/^(0x)?10de$/) {
5858 if (defined($conf->{smbios1
})) {
5859 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
5860 $uuid = $smbios_conf->{uuid
} if defined($smbios_conf->{uuid
});
5862 $uuid = PVE
::QemuServer
::PCI
::generate_mdev_uuid
($vmid, $index) if !defined($uuid);
5865 push @$cmd, '-uuid', $uuid if defined($uuid);
5868 eval { cleanup_pci_devices
($vmid, $conf) };
5873 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5876 my %silence_std_outs = (outfunc
=> sub {}, errfunc
=> sub {});
5877 eval { run_command
(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5878 eval { run_command
(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5879 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5880 # timeout should be more than enough here...
5881 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 20);
5883 my $cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{cpuunits
});
5886 timeout
=> $statefile ?
undef : $start_timeout,
5891 # when migrating, prefix QEMU output so other side can pick up any
5892 # errors that might occur and show the user
5893 if ($migratedfrom) {
5894 $run_params{quiet
} = 1;
5895 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5898 my %systemd_properties = (
5899 Slice
=> 'qemu.slice',
5900 KillMode
=> 'process',
5902 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5905 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5906 $systemd_properties{CPUWeight
} = $cpuunits;
5908 $systemd_properties{CPUShares
} = $cpuunits;
5911 if (my $cpulimit = $conf->{cpulimit
}) {
5912 $systemd_properties{CPUQuota
} = int($cpulimit * 100);
5914 $systemd_properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5916 my $run_qemu = sub {
5917 PVE
::Tools
::run_fork
sub {
5918 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5921 if ((my $tpm = $conf->{tpmstate0
}) && !PVE
::QemuConfig-
>is_template($conf)) {
5922 # start the TPM emulator so QEMU can connect on start
5923 $tpmpid = start_swtpm
($storecfg, $vmid, $tpm, $migratedfrom);
5926 my $exitcode = run_command
($cmd, %run_params);
5929 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5930 kill 'TERM', $tpmpid;
5932 die "QEMU exited with code $exitcode\n";
5937 if ($conf->{hugepages
}) {
5940 my $hotplug_features =
5941 parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
5942 my $hugepages_topology =
5943 PVE
::QemuServer
::Memory
::hugepages_topology
($conf, $hotplug_features->{memory
});
5945 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5947 PVE
::QemuServer
::Memory
::hugepages_mount
();
5948 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5950 eval { $run_qemu->() };
5952 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5953 if !$conf->{keephugepages
};
5957 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5958 if !$conf->{keephugepages
};
5960 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5963 eval { $run_qemu->() };
5967 # deactivate volumes if start fails
5968 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5970 eval { cleanup_pci_devices
($vmid, $conf) };
5973 die "start failed: $err";
5976 # re-reserve all PCI IDs now that we can know the actual VM PID
5977 my $pid = PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
5978 eval { PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, undef, $pid) };
5981 if (defined($res->{migrate
})) {
5982 print "migration listens on $res->{migrate}->{uri}\n";
5983 } elsif ($statefile) {
5984 eval { mon_cmd
($vmid, "cont"); };
5988 #start nbd server for storage migration
5989 if (my $nbd = $migrate_opts->{nbd
}) {
5990 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5992 my $migrate_storage_uri;
5993 # nbd_protocol_version > 0 for unix socket support
5994 if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5995 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5996 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
5997 $migrate_storage_uri = "nbd:unix:$socket_path";
5998 $res->{migrate
}->{unix_sockets
} = [$socket_path];
6000 my $nodename = nodename
();
6001 my $localip = $get_migration_ip->($nodename);
6002 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
6003 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
6005 mon_cmd
($vmid, "nbd-server-start", addr
=> {
6008 host
=> "${localip}",
6009 port
=> "${storage_migrate_port}",
6012 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
6013 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
6016 my $block_info = mon_cmd
($vmid, "query-block");
6017 $block_info = { map { $_->{device
} => $_ } $block_info->@* };
6019 foreach my $opt (sort keys %$nbd) {
6020 my $drivestr = $nbd->{$opt}->{drivestr
};
6021 my $volid = $nbd->{$opt}->{volid
};
6023 my $block_node = $block_info->{"drive-$opt"}->{inserted
}->{'node-name'};
6029 'node-name' => $block_node,
6030 writable
=> JSON
::true
,
6032 name
=> "drive-$opt", # NBD export name
6035 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6036 print "storage migration listens on $nbd_uri volume:$drivestr\n";
6037 print "re-using replicated volume: $opt - $volid\n"
6038 if $nbd->{$opt}->{replicated
};
6040 $res->{drives
}->{$opt} = $nbd->{$opt};
6041 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
6045 if ($migratedfrom) {
6047 set_migration_caps
($vmid);
6052 print "spice listens on port $spice_port\n";
6053 $res->{spice_port
} = $spice_port;
6054 if ($migrate_opts->{spice_ticket
}) {
6055 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
6056 $migrate_opts->{spice_ticket
});
6057 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
6062 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
6063 if !$statefile && $conf->{balloon
};
6065 foreach my $opt (keys %$conf) {
6066 next if $opt !~ m/^net\d+$/;
6067 my $nicconf = parse_net
($conf->{$opt});
6068 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
6070 add_nets_bridge_fdb
($conf, $vmid);
6073 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
6078 path
=> "machine/peripheral/balloon0",
6079 property
=> "guest-stats-polling-interval",
6083 log_warn
("could not set polling interval for ballooning - $@") if $@;
6087 print "Resumed VM, removing state\n";
6088 if (my $vmstate = $conf->{vmstate
}) {
6089 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6090 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6092 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6093 PVE
::QemuConfig-
>write_config($vmid, $conf);
6096 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
6098 my ($current_machine, $is_deprecated) =
6099 PVE
::QemuServer
::Machine
::get_current_qemu_machine
($vmid);
6100 if ($is_deprecated) {
6102 "current machine version '$current_machine' is deprecated - see the documentation and ".
6103 "change to a newer one",
6110 sub vm_commandline
{
6111 my ($storecfg, $vmid, $snapname) = @_;
6113 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6115 my ($forcemachine, $forcecpu);
6117 my $snapshot = $conf->{snapshots
}->{$snapname};
6118 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6120 # check for machine or CPU overrides in snapshot
6121 $forcemachine = $snapshot->{runningmachine
};
6122 $forcecpu = $snapshot->{runningcpu
};
6124 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
6129 my $defaults = load_defaults
();
6131 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6133 return PVE
::Tools
::cmd2string
($cmd);
6137 my ($vmid, $skiplock) = @_;
6139 PVE
::QemuConfig-
>lock_config($vmid, sub {
6141 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6143 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6145 mon_cmd
($vmid, "system_reset");
6149 sub get_vm_volumes
{
6153 foreach_volid
($conf, sub {
6154 my ($volid, $attr) = @_;
6156 return if $volid =~ m
|^/|;
6158 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6161 push @$vollist, $volid;
6167 sub cleanup_pci_devices
{
6168 my ($vmid, $conf) = @_;
6170 foreach my $key (keys %$conf) {
6171 next if $key !~ m/^hostpci(\d+)$/;
6172 my $hostpciindex = $1;
6173 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
6174 my $d = parse_hostpci
($conf->{$key});
6176 # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6177 # don't want to break ABI just for this two liner
6178 my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6180 # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6181 # out when we do it first. so wait for 10 seconds and then try it
6182 if ($d->{ids
}->[0]->[0]->{vendor
} =~ m/^(0x)?10de$/) {
6186 PVE
::SysFSTools
::file_write
("$dev_sysfs_dir/remove", "1") if -e
$dev_sysfs_dir;
6189 PVE
::QemuServer
::PCI
::remove_pci_reservation
($vmid);
6192 sub vm_stop_cleanup
{
6193 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6198 my $vollist = get_vm_volumes
($conf);
6199 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6201 if (my $tpmdrive = $conf->{tpmstate0
}) {
6202 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
6203 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
6205 PVE
::Storage
::unmap_volume
($storecfg, $tpm->{file
});
6210 foreach my $ext (qw(mon qmp pid vnc qga)) {
6211 unlink "/var/run/qemu-server/${vmid}.$ext";
6214 if ($conf->{ivshmem
}) {
6215 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
6216 # just delete it for now, VMs which have this already open do not
6217 # are affected, but new VMs will get a separated one. If this
6218 # becomes an issue we either add some sort of ref-counting or just
6219 # add a "don't delete on stop" flag to the ivshmem format.
6220 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
6223 cleanup_pci_devices
($vmid, $conf);
6225 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
6227 warn $@ if $@; # avoid errors - just warn
6230 # call only in locked context
6232 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6234 my $pid = check_running
($vmid, $nocheck);
6239 $conf = PVE
::QemuConfig-
>load_config($vmid);
6240 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6241 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
6242 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
6243 $timeout = $opts->{down
} if $opts->{down
};
6245 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
6250 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
6251 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
6253 mon_cmd
($vmid, "system_powerdown");
6256 mon_cmd
($vmid, "quit");
6262 $timeout = 60 if !defined($timeout);
6265 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6270 if ($count >= $timeout) {
6272 warn "VM still running - terminating now with SIGTERM\n";
6275 die "VM quit/powerdown failed - got timeout\n";
6278 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6282 if (!check_running
($vmid, $nocheck)) {
6283 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6287 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6290 die "VM quit/powerdown failed\n";
6298 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6303 if ($count >= $timeout) {
6304 warn "VM still running - terminating now with SIGKILL\n";
6309 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6312 # Note: use $nocheck to skip tests if VM configuration file exists.
6313 # We need that when migration VMs to other nodes (files already moved)
6314 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6316 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6318 $force = 1 if !defined($force) && !$shutdown;
6321 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
6322 kill 15, $pid if $pid;
6323 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
6324 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
6328 PVE
::QemuConfig-
>lock_config($vmid, sub {
6329 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6334 my ($vmid, $timeout) = @_;
6336 PVE
::QemuConfig-
>lock_config($vmid, sub {
6339 # only reboot if running, as qmeventd starts it again on a stop event
6340 return if !check_running
($vmid);
6342 create_reboot_request
($vmid);
6344 my $storecfg = PVE
::Storage
::config
();
6345 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
6349 # avoid that the next normal shutdown will be confused for a reboot
6350 clear_reboot_request
($vmid);
6356 # note: if using the statestorage parameter, the caller has to check privileges
6358 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6365 PVE
::QemuConfig-
>lock_config($vmid, sub {
6367 $conf = PVE
::QemuConfig-
>load_config($vmid);
6369 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
6370 PVE
::QemuConfig-
>check_lock($conf)
6371 if !($skiplock || $is_backing_up);
6373 die "cannot suspend to disk during backup\n"
6374 if $is_backing_up && $includestate;
6376 if ($includestate) {
6377 $conf->{lock} = 'suspending';
6378 my $date = strftime
("%Y-%m-%d", localtime(time()));
6379 $storecfg = PVE
::Storage
::config
();
6380 if (!$statestorage) {
6381 $statestorage = find_vmstate_storage
($conf, $storecfg);
6382 # check permissions for the storage
6383 my $rpcenv = PVE
::RPCEnvironment
::get
();
6384 if ($rpcenv->{type
} ne 'cli') {
6385 my $authuser = $rpcenv->get_user();
6386 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6391 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
6392 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6393 $path = PVE
::Storage
::path
($storecfg, $vmstate);
6394 PVE
::QemuConfig-
>write_config($vmid, $conf);
6396 mon_cmd
($vmid, "stop");
6400 if ($includestate) {
6402 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
6405 set_migration_caps
($vmid, 1);
6406 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
6408 my $state = mon_cmd
($vmid, "query-savevm");
6409 if (!$state->{status
}) {
6410 die "savevm not active\n";
6411 } elsif ($state->{status
} eq 'active') {
6414 } elsif ($state->{status
} eq 'completed') {
6415 print "State saved, quitting\n";
6417 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
6418 die "query-savevm failed with error '$state->{error}'\n"
6420 die "query-savevm returned status '$state->{status}'\n";
6426 PVE
::QemuConfig-
>lock_config($vmid, sub {
6427 $conf = PVE
::QemuConfig-
>load_config($vmid);
6429 # cleanup, but leave suspending lock, to indicate something went wrong
6431 mon_cmd
($vmid, "savevm-end");
6432 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6433 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6434 delete $conf->@{qw(vmstate runningmachine runningcpu)};
6435 PVE
::QemuConfig-
>write_config($vmid, $conf);
6441 die "lock changed unexpectedly\n"
6442 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
6444 mon_cmd
($vmid, "quit");
6445 $conf->{lock} = 'suspended';
6446 PVE
::QemuConfig-
>write_config($vmid, $conf);
6451 # $nocheck is set when called as part of a migration - in this context the
6452 # location of the config file (source or target node) is not deterministic,
6453 # since migration cannot wait for pmxcfs to process the rename
6455 my ($vmid, $skiplock, $nocheck) = @_;
6457 PVE
::QemuConfig-
>lock_config($vmid, sub {
6458 my $res = mon_cmd
($vmid, 'query-status');
6459 my $resume_cmd = 'cont';
6463 $conf = eval { PVE
::QemuConfig-
>load_config($vmid) }; # try on target node
6465 my $vmlist = PVE
::Cluster
::get_vmlist
();
6466 if (exists($vmlist->{ids
}->{$vmid})) {
6467 my $node = $vmlist->{ids
}->{$vmid}->{node
};
6468 $conf = eval { PVE
::QemuConfig-
>load_config($vmid, $node) }; # try on source node
6471 PVE
::Cluster
::cfs_update
(); # vmlist was wrong, invalidate cache
6472 $conf = PVE
::QemuConfig-
>load_config($vmid); # last try on target node again
6476 $conf = PVE
::QemuConfig-
>load_config($vmid);
6479 if ($res->{status
}) {
6480 return if $res->{status
} eq 'running'; # job done, go home
6481 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
6482 $reset = 1 if $res->{status
} eq 'shutdown';
6486 PVE
::QemuConfig-
>check_lock($conf)
6487 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
6491 # required if a VM shuts down during a backup and we get a resume
6492 # request before the backup finishes for example
6493 mon_cmd
($vmid, "system_reset");
6496 add_nets_bridge_fdb
($conf, $vmid) if $resume_cmd eq 'cont';
6498 mon_cmd
($vmid, $resume_cmd);
6503 my ($vmid, $skiplock, $key) = @_;
6505 PVE
::QemuConfig-
>lock_config($vmid, sub {
6507 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6509 # there is no qmp command, so we use the human monitor command
6510 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
6511 die $res if $res ne '';
6515 sub check_bridge_access
{
6516 my ($rpcenv, $authuser, $conf) = @_;
6518 return 1 if $authuser eq 'root@pam';
6520 for my $opt (sort keys $conf->%*) {
6521 next if $opt !~ m/^net\d+$/;
6522 my $net = parse_net
($conf->{$opt});
6523 my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6524 PVE
::GuestHelpers
::check_vnet_access
($rpcenv, $authuser, $bridge, $tag, $trunks);
6529 sub check_mapping_access
{
6530 my ($rpcenv, $user, $conf) = @_;
6532 for my $opt (keys $conf->%*) {
6533 if ($opt =~ m/^usb\d+$/) {
6534 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-usb', $conf->{$opt});
6535 if (my $host = $device->{host
}) {
6536 die "only root can set '$opt' config for real devices\n"
6537 if $host !~ m/^spice$/i && $user ne 'root@pam';
6538 } elsif ($device->{mapping
}) {
6539 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6541 die "either 'host' or 'mapping' must be set.\n";
6543 } elsif ($opt =~ m/^hostpci\d+$/) {
6544 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-hostpci', $conf->{$opt});
6545 if ($device->{host
}) {
6546 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6547 } elsif ($device->{mapping
}) {
6548 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6550 die "either 'host' or 'mapping' must be set.\n";
6556 sub check_restore_permissions
{
6557 my ($rpcenv, $user, $conf) = @_;
6559 check_bridge_access
($rpcenv, $user, $conf);
6560 check_mapping_access
($rpcenv, $user, $conf);
6562 # vzdump restore implementaion
6564 sub tar_archive_read_firstfile
{
6565 my $archive = shift;
6567 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
6569 # try to detect archive type first
6570 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6571 die "unable to open file '$archive'\n";
6572 my $firstfile = <$fh>;
6576 die "ERROR: archive contaions no data\n" if !$firstfile;
6582 sub tar_restore_cleanup
{
6583 my ($storecfg, $statfile) = @_;
6585 print STDERR
"starting cleanup\n";
6587 if (my $fd = IO
::File-
>new($statfile, "r")) {
6588 while (defined(my $line = <$fd>)) {
6589 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6592 if ($volid =~ m
|^/|) {
6593 unlink $volid || die 'unlink failed\n';
6595 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6597 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6599 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6601 print STDERR
"unable to parse line in statfile - $line";
6608 sub restore_file_archive
{
6609 my ($archive, $vmid, $user, $opts) = @_;
6611 return restore_vma_archive
($archive, $vmid, $user, $opts)
6614 my $info = PVE
::Storage
::archive_info
($archive);
6615 my $format = $opts->{format
} // $info->{format
};
6616 my $comp = $info->{compression
};
6618 # try to detect archive format
6619 if ($format eq 'tar') {
6620 return restore_tar_archive
($archive, $vmid, $user, $opts);
6622 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
6626 # hepler to remove disks that will not be used after restore
6627 my $restore_cleanup_oldconf = sub {
6628 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6630 my $kept_disks = {};
6632 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
6633 my ($ds, $drive) = @_;
6635 return if drive_is_cdrom
($drive, 1);
6637 my $volid = $drive->{file
};
6638 return if !$volid || $volid =~ m
|^/|;
6640 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
6641 return if !$path || !$owner || ($owner != $vmid);
6643 # Note: only delete disk we want to restore
6644 # other volumes will become unused
6645 if ($virtdev_hash->{$ds}) {
6646 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
6651 $kept_disks->{$volid} = 1;
6655 # after the restore we have no snapshots anymore
6656 for my $snapname (keys $oldconf->{snapshots
}->%*) {
6657 my $snap = $oldconf->{snapshots
}->{$snapname};
6658 if ($snap->{vmstate
}) {
6659 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
6665 for my $volid (keys $kept_disks->%*) {
6666 eval { PVE
::Storage
::volume_snapshot_delete
($storecfg, $volid, $snapname); };
6672 # Helper to parse vzdump backup device hints
6674 # $rpcenv: Environment, used to ckeck storage permissions
6675 # $user: User ID, to check storage permissions
6676 # $storecfg: Storage configuration
6677 # $fh: the file handle for reading the configuration
6678 # $devinfo: should contain device sizes for all backu-up'ed devices
6679 # $options: backup options (pool, default storage)
6681 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6682 my $parse_backup_hints = sub {
6683 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6685 my $check_storage = sub { # assert if an image can be allocate
6686 my ($storeid, $scfg) = @_;
6687 die "Content type 'images' is not available on storage '$storeid'\n"
6688 if !$scfg->{content
}->{images
};
6689 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6690 if $user ne 'root@pam';
6693 my $virtdev_hash = {};
6694 while (defined(my $line = <$fh>)) {
6695 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6696 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6697 die "archive does not contain data for drive '$virtdev'\n"
6698 if !$devinfo->{$devname};
6700 if (defined($options->{storage
})) {
6701 $storeid = $options->{storage
} || 'local';
6702 } elsif (!$storeid) {
6705 $format = 'raw' if !$format;
6706 $devinfo->{$devname}->{devname
} = $devname;
6707 $devinfo->{$devname}->{virtdev
} = $virtdev;
6708 $devinfo->{$devname}->{format
} = $format;
6709 $devinfo->{$devname}->{storeid
} = $storeid;
6711 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6712 $check_storage->($storeid, $scfg); # permission and content type check
6714 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6715 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6717 my $drive = parse_drive
($virtdev, $2);
6719 if (drive_is_cloudinit
($drive)) {
6720 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6721 $storeid = $options->{storage
} if defined ($options->{storage
});
6722 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6723 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6725 $check_storage->($storeid, $scfg); # permission and content type check
6727 $virtdev_hash->{$virtdev} = {
6729 storeid
=> $storeid,
6730 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6737 return $virtdev_hash;
6740 # Helper to allocate and activate all volumes required for a restore
6742 # $storecfg: Storage configuration
6743 # $virtdev_hash: as returned by parse_backup_hints()
6745 # Returns: { $virtdev => $volid }
6746 my $restore_allocate_devices = sub {
6747 my ($storecfg, $virtdev_hash, $vmid) = @_;
6750 foreach my $virtdev (sort keys %$virtdev_hash) {
6751 my $d = $virtdev_hash->{$virtdev};
6752 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6753 my $storeid = $d->{storeid
};
6754 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6756 # test if requested format is supported
6757 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6758 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6759 $d->{format
} = $defFormat if !$supported;
6762 if ($d->{is_cloudinit
}) {
6763 $name = "vm-$vmid-cloudinit";
6764 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6765 if ($scfg->{path
}) {
6766 $name .= ".$d->{format}";
6770 my $volid = PVE
::Storage
::vdisk_alloc
(
6771 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6773 print STDERR
"new volume ID is '$volid'\n";
6774 $d->{volid
} = $volid;
6776 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6778 $map->{$virtdev} = $volid;
6784 sub restore_update_config_line
{
6785 my ($cookie, $map, $line, $unique) = @_;
6787 return '' if $line =~ m/^\#qmdump\#/;
6788 return '' if $line =~ m/^\#vzdump\#/;
6789 return '' if $line =~ m/^lock:/;
6790 return '' if $line =~ m/^unused\d+:/;
6791 return '' if $line =~ m/^parent:/;
6795 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6796 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6797 # try to convert old 1.X settings
6798 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6799 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6800 my ($model, $macaddr) = split(/\=/, $devconfig);
6801 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6804 bridge
=> "vmbr$ind",
6805 macaddr
=> $macaddr,
6807 my $netstr = print_net
($net);
6809 $res .= "net$cookie->{netcount}: $netstr\n";
6810 $cookie->{netcount
}++;
6812 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6813 my ($id, $netstr) = ($1, $2);
6814 my $net = parse_net
($netstr);
6815 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6816 $netstr = print_net
($net);
6817 $res .= "$id: $netstr\n";
6818 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6821 my $di = parse_drive
($virtdev, $value);
6822 if (defined($di->{backup
}) && !$di->{backup
}) {
6824 } elsif ($map->{$virtdev}) {
6825 delete $di->{format
}; # format can change on restore
6826 $di->{file
} = $map->{$virtdev};
6827 $value = print_drive
($di);
6828 $res .= "$virtdev: $value\n";
6832 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6834 if ($vmgenid ne '0') {
6835 # always generate a new vmgenid if there was a valid one setup
6836 $vmgenid = generate_uuid
();
6838 $res .= "vmgenid: $vmgenid\n";
6839 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6840 my ($uuid, $uuid_str);
6841 UUID
::generate
($uuid);
6842 UUID
::unparse
($uuid, $uuid_str);
6843 my $smbios1 = parse_smbios1
($2);
6844 $smbios1->{uuid
} = $uuid_str;
6845 $res .= $1.print_smbios1
($smbios1)."\n";
6853 my $restore_deactivate_volumes = sub {
6854 my ($storecfg, $virtdev_hash) = @_;
6857 for my $dev (values $virtdev_hash->%*) {
6858 push $vollist->@*, $dev->{volid
} if $dev->{volid
};
6861 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
6862 print STDERR
$@ if $@;
6865 my $restore_destroy_volumes = sub {
6866 my ($storecfg, $virtdev_hash) = @_;
6868 for my $dev (values $virtdev_hash->%*) {
6869 my $volid = $dev->{volid
} or next;
6871 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6872 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6874 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6878 sub restore_merge_config
{
6879 my ($filename, $backup_conf_raw, $override_conf) = @_;
6881 my $backup_conf = parse_vm_config
($filename, $backup_conf_raw);
6882 for my $key (keys $override_conf->%*) {
6883 $backup_conf->{$key} = $override_conf->{$key};
6886 return $backup_conf;
6890 my ($cfg, $vmid) = @_;
6892 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6894 my $volid_hash = {};
6895 foreach my $storeid (keys %$info) {
6896 foreach my $item (@{$info->{$storeid}}) {
6897 next if !($item->{volid
} && $item->{size
});
6898 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6899 $volid_hash->{$item->{volid
}} = $item;
6906 sub update_disk_config
{
6907 my ($vmid, $conf, $volid_hash) = @_;
6910 my $prefix = "VM $vmid";
6912 # used and unused disks
6913 my $referenced = {};
6915 # Note: it is allowed to define multiple storages with same path (alias), so
6916 # we need to check both 'volid' and real 'path' (two different volid can point
6917 # to the same path).
6919 my $referencedpath = {};
6922 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6923 my ($opt, $drive) = @_;
6925 my $volid = $drive->{file
};
6927 my $volume = $volid_hash->{$volid};
6929 # mark volid as "in-use" for next step
6930 $referenced->{$volid} = 1;
6931 if ($volume && (my $path = $volume->{path
})) {
6932 $referencedpath->{$path} = 1;
6935 return if drive_is_cdrom
($drive);
6938 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6939 if (defined($updated)) {
6941 $conf->{$opt} = print_drive
($updated);
6942 print "$prefix ($opt): $msg\n";
6946 # remove 'unusedX' entry if volume is used
6947 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6948 my ($opt, $drive) = @_;
6950 my $volid = $drive->{file
};
6954 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6955 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6956 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6958 delete $conf->{$opt};
6961 $referenced->{$volid} = 1;
6962 $referencedpath->{$path} = 1 if $path;
6965 foreach my $volid (sort keys %$volid_hash) {
6966 next if $volid =~ m/vm-$vmid-state-/;
6967 next if $referenced->{$volid};
6968 my $path = $volid_hash->{$volid}->{path
};
6969 next if !$path; # just to be sure
6970 next if $referencedpath->{$path};
6972 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6973 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6974 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6981 my ($vmid, $nolock, $dryrun) = @_;
6983 my $cfg = PVE
::Storage
::config
();
6985 print "rescan volumes...\n";
6986 my $volid_hash = scan_volids
($cfg, $vmid);
6988 my $updatefn = sub {
6991 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6993 PVE
::QemuConfig-
>check_lock($conf);
6996 foreach my $volid (keys %$volid_hash) {
6997 my $info = $volid_hash->{$volid};
6998 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
7001 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
7003 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
7006 if (defined($vmid)) {
7010 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
7013 my $vmlist = config_list
();
7014 foreach my $vmid (keys %$vmlist) {
7018 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
7024 sub restore_proxmox_backup_archive
{
7025 my ($archive, $vmid, $user, $options) = @_;
7027 my $storecfg = PVE
::Storage
::config
();
7029 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
7030 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7032 my $fingerprint = $scfg->{fingerprint
};
7033 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
7035 my $repo = PVE
::PBSClient
::get_repository
($scfg);
7036 my $namespace = $scfg->{namespace
};
7038 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7039 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
7040 local $ENV{PBS_PASSWORD
} = $password;
7041 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
7043 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7044 PVE
::Storage
::parse_volname
($storecfg, $archive);
7046 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7048 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7050 my $tmpdir = "/var/tmp/vzdumptmp$$";
7054 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7055 # disable interrupts (always do cleanups)
7059 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7061 # Note: $oldconf is undef if VM does not exists
7062 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7063 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7064 my $new_conf_raw = '';
7066 my $rpcenv = PVE
::RPCEnvironment
::get
();
7067 my $devinfo = {}; # info about drives included in backup
7068 my $virtdev_hash = {}; # info about allocated drives
7076 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7078 my $cfgfn = "$tmpdir/qemu-server.conf";
7079 my $firewall_config_fn = "$tmpdir/fw.conf";
7080 my $index_fn = "$tmpdir/index.json";
7082 my $cmd = "restore";
7084 my $param = [$pbs_backup_name, "index.json", $index_fn];
7085 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7086 my $index = PVE
::Tools
::file_get_contents
($index_fn);
7087 $index = decode_json
($index);
7089 foreach my $info (@{$index->{files
}}) {
7090 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
7092 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
7093 $devinfo->{$devname}->{size
} = $1;
7095 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7100 my $is_qemu_server_backup = scalar(
7101 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
7103 if (!$is_qemu_server_backup) {
7104 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7106 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
7108 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7109 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7111 if ($has_firewall_config) {
7112 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7113 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7115 my $pve_firewall_dir = '/etc/pve/firewall';
7116 mkdir $pve_firewall_dir; # make sure the dir exists
7117 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7120 my $fh = IO
::File-
>new($cfgfn, "r") ||
7121 die "unable to read qemu-server.conf - $!\n";
7123 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7125 # fixme: rate limit?
7127 # create empty/temp config
7128 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
7130 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7133 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7135 foreach my $virtdev (sort keys %$virtdev_hash) {
7136 my $d = $virtdev_hash->{$virtdev};
7137 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7139 # this fails if storage is unavailable
7140 my $volid = $d->{volid
};
7141 my $path = PVE
::Storage
::path
($storecfg, $volid);
7143 # for live-restore we only want to preload the efidisk and TPM state
7144 next if $options->{live
} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7147 if (defined(my $ns = $scfg->{namespace
})) {
7148 @ns_arg = ('--ns', $ns);
7151 my $pbs_restore_cmd = [
7152 '/usr/bin/pbs-restore',
7153 '--repository', $repo,
7156 "$d->{devname}.img.fidx",
7161 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
7162 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
7164 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
7165 push @$pbs_restore_cmd, '--skip-zero';
7168 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
7169 print "restore proxmox backup image: $dbg_cmdstring\n";
7170 run_command
($pbs_restore_cmd);
7173 $fh->seek(0, 0) || die "seek failed - $!\n";
7175 my $cookie = { netcount
=> 0 };
7176 while (defined(my $line = <$fh>)) {
7177 $new_conf_raw .= restore_update_config_line
(
7189 if ($err || !$options->{live
}) {
7190 $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7196 $restore_destroy_volumes->($storecfg, $virtdev_hash);
7200 if ($options->{live
}) {
7201 # keep lock during live-restore
7202 $new_conf_raw .= "\nlock: create";
7205 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $options->{override_conf
});
7206 check_restore_permissions
($rpcenv, $user, $new_conf);
7207 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7209 eval { rescan
($vmid, 1); };
7212 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
7214 if ($options->{live
}) {
7220 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
7222 my $conf = PVE
::QemuConfig-
>load_config($vmid);
7223 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
7225 # these special drives are already restored before start
7226 delete $devinfo->{'drive-efidisk0'};
7227 delete $devinfo->{'drive-tpmstate0-backup'};
7231 keyfile
=> $keyfile,
7232 snapshot
=> $pbs_backup_name,
7233 namespace
=> $namespace,
7235 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7237 PVE
::QemuConfig-
>remove_lock($vmid, "create");
7241 sub pbs_live_restore
{
7242 my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7244 print "starting VM for live-restore\n";
7245 print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7247 my $pbs_backing = {};
7248 for my $ds (keys %$restored_disks) {
7249 $ds =~ m/^drive-(.*)$/;
7251 $pbs_backing->{$confname} = {
7252 repository
=> $opts->{repo
},
7253 snapshot
=> $opts->{snapshot
},
7254 archive
=> "$ds.img.fidx",
7256 $pbs_backing->{$confname}->{keyfile
} = $opts->{keyfile
} if -e
$opts->{keyfile
};
7257 $pbs_backing->{$confname}->{namespace
} = $opts->{namespace
} if defined($opts->{namespace
});
7259 my $drive = parse_drive
($confname, $conf->{$confname});
7260 print "restoring '$ds' to '$drive->{file}'\n";
7263 my $drives_streamed = 0;
7265 # make sure HA doesn't interrupt our restore by stopping the VM
7266 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
7267 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7270 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7271 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7272 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'pbs-backing' => $pbs_backing}, {});
7274 my $qmeventd_fd = register_qmeventd_handle
($vmid);
7276 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7277 # this will effectively collapse the backing image chain consisting of
7278 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7279 # removes itself once all backing images vanish with 'auto-remove=on')
7281 for my $ds (sort keys %$restored_disks) {
7282 my $job_id = "restore-$ds";
7283 mon_cmd
($vmid, 'block-stream',
7284 'job-id' => $job_id,
7287 $jobs->{$job_id} = {};
7290 mon_cmd
($vmid, 'cont');
7291 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
7293 print "restore-drive jobs finished successfully, removing all tracking block devices"
7294 ." to disconnect from Proxmox Backup Server\n";
7296 for my $ds (sort keys %$restored_disks) {
7297 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7300 close($qmeventd_fd);
7306 warn "An error occurred during live-restore: $err\n";
7307 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
7308 die "live-restore failed\n";
7312 sub restore_vma_archive
{
7313 my ($archive, $vmid, $user, $opts, $comp) = @_;
7315 my $readfrom = $archive;
7317 my $cfg = PVE
::Storage
::config
();
7319 my $bwlimit = $opts->{bwlimit
};
7321 my $dbg_cmdstring = '';
7322 my $add_pipe = sub {
7324 push @$commands, $cmd;
7325 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7326 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
7331 if ($archive eq '-') {
7334 # If we use a backup from a PVE defined storage we also consider that
7335 # storage's rate limit:
7336 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
7337 if (defined($volid)) {
7338 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
7339 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
7341 print STDERR
"applying read rate limit: $readlimit\n";
7342 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7343 $add_pipe->($cstream);
7349 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
7350 my $cmd = $info->{decompressor
};
7351 push @$cmd, $readfrom;
7355 my $tmpdir = "/var/tmp/vzdumptmp$$";
7358 # disable interrupts (always do cleanups)
7362 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
7364 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7365 POSIX
::mkfifo
($mapfifo, 0600);
7367 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7369 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7371 my $devinfo = {}; # info about drives included in backup
7372 my $virtdev_hash = {}; # info about allocated drives
7374 my $rpcenv = PVE
::RPCEnvironment
::get
();
7376 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7378 # Note: $oldconf is undef if VM does not exist
7379 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7380 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7381 my $new_conf_raw = '';
7385 my $print_devmap = sub {
7386 my $cfgfn = "$tmpdir/qemu-server.conf";
7388 # we can read the config - that is already extracted
7389 my $fh = IO
::File-
>new($cfgfn, "r") ||
7390 die "unable to read qemu-server.conf - $!\n";
7392 my $fwcfgfn = "$tmpdir/qemu-server.fw";
7394 my $pve_firewall_dir = '/etc/pve/firewall';
7395 mkdir $pve_firewall_dir; # make sure the dir exists
7396 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7399 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7401 foreach my $info (values %{$virtdev_hash}) {
7402 my $storeid = $info->{storeid
};
7403 next if defined($storage_limits{$storeid});
7405 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
7406 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7407 $storage_limits{$storeid} = $limit * 1024;
7410 foreach my $devname (keys %$devinfo) {
7411 die "found no device mapping information for device '$devname'\n"
7412 if !$devinfo->{$devname}->{virtdev
};
7415 # create empty/temp config
7417 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
7418 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7422 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7424 # print restore information to $fifofh
7425 foreach my $virtdev (sort keys %$virtdev_hash) {
7426 my $d = $virtdev_hash->{$virtdev};
7427 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7429 my $storeid = $d->{storeid
};
7430 my $volid = $d->{volid
};
7433 if (my $limit = $storage_limits{$storeid}) {
7434 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7437 my $write_zeros = 1;
7438 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
7442 my $path = PVE
::Storage
::path
($cfg, $volid);
7444 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7446 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7449 $fh->seek(0, 0) || die "seek failed - $!\n";
7451 my $cookie = { netcount
=> 0 };
7452 while (defined(my $line = <$fh>)) {
7453 $new_conf_raw .= restore_update_config_line
(
7472 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7473 local $SIG{ALRM
} = sub { die "got timeout\n"; };
7475 $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7482 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7483 my ($dev_id, $size, $devname) = ($1, $2, $3);
7484 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
7485 } elsif ($line =~ m/^CTIME: /) {
7486 # we correctly received the vma config, so we can disable
7487 # the timeout now for disk allocation
7488 alarm($oldtimeout || 0);
7489 $oldtimeout = undef;
7491 print $fifofh "done\n";
7497 print "restore vma archive: $dbg_cmdstring\n";
7498 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
7502 alarm($oldtimeout) if $oldtimeout;
7504 $restore_deactivate_volumes->($cfg, $virtdev_hash);
7506 close($fifofh) if $fifofh;
7511 $restore_destroy_volumes->($cfg, $virtdev_hash);
7515 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $opts->{override_conf
});
7516 check_restore_permissions
($rpcenv, $user, $new_conf);
7517 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7519 eval { rescan
($vmid, 1); };
7522 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
7525 sub restore_tar_archive
{
7526 my ($archive, $vmid, $user, $opts) = @_;
7528 if (scalar(keys $opts->{override_conf
}->%*) > 0) {
7529 my $keystring = join(' ', keys $opts->{override_conf
}->%*);
7530 die "cannot pass along options ($keystring) when restoring from tar archive\n";
7533 if ($archive ne '-') {
7534 my $firstfile = tar_archive_read_firstfile
($archive);
7535 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7536 if $firstfile ne 'qemu-server.conf';
7539 my $storecfg = PVE
::Storage
::config
();
7541 # avoid zombie disks when restoring over an existing VM -> cleanup first
7542 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7543 # skiplock=1 because qmrestore has set the 'create' lock itself already
7544 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
7545 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
7547 my $tocmd = "/usr/lib/qemu-server/qmextract";
7549 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
7550 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
7551 $tocmd .= ' --prealloc' if $opts->{prealloc
};
7552 $tocmd .= ' --info' if $opts->{info
};
7554 # tar option "xf" does not autodetect compression when read from STDIN,
7555 # so we pipe to zcat
7556 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
7557 PVE
::Tools
::shellquote
("--to-command=$tocmd");
7559 my $tmpdir = "/var/tmp/vzdumptmp$$";
7562 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
7563 local $ENV{VZDUMP_VMID
} = $vmid;
7564 local $ENV{VZDUMP_USER
} = $user;
7566 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7567 my $new_conf_raw = '';
7569 # disable interrupts (always do cleanups)
7573 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7581 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7583 if ($archive eq '-') {
7584 print "extracting archive from STDIN\n";
7585 run_command
($cmd, input
=> "<&STDIN");
7587 print "extracting archive '$archive'\n";
7591 return if $opts->{info
};
7595 my $statfile = "$tmpdir/qmrestore.stat";
7596 if (my $fd = IO
::File-
>new($statfile, "r")) {
7597 while (defined (my $line = <$fd>)) {
7598 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7599 $map->{$1} = $2 if $1;
7601 print STDERR
"unable to parse line in statfile - $line\n";
7607 my $confsrc = "$tmpdir/qemu-server.conf";
7609 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7611 my $cookie = { netcount
=> 0 };
7612 while (defined (my $line = <$srcfd>)) {
7613 $new_conf_raw .= restore_update_config_line
(
7624 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
7630 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7632 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7634 eval { rescan
($vmid, 1); };
7638 sub foreach_storage_used_by_vm
{
7639 my ($conf, $func) = @_;
7643 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7644 my ($ds, $drive) = @_;
7645 return if drive_is_cdrom
($drive);
7647 my $volid = $drive->{file
};
7649 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
7650 $sidhash->{$sid} = $sid if $sid;
7653 foreach my $sid (sort keys %$sidhash) {
7658 my $qemu_snap_storage = {
7661 sub do_snapshots_with_qemu
{
7662 my ($storecfg, $volid, $deviceid) = @_;
7664 return if $deviceid =~ m/tpmstate0/;
7666 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
7667 my $scfg = $storecfg->{ids
}->{$storage_name};
7668 die "could not find storage '$storage_name'\n" if !defined($scfg);
7670 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
7674 if ($volid =~ m/\.(qcow2|qed)$/){
7681 sub qga_check_running
{
7682 my ($vmid, $nowarn) = @_;
7684 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
7686 warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7692 sub template_create
{
7693 my ($vmid, $conf, $disk) = @_;
7695 my $storecfg = PVE
::Storage
::config
();
7697 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7698 my ($ds, $drive) = @_;
7700 return if drive_is_cdrom
($drive);
7701 return if $disk && $ds ne $disk;
7703 my $volid = $drive->{file
};
7704 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
7706 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
7707 $drive->{file
} = $voliddst;
7708 $conf->{$ds} = print_drive
($drive);
7709 PVE
::QemuConfig-
>write_config($vmid, $conf);
7713 sub convert_iscsi_path
{
7716 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
7721 my $initiator_name = get_initiator_name
();
7723 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7724 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7727 die "cannot convert iscsi path '$path', unkown format\n";
7730 sub qemu_img_convert
{
7731 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7733 my $storecfg = PVE
::Storage
::config
();
7734 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
7735 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
7737 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7741 my $src_is_iscsi = 0;
7745 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
7746 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7747 $src_format = qemu_img_format
($src_scfg, $src_volname);
7748 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
7749 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
7750 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
7751 } elsif (-f
$src_volid || -b
$src_volid) {
7752 $src_path = $src_volid;
7753 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7758 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7760 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7761 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7762 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7763 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7766 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7767 push @$cmd, '-l', "snapshot.name=$snapname"
7768 if $snapname && $src_format && $src_format eq "qcow2";
7769 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7770 push @$cmd, '-T', $cachemode if defined($cachemode);
7771 push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7773 if ($src_is_iscsi) {
7774 push @$cmd, '--image-opts';
7775 $src_path = convert_iscsi_path
($src_path);
7776 } elsif ($src_format) {
7777 push @$cmd, '-f', $src_format;
7780 if ($dst_is_iscsi) {
7781 push @$cmd, '--target-image-opts';
7782 $dst_path = convert_iscsi_path
($dst_path);
7784 push @$cmd, '-O', $dst_format;
7787 push @$cmd, $src_path;
7789 if (!$dst_is_iscsi && $is_zero_initialized) {
7790 push @$cmd, "zeroinit:$dst_path";
7792 push @$cmd, $dst_path;
7797 if($line =~ m/\((\S+)\/100\
%\)/){
7799 my $transferred = int($size * $percent / 100);
7800 my $total_h = render_bytes
($size, 1);
7801 my $transferred_h = render_bytes
($transferred, 1);
7803 print "transferred $transferred_h of $total_h ($percent%)\n";
7808 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7810 die "copy failed: $err" if $err;
7813 sub qemu_img_format
{
7814 my ($scfg, $volname) = @_;
7816 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7823 sub qemu_drive_mirror
{
7824 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7826 $jobs = {} if !$jobs;
7830 $jobs->{"drive-$drive"} = {};
7832 if ($dst_volid =~ /^nbd:/) {
7833 $qemu_target = $dst_volid;
7836 my $storecfg = PVE
::Storage
::config
();
7837 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7839 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7841 $format = qemu_img_format
($dst_scfg, $dst_volname);
7843 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7845 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7848 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7849 $opts->{format
} = $format if $format;
7851 if (defined($src_bitmap)) {
7852 $opts->{sync
} = 'incremental';
7853 $opts->{bitmap
} = $src_bitmap;
7854 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7857 if (defined($bwlimit)) {
7858 $opts->{speed
} = $bwlimit * 1024;
7859 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7861 print "drive mirror is starting for drive-$drive\n";
7864 # if a job already runs for this device we get an error, catch it for cleanup
7865 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7867 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7869 die "mirroring error: $err\n";
7872 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7875 # $completion can be either
7876 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7877 # 'cancel': wait until all jobs are ready, block-job-cancel them
7878 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7879 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7880 sub qemu_drive_mirror_monitor
{
7881 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7883 $completion //= 'complete';
7887 my $err_complete = 0;
7889 my $starttime = time ();
7891 die "block job ('$op') timed out\n" if $err_complete > 300;
7893 my $stats = mon_cmd
($vmid, "query-block-jobs");
7896 my $running_jobs = {};
7897 for my $stat (@$stats) {
7898 next if $stat->{type
} ne $op;
7899 $running_jobs->{$stat->{device
}} = $stat;
7902 my $readycounter = 0;
7904 for my $job_id (sort keys %$jobs) {
7905 my $job = $running_jobs->{$job_id};
7907 my $vanished = !defined($job);
7908 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7909 if($complete || ($vanished && $completion eq 'auto')) {
7910 print "$job_id: $op-job finished\n";
7911 delete $jobs->{$job_id};
7915 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7917 my $busy = $job->{busy
};
7918 my $ready = $job->{ready
};
7919 if (my $total = $job->{len
}) {
7920 my $transferred = $job->{offset
} || 0;
7921 my $remaining = $total - $transferred;
7922 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7924 my $duration = $ctime - $starttime;
7925 my $total_h = render_bytes
($total, 1);
7926 my $transferred_h = render_bytes
($transferred, 1);
7928 my $status = sprintf(
7929 "transferred $transferred_h of $total_h ($percent%%) in %s",
7930 render_duration
($duration),
7935 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7937 $status .= ", ready";
7940 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
7941 $jobs->{$job_id}->{ready
} = $ready;
7944 $readycounter++ if $job->{ready
};
7947 last if scalar(keys %$jobs) == 0;
7949 if ($readycounter == scalar(keys %$jobs)) {
7950 print "all '$op' jobs are ready\n";
7952 # do the complete later (or has already been done)
7953 last if $completion eq 'skip' || $completion eq 'auto';
7955 if ($vmiddst && $vmiddst != $vmid) {
7956 my $agent_running = $qga && qga_check_running
($vmid);
7957 if ($agent_running) {
7958 print "freeze filesystem\n";
7959 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
7962 print "suspend vm\n";
7963 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
7967 # if we clone a disk for a new target vm, we don't switch the disk
7968 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
7970 if ($agent_running) {
7971 print "unfreeze filesystem\n";
7972 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
7975 print "resume vm\n";
7976 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
7983 for my $job_id (sort keys %$jobs) {
7984 # try to switch the disk if source and destination are on the same guest
7985 print "$job_id: Completing block job_id...\n";
7988 if ($completion eq 'complete') {
7989 $op = 'block-job-complete';
7990 } elsif ($completion eq 'cancel') {
7991 $op = 'block-job-cancel';
7993 die "invalid completion value: $completion\n";
7995 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
7996 if ($@ =~ m/cannot be completed/) {
7997 print "$job_id: block job cannot be completed, trying again.\n";
8000 print "$job_id: Completed successfully.\n";
8001 $jobs->{$job_id}->{complete
} = 1;
8012 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
8013 die "block job ($op) error: $err";
8017 sub qemu_blockjobs_cancel
{
8018 my ($vmid, $jobs) = @_;
8020 foreach my $job (keys %$jobs) {
8021 print "$job: Cancelling block job\n";
8022 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
8023 $jobs->{$job}->{cancel
} = 1;
8027 my $stats = mon_cmd
($vmid, "query-block-jobs");
8029 my $running_jobs = {};
8030 foreach my $stat (@$stats) {
8031 $running_jobs->{$stat->{device
}} = $stat;
8034 foreach my $job (keys %$jobs) {
8036 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
8037 print "$job: Done.\n";
8038 delete $jobs->{$job};
8042 last if scalar(keys %$jobs) == 0;
8048 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8049 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8050 my sub clone_disk_check_io_uring
{
8051 my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8053 return if !$use_drive_mirror;
8055 # Don't complain when not changing storage.
8056 # Assume if it works for the source, it'll work for the target too.
8057 return if $src_storeid eq $dst_storeid;
8059 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
8060 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
8062 my $cache_direct = drive_uses_cache_direct
($src_drive);
8064 my $src_uses_io_uring;
8065 if ($src_drive->{aio
}) {
8066 $src_uses_io_uring = $src_drive->{aio
} eq 'io_uring';
8068 $src_uses_io_uring = storage_allows_io_uring_default
($src_scfg, $cache_direct);
8071 die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8072 if $src_uses_io_uring && !storage_allows_io_uring_default
($dst_scfg, $cache_direct);
8076 my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8078 my ($vmid, $running) = $source->@{qw(vmid running)};
8079 my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8081 my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8082 my ($storage, $format) = $dest->@{qw(storage format)};
8084 my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8086 if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8087 die "cloning from/to EFI disk requires EFI disk\n"
8088 if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8089 die "cloning from/to TPM state requires TPM state\n"
8090 if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8092 # This would lead to two device nodes in QEMU pointing to the same backing image!
8093 die "cannot change drive name when cloning disk from/to the same VM\n"
8094 if $use_drive_mirror && $vmid == $newvmid;
8097 die "cannot move TPM state while VM is running\n"
8098 if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8102 print "create " . ($full ?
'full' : 'linked') . " clone of drive ";
8103 print "$src_drivename " if $src_drivename;
8104 print "($drive->{file})\n";
8107 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
8108 push @$newvollist, $newvolid;
8110 my ($src_storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
8111 my $storeid = $storage || $src_storeid;
8113 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
8117 if (drive_is_cloudinit
($drive)) {
8118 $name = "vm-$newvmid-cloudinit";
8119 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8120 if ($scfg->{path
}) {
8121 $name .= ".$dst_format";
8124 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
8125 } elsif ($dst_drivename eq 'efidisk0') {
8126 $size = $efisize or die "internal error - need to specify EFI disk size\n";
8127 } elsif ($dst_drivename eq 'tpmstate0') {
8128 $dst_format = 'raw';
8129 $size = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8131 clone_disk_check_io_uring
($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8133 $size = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
8135 $newvolid = PVE
::Storage
::vdisk_alloc
(
8136 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8138 push @$newvollist, $newvolid;
8140 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
8142 if (drive_is_cloudinit
($drive)) {
8143 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8144 # if this is the case, we have to complete any block-jobs still there from
8145 # previous drive-mirrors
8146 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8147 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
8152 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
8153 if ($use_drive_mirror) {
8154 qemu_drive_mirror
($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8155 $completion, $qga, $bwlimit);
8157 if ($dst_drivename eq 'efidisk0') {
8158 # the relevant data on the efidisk may be smaller than the source
8159 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8160 # that is given by the OVMF_VARS.fd
8161 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
8162 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
8164 my $src_format = (PVE
::Storage
::parse_volname
($storecfg, $drive->{file
}))[6];
8166 # better for Ceph if block size is not too small, see bug #3324
8169 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8171 if ($src_format eq 'qcow2' && $snapname) {
8172 die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8173 if !min_version
(kvm_user_version
(), 6, 2);
8174 push $cmd->@*, '-l', $snapname;
8176 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8179 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8185 my $size = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
8187 my $disk = dclone
($drive);
8188 delete $disk->{format
};
8189 $disk->{file
} = $newvolid;
8190 $disk->{size
} = $size if defined($size);
8195 sub get_running_qemu_version
{
8197 my $res = mon_cmd
($vmid, "query-version");
8198 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8201 sub qemu_use_old_bios_files
{
8202 my ($machine_type) = @_;
8204 return if !$machine_type;
8206 my $use_old_bios_files = undef;
8208 if ($machine_type =~ m/^(\S+)\.pxe$/) {
8210 $use_old_bios_files = 1;
8212 my $version = extract_version
($machine_type, kvm_user_version
());
8213 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8214 # load new efi bios files on migration. So this hack is required to allow
8215 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8216 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8217 $use_old_bios_files = !min_version
($version, 2, 4);
8220 return ($use_old_bios_files, $machine_type);
8223 sub get_efivars_size
{
8224 my ($conf, $efidisk) = @_;
8226 my $arch = get_vm_arch
($conf);
8227 $efidisk //= $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
8228 my $smm = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
8229 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8230 return -s
$ovmf_vars;
8233 sub update_efidisk_size
{
8236 return if !defined($conf->{efidisk0
});
8238 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
8239 $disk->{size
} = get_efivars_size
($conf);
8240 $conf->{efidisk0
} = print_drive
($disk);
8245 sub update_tpmstate_size
{
8248 my $disk = PVE
::QemuServer
::parse_drive
('tpmstate0', $conf->{tpmstate0
});
8249 $disk->{size
} = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8250 $conf->{tpmstate0
} = print_drive
($disk);
8253 sub create_efidisk
($$$$$$$) {
8254 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8256 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8258 my $vars_size_b = -s
$ovmf_vars;
8259 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
8260 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8261 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
8263 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
8264 my $size = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
8266 return ($volid, $size/1024);
8269 sub vm_iothreads_list
{
8272 my $res = mon_cmd
($vmid, 'query-iothreads');
8275 foreach my $iothread (@$res) {
8276 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
8283 my ($conf, $drive) = @_;
8287 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
8289 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
8295 my $controller = int($drive->{index} / $maxdev);
8296 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
8300 return ($maxdev, $controller, $controller_prefix);
8303 sub resolve_dst_disk_format
{
8304 my ($storecfg, $storeid, $src_volname, $format) = @_;
8305 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
8308 # if no target format is specified, use the source disk format as hint
8310 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8311 $format = qemu_img_format
($scfg, $src_volname);
8317 # test if requested format is supported - else use default
8318 my $supported = grep { $_ eq $format } @$validFormats;
8319 $format = $defFormat if !$supported;
8323 # NOTE: if this logic changes, please update docs & possibly gui logic
8324 sub find_vmstate_storage
{
8325 my ($conf, $storecfg) = @_;
8327 # first, return storage from conf if set
8328 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
8330 my ($target, $shared, $local);
8332 foreach_storage_used_by_vm
($conf, sub {
8334 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
8335 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
8336 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
8339 # second, use shared storage where VM has at least one disk
8340 # third, use local storage where VM has at least one disk
8341 # fall back to local storage
8342 $target = $shared // $local // 'local';
8348 my ($uuid, $uuid_str);
8349 UUID
::generate
($uuid);
8350 UUID
::unparse
($uuid, $uuid_str);
8354 sub generate_smbios1_uuid
{
8355 return "uuid=".generate_uuid
();
8361 mon_cmd
($vmid, 'nbd-server-stop', timeout
=> 25);
8364 sub create_reboot_request
{
8366 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8367 or die "failed to create reboot trigger file: $!\n";
8371 sub clear_reboot_request
{
8373 my $path = "/run/qemu-server/$vmid.reboot";
8376 $res = unlink($path);
8377 die "could not remove reboot request for $vmid: $!"
8378 if !$res && $! != POSIX
::ENOENT
;
8383 sub bootorder_from_legacy
{
8384 my ($conf, $bootcfg) = @_;
8386 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
8387 my $bootindex_hash = {};
8389 foreach my $o (split(//, $boot)) {
8390 $bootindex_hash->{$o} = $i*100;
8396 PVE
::QemuConfig-
>foreach_volume($conf, sub {
8397 my ($ds, $drive) = @_;
8399 if (drive_is_cdrom
($drive, 1)) {
8400 if ($bootindex_hash->{d
}) {
8401 $bootorder->{$ds} = $bootindex_hash->{d
};
8402 $bootindex_hash->{d
} += 1;
8404 } elsif ($bootindex_hash->{c
}) {
8405 $bootorder->{$ds} = $bootindex_hash->{c
}
8406 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
8407 $bootindex_hash->{c
} += 1;
8411 if ($bootindex_hash->{n
}) {
8412 for (my $i = 0; $i < $MAX_NETS; $i++) {
8413 my $netname = "net$i";
8414 next if !$conf->{$netname};
8415 $bootorder->{$netname} = $bootindex_hash->{n
};
8416 $bootindex_hash->{n
} += 1;
8423 # Generate default device list for 'boot: order=' property. Matches legacy
8424 # default boot order, but with explicit device names. This is important, since
8425 # the fallback for when neither 'order' nor the old format is specified relies
8426 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8427 sub get_default_bootdevices
{
8433 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
8434 push @ret, $first if $first;
8437 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
8438 push @ret, $first if $first;
8441 for (my $i = 0; $i < $MAX_NETS; $i++) {
8442 my $netname = "net$i";
8443 next if !$conf->{$netname};
8444 push @ret, $netname;
8451 sub device_bootorder
{
8454 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
8456 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
8459 if (!defined($boot) || $boot->{legacy
}) {
8460 $bootorder = bootorder_from_legacy
($conf, $boot);
8461 } elsif ($boot->{order
}) {
8462 my $i = 100; # start at 100 to allow user to insert devices before us with -args
8463 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
8464 $bootorder->{$dev} = $i++;
8471 sub register_qmeventd_handle
{
8475 my $peer = "/var/run/qmeventd.sock";
8480 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
8482 if ($! != EINTR
&& $! != EAGAIN
) {
8483 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8486 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8487 . "after $count retries\n";
8492 # send handshake to mark VM as backing up
8493 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
8495 # return handle to be closed later when inhibit is no longer required
8499 # bash completion helper
8501 sub complete_backup_archives
{
8502 my ($cmdname, $pname, $cvalue) = @_;
8504 my $cfg = PVE
::Storage
::config
();
8508 if ($cvalue =~ m/^([^:]+):/) {
8512 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
8515 foreach my $id (keys %$data) {
8516 foreach my $item (@{$data->{$id}}) {
8517 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8518 push @$res, $item->{volid
} if defined($item->{volid
});
8525 my $complete_vmid_full = sub {
8528 my $idlist = vmstatus
();
8532 foreach my $id (keys %$idlist) {
8533 my $d = $idlist->{$id};
8534 if (defined($running)) {
8535 next if $d->{template
};
8536 next if $running && $d->{status
} ne 'running';
8537 next if !$running && $d->{status
} eq 'running';
8546 return &$complete_vmid_full();
8549 sub complete_vmid_stopped
{
8550 return &$complete_vmid_full(0);
8553 sub complete_vmid_running
{
8554 return &$complete_vmid_full(1);
8557 sub complete_storage
{
8559 my $cfg = PVE
::Storage
::config
();
8560 my $ids = $cfg->{ids
};
8563 foreach my $sid (keys %$ids) {
8564 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
8565 next if !$ids->{$sid}->{content
}->{images
};
8572 sub complete_migration_storage
{
8573 my ($cmd, $param, $current_value, $all_args) = @_;
8575 my $targetnode = @$all_args[1];
8577 my $cfg = PVE
::Storage
::config
();
8578 my $ids = $cfg->{ids
};
8581 foreach my $sid (keys %$ids) {
8582 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
8583 next if !$ids->{$sid}->{content
}->{images
};
8591 my ($vmid, $include_suspended) = @_;
8592 my $qmpstatus = eval {
8593 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
8594 mon_cmd
($vmid, "query-status");
8597 return $qmpstatus && (
8598 $qmpstatus->{status
} eq "paused" ||
8599 $qmpstatus->{status
} eq "prelaunch" ||
8600 ($include_suspended && $qmpstatus->{status
} eq "suspended")
8604 sub check_volume_storage_type
{
8605 my ($storecfg, $vol) = @_;
8607 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
8608 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8609 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
8611 die "storage '$storeid' does not support content-type '$vtype'\n"
8612 if !$scfg->{content
}->{$vtype};
8617 sub add_nets_bridge_fdb
{
8618 my ($conf, $vmid) = @_;
8620 for my $opt (keys %$conf) {
8621 next if $opt !~ m/^net(\d+)$/;
8622 my $iface = "tap${vmid}i$1";
8623 # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8624 my $net = parse_net
($conf->{$opt}, 1) or next;
8626 my $mac = $net->{macaddr
};
8628 log_warn
("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8629 if !file_read_firstline
("/sys/class/net/$iface/brport/learning");
8633 my $bridge = $net->{bridge
};
8635 log_warn
("Interface '$iface' not attached to any bridge.");
8639 PVE
::Network
::SDN
::Zones
::add_bridge_fdb
($iface, $mac, $bridge);
8640 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8641 PVE
::Network
::add_bridge_fdb
($iface, $mac);
8646 sub del_nets_bridge_fdb
{
8647 my ($conf, $vmid) = @_;
8649 for my $opt (keys %$conf) {
8650 next if $opt !~ m/^net(\d+)$/;
8651 my $iface = "tap${vmid}i$1";
8653 my $net = parse_net
($conf->{$opt}) or next;
8654 my $mac = $net->{macaddr
} or next;
8656 my $bridge = $net->{bridge
};
8658 PVE
::Network
::SDN
::Zones
::del_bridge_fdb
($iface, $mac, $bridge);
8659 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8660 PVE
::Network
::del_bridge_fdb
($iface, $mac);
8665 sub create_ifaces_ipams_ips
{
8666 my ($conf, $vmid) = @_;
8668 return if !$have_sdn;
8670 foreach my $opt (keys %$conf) {
8671 if ($opt =~ m/^net(\d+)$/) {
8672 my $value = $conf->{$opt};
8673 my $net = PVE
::QemuServer
::parse_net
($value);
8674 eval { PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($net->{bridge
}, $conf->{name
}, $net->{macaddr
}, $vmid, undef, 1) };
8680 sub delete_ifaces_ipams_ips
{
8681 my ($conf, $vmid) = @_;
8683 return if !$have_sdn;
8685 foreach my $opt (keys %$conf) {
8686 if ($opt =~ m/^net(\d+)$/) {
8687 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
8688 eval { PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
}) };