1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
22 use List
::Util
qw(first);
25 use Storable
qw(dclone);
26 use Time
::HiRes
qw(gettimeofday usleep);
30 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
33 use PVE
::DataCenterConfig
;
34 use PVE
::Exception
qw(raise raise_param_exc);
35 use PVE
::Format
qw(render_duration render_bytes);
36 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
38 use PVE
::Mapping
::PCI
;
39 use PVE
::Mapping
::USB
;
41 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
44 use PVE
::RESTEnvironment
qw(log_warn);
45 use PVE
::RPCEnvironment
;
49 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
53 use PVE
::QemuServer
::Helpers
qw(config_aware_timeout min_version windows_version);
54 use PVE
::QemuServer
::Cloudinit
;
55 use PVE
::QemuServer
::CGroup
;
56 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch);
57 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
58 use PVE
::QemuServer
::Machine
;
59 use PVE
::QemuServer
::Memory
qw(get_current_memory);
60 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
61 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
62 use PVE
::QemuServer
::QMPHelpers
qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
63 use PVE
::QemuServer
::USB
;
67 require PVE
::Network
::SDN
::Zones
;
68 require PVE
::Network
::SDN
::Vnets
;
72 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
76 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
77 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
80 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
81 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
84 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
85 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
88 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
89 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
91 # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
92 # anymore. how can we deperacate this sanely without breaking existing instances, or using
93 # older backups and snapshot?
95 "$EDK2_FW_BASE/OVMF_CODE.fd",
96 "$EDK2_FW_BASE/OVMF_VARS.fd",
101 "$EDK2_FW_BASE/AAVMF_CODE.fd",
102 "$EDK2_FW_BASE/AAVMF_VARS.fd",
107 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
109 # Note about locking: we use flock on the config file protect against concurent actions.
110 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
111 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
112 # But you can ignore this kind of lock with the --skiplock flag.
120 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
121 description
=> "Some command save/restore state from this location.",
127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
130 $nodename_cache //= PVE
::INotify
::nodename
();
131 return $nodename_cache;
138 enum
=> [qw(i6300esb ib700)],
139 description
=> "Watchdog type to emulate.",
140 default => 'i6300esb',
145 enum
=> [qw(reset shutdown poweroff pause debug none)],
146 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
150 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
154 description
=> "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
159 fstrim_cloned_disks
=> {
160 description
=> "Run fstrim after moving a disk or migrating the VM.",
165 'freeze-fs-on-backup' => {
166 description
=> "Freeze/thaw guest filesystems on backup for consistency.",
172 description
=> "Select the agent type",
176 enum
=> [qw(virtio isa)],
182 description
=> "Select the VGA type.",
187 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
190 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
197 description
=> 'Enable a specific clipboard. If not set, depending on the display type the'
198 .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
209 description
=> "The size of the file in MB.",
213 pattern
=> '[a-zA-Z0-9\-]+',
215 format_description
=> 'string',
216 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
223 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
224 description
=> "Configure an audio device."
228 enum
=> ['spice', 'none'],
231 description
=> "Driver backend for the audio device."
235 my $spice_enhancements_fmt = {
240 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
244 enum
=> ['off', 'all', 'filter'],
247 description
=> "Enable video streaming. Uses compression for detected video streams."
254 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
256 description
=> "The file on the host to gather entropy from. In most cases '/dev/urandom'"
257 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
258 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
259 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
260 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
261 ." a hardware RNG from the host.",
265 description
=> "Maximum bytes of entropy allowed to get injected into the guest every"
266 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
267 ." `0` to disable limiting (potentially dangerous!).",
270 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
271 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
272 # reading from /dev/urandom
277 description
=> "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
278 ." the guest to retrieve another 'max_bytes' of entropy.",
284 my $meta_info_fmt = {
287 description
=> "The guest creation timestamp as UNIX epoch time",
293 description
=> "The QEMU (machine) version from the time this VM was created.",
294 pattern
=> '\d+(\.\d+)+',
303 description
=> "Specifies whether a VM will be started during system bootup.",
309 description
=> "Automatic restart after crash (currently ignored).",
314 type
=> 'string', format
=> 'pve-hotplug-features',
315 description
=> "Selectively enable hotplug features. This is a comma separated list of"
316 ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
317 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
318 ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
320 default => 'network,disk,usb',
325 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
331 description
=> "Lock/unlock the VM.",
332 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
337 description
=> "Limit of CPU usage.",
338 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
339 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
347 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
348 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
349 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
350 ." weights of all the other running VMs.",
353 default => 'cgroup v1: 1024, cgroup v2: 100',
358 description
=> "Memory properties.",
359 format
=> $PVE::QemuServer
::Memory
::memory_fmt
364 description
=> "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
370 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the"
371 ." more memory this VM gets. Number is relative to weights of all other running VMs."
372 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
380 description
=> "Keyboard layout for VNC server. This option is generally not required and"
381 ." is often better handled from within the guest OS.",
382 enum
=> PVE
::Tools
::kvmkeymaplist
(),
387 type
=> 'string', format
=> 'dns-name',
388 description
=> "Set a name for the VM. Only used on the configuration web interface.",
393 description
=> "SCSI controller model",
394 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
400 description
=> "Description for the VM. Shown in the web-interface VM's summary."
401 ." This is saved as comment inside the configuration file.",
402 maxLength
=> 1024 * 8,
407 # NOTE: When extending, also consider extending `%guest_types` in `Import/ESXi.pm`.
408 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
409 description
=> "Specify guest operating system.",
410 verbose_description
=> <<EODESC,
411 Specify guest operating system. This is used to enable special
412 optimization/features for specific operating systems:
415 other;; unspecified OS
416 wxp;; Microsoft Windows XP
417 w2k;; Microsoft Windows 2000
418 w2k3;; Microsoft Windows 2003
419 w2k8;; Microsoft Windows 2008
420 wvista;; Microsoft Windows Vista
421 win7;; Microsoft Windows 7
422 win8;; Microsoft Windows 8/2012/2012r2
423 win10;; Microsoft Windows 10/2016/2019
424 win11;; Microsoft Windows 11/2022/2025
425 l24;; Linux 2.4 Kernel
426 l26;; Linux 2.6 - 6.X Kernel
427 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
432 type
=> 'string', format
=> 'pve-qm-boot',
433 description
=> "Specify guest boot order. Use the 'order=' sub-property as usage with no"
434 ." key or 'legacy=' is deprecated.",
438 type
=> 'string', format
=> 'pve-qm-bootdisk',
439 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
440 pattern
=> '(ide|sata|scsi|virtio)\d+',
445 description
=> "The number of CPUs. Please use option -sockets instead.",
452 description
=> "The number of CPU sockets.",
459 description
=> "The number of cores per socket.",
466 description
=> "Enable/disable NUMA.",
472 description
=> "Enable/disable hugepages memory.",
473 enum
=> [qw(any 2 1024)],
479 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
480 ." after VM shutdown and can be used for subsequent starts.",
485 description
=> "Number of hotplugged vcpus.",
492 description
=> "Enable/disable ACPI.",
497 description
=> "Enable/disable communication with the QEMU Guest Agent and its properties.",
499 format
=> $agent_fmt,
504 description
=> "Enable/disable KVM hardware virtualization.",
510 description
=> "Enable/disable time drift fix.",
516 description
=> "Set the real time clock (RTC) to local time. This is enabled by default if"
517 ." the `ostype` indicates a Microsoft Windows OS.",
522 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
526 type
=> 'string', format
=> $vga_fmt,
527 description
=> "Configure the VGA hardware.",
528 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
529 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
530 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
531 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
532 ." display server. For win* OS you can select how many independent displays you want,"
533 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
534 ." using a serial device as terminal.",
538 type
=> 'string', format
=> 'pve-qm-watchdog',
539 description
=> "Create a virtual hardware watchdog device.",
540 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
541 ." action), the watchdog must be periodically polled by an agent inside the guest or"
542 ." else the watchdog will reset the guest (or execute the respective action specified)",
547 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
548 description
=> "Set the initial date of the real time clock. Valid format for date are:"
549 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
550 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
553 startup
=> get_standard_option
('pve-startup-order'),
557 description
=> "Enable/disable Template.",
563 description
=> "Arbitrary arguments passed to kvm.",
564 verbose_description
=> <<EODESCR,
565 Arbitrary arguments passed to kvm, for example:
567 args: -no-reboot -smbios 'type=0,vendor=FOO'
569 NOTE: this option is for experts only.
576 description
=> "Enable/disable the USB tablet device.",
577 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
578 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
579 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
580 ." may consider disabling this to save some context switches. This is turned off by"
581 ." default if you use spice (`qm set <vmid> --vga qxl`).",
586 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
590 migrate_downtime
=> {
593 description
=> "Set maximum tolerated downtime (in seconds) for migrations. Should the"
594 ." migration not be able to converge in the very end, because too much newly dirtied"
595 ." RAM needs to be transferred, the limit will be increased automatically step-by-step"
596 ." until migration can converge.",
602 type
=> 'string', format
=> 'pve-qm-ide',
603 typetext
=> '<volume>',
604 description
=> "This is an alias for option -ide2",
608 description
=> "Emulated CPU type.",
610 format
=> 'pve-vm-cpu-conf',
612 parent
=> get_standard_option
('pve-snapshot-name', {
614 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
618 description
=> "Timestamp for snapshots.",
624 type
=> 'string', format
=> 'pve-volume-id',
625 description
=> "Reference to a volume which stores the VM state. This is used internally"
628 vmstatestorage
=> get_standard_option
('pve-storage-id', {
629 description
=> "Default storage for VM state volumes/files.",
632 runningmachine
=> get_standard_option
('pve-qemu-machine', {
633 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
637 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
638 ." internally for snapshots.",
641 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
642 format_description
=> 'QEMU -cpu parameter'
644 machine
=> get_standard_option
('pve-qemu-machine'),
646 description
=> "Virtual processor architecture. Defaults to the host.",
649 enum
=> [qw(x86_64 aarch64)],
652 description
=> "Specify SMBIOS type 1 fields.",
653 type
=> 'string', format
=> 'pve-qm-smbios1',
660 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
661 ." remove disk operations.",
667 enum
=> [ qw(seabios ovmf) ],
668 description
=> "Select BIOS implementation.",
669 default => 'seabios',
673 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
674 format_description
=> 'UUID',
675 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
676 ." to disable explicitly.",
677 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
678 ." value identifier to the guest OS. This allows to notify the guest operating system"
679 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
680 ." execution or creation from a template). The guest operating system notices the"
681 ." change, and is then able to react as appropriate by marking its copies of"
682 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
683 ."Note that auto-creation only works when done through API/CLI create or update methods"
684 .", but not when manually editing the config file.",
685 default => "1 (autogenerated)",
690 format
=> 'pve-volume-id',
692 description
=> "Script that will be executed during various steps in the vms lifetime.",
696 format
=> $ivshmem_fmt,
697 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
703 format
=> $audio_fmt,
704 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
707 spice_enhancements
=> {
709 format
=> $spice_enhancements_fmt,
710 description
=> "Configure additional enhancements for SPICE.",
714 type
=> 'string', format
=> 'pve-tag-list',
715 description
=> 'Tags of the VM. This is only meta information.',
721 description
=> "Configure a VirtIO-based Random Number Generator.",
726 format
=> $meta_info_fmt,
727 description
=> "Some (read-only) meta-information about this guest.",
731 type
=> 'string', format
=> 'pve-cpuset',
732 description
=> "List of host cores used to execute guest processes, for example: 0,5,8-11",
741 description
=> 'Specify a custom file containing all meta data passed to the VM via"
742 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
743 format
=> 'pve-volume-id',
744 format_description
=> 'volume',
749 description
=> 'To pass a custom file containing all network data to the VM via cloud-init.',
750 format
=> 'pve-volume-id',
751 format_description
=> 'volume',
756 description
=> 'To pass a custom file containing all user data to the VM via cloud-init.',
757 format
=> 'pve-volume-id',
758 format_description
=> 'volume',
763 description
=> 'To pass a custom file containing all vendor data to the VM via cloud-init.',
764 format
=> 'pve-volume-id',
765 format_description
=> 'volume',
768 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
770 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
771 my $confdesc_cloudinit = {
775 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
776 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
777 .' and `configdrive2` for windows.',
778 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
783 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
784 ." image's configured default user.",
789 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
790 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
791 .' support hashed passwords.',
796 description
=> 'cloud-init: do an automatic package upgrade after the first boot.',
802 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
804 format
=> 'pve-qm-cicustom',
809 description
=> 'cloud-init: Sets DNS search domains for a container. Create will'
810 .' automatically use the setting from the host if neither searchdomain nor nameserver'
815 type
=> 'string', format
=> 'address-list',
816 description
=> 'cloud-init: Sets DNS server IP address for a container. Create will'
817 .' automatically use the setting from the host if neither searchdomain nor nameserver'
823 format
=> 'urlencoded',
824 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
828 # what about other qemu settings ?
830 #machine => 'string',
843 ##soundhw => 'string',
845 while (my ($k, $v) = each %$confdesc) {
846 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
850 my $MAX_SERIAL_PORTS = 4;
851 my $MAX_PARALLEL_PORTS = 3;
853 for (my $i = 0; $i < $PVE::QemuServer
::Memory
::MAX_NUMA
; $i++) {
854 $confdesc->{"numa$i"} = $PVE::QemuServer
::Memory
::numadesc
;
857 my $nic_model_list = [
873 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
875 my $net_fmt_bridge_descr = <<__EOD__;
876 Bridge to attach the network device to. The Proxmox VE standard bridge
879 If you do not specify a bridge, we create a kvm user (NATed) network
880 device, which provides DHCP and DNS services. The following addresses
887 The DHCP server assign addresses to the guest starting from 10.0.2.15.
891 macaddr
=> get_standard_option
('mac-addr', {
892 description
=> "MAC address. That address must be unique withing your network. This is"
893 ." automatically generated if not specified.",
897 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
898 ." very low CPU overhead. If your guest does not support this driver, it is usually"
899 ." best to use 'e1000'.",
900 enum
=> $nic_model_list,
903 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
904 bridge
=> get_standard_option
('pve-bridge-id', {
905 description
=> $net_fmt_bridge_descr,
910 minimum
=> 0, maximum
=> 64,
911 description
=> 'Number of packet queues to be used on the device.',
917 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
922 minimum
=> 1, maximum
=> 4094,
923 description
=> 'VLAN tag to apply to packets on this interface.',
928 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
929 description
=> 'VLAN trunks to pass through this interface.',
930 format_description
=> 'vlanid[;vlanid...]',
935 description
=> 'Whether this interface should be protected by the firewall.',
940 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
945 minimum
=> 1, maximum
=> 65520,
946 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
953 type
=> 'string', format
=> $net_fmt,
954 description
=> "Specify network devices.",
957 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
962 format
=> 'pve-ipv4-config',
963 format_description
=> 'IPv4Format/CIDR',
964 description
=> 'IPv4 address in CIDR format.',
971 format_description
=> 'GatewayIPv4',
972 description
=> 'Default gateway for IPv4 traffic.',
978 format
=> 'pve-ipv6-config',
979 format_description
=> 'IPv6Format/CIDR',
980 description
=> 'IPv6 address in CIDR format.',
987 format_description
=> 'GatewayIPv6',
988 description
=> 'Default gateway for IPv6 traffic.',
993 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
996 type
=> 'string', format
=> 'pve-qm-ipconfig',
997 description
=> <<'EODESCR',
998 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1000 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1002 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1003 gateway should be provided.
1004 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1005 cloud-init 19.4 or newer.
1007 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1011 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
1013 for (my $i = 0; $i < $MAX_NETS; $i++) {
1014 $confdesc->{"net$i"} = $netdesc;
1015 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1018 foreach my $key (keys %$confdesc_cloudinit) {
1019 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1022 PVE
::JSONSchema
::register_format
('pve-cpuset', \
&pve_verify_cpuset
);
1023 sub pve_verify_cpuset
{
1024 my ($set_text, $noerr) = @_;
1026 my ($count, $members) = eval { PVE
::CpuSet
::parse_cpuset
($set_text) };
1030 die "unable to parse cpuset option\n";
1033 return PVE
::CpuSet-
>new($members)->short_string();
1036 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1037 sub verify_volume_id_or_qm_path
{
1038 my ($volid, $noerr) = @_;
1040 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1042 return verify_volume_id_or_absolute_path
($volid, $noerr);
1045 PVE
::JSONSchema
::register_format
('pve-volume-id-or-absolute-path', \
&verify_volume_id_or_absolute_path
);
1046 sub verify_volume_id_or_absolute_path
{
1047 my ($volid, $noerr) = @_;
1049 return $volid if $volid =~ m
|^/|;
1051 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1062 pattern
=> '(/dev/.+|socket)',
1063 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1064 verbose_description
=> <<EODESCR,
1065 Create a serial device inside the VM (n is 0 to 3), and pass through a
1066 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1067 host side (use 'qm terminal' to open a terminal connection).
1069 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1070 use with special care.
1072 CAUTION: Experimental! User reported problems with this option.
1079 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1080 description
=> "Map host parallel devices (n is 0 to 2).",
1081 verbose_description
=> <<EODESCR,
1082 Map host parallel devices (n is 0 to 2).
1084 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1085 machines - use with special care.
1087 CAUTION: Experimental! User reported problems with this option.
1091 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1092 $confdesc->{"parallel$i"} = $paralleldesc;
1095 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1096 $confdesc->{"serial$i"} = $serialdesc;
1099 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1100 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1103 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1104 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1107 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
1108 $confdesc->{"usb$i"} = $PVE::QemuServer
::USB
::usbdesc
;
1116 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1117 . " Deprecated, use 'order=' instead.",
1118 pattern
=> '[acdn]{1,4}',
1119 format_description
=> "[acdn]{1,4}",
1121 # note: this is also the fallback if boot: is not given at all
1127 format
=> 'pve-qm-bootdev-list',
1128 format_description
=> "device[;device...]",
1129 description
=> <<EODESC,
1130 The guest will attempt to boot from devices in the order they appear here.
1132 Disks, optical drives and passed-through storage USB devices will be directly
1133 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1134 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1136 Note that only devices in this list will be marked as bootable and thus loaded
1137 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1138 (e.g. software-raid), you need to specify all of them here.
1140 Overrides the deprecated 'legacy=[acdn]*' value when given.
1144 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1146 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1147 sub verify_bootdev
{
1148 my ($dev, $noerr) = @_;
1150 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1151 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && !$special;
1155 return 0 if $dev !~ m/^$base\d+$/;
1156 return 0 if !$confdesc->{$dev};
1160 return $dev if $check->("net");
1161 return $dev if $check->("usb");
1162 return $dev if $check->("hostpci");
1165 die "invalid boot device '$dev'\n";
1168 sub print_bootorder
{
1170 return "" if !@$devs;
1171 my $data = { order
=> join(';', @$devs) };
1172 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1175 my $kvm_api_version = 0;
1178 return $kvm_api_version if $kvm_api_version;
1180 open my $fh, '<', '/dev/kvm' or return;
1182 # 0xae00 => KVM_GET_API_VERSION
1183 $kvm_api_version = ioctl($fh, 0xae00, 0);
1186 return $kvm_api_version;
1189 my $kvm_user_version = {};
1192 sub kvm_user_version
{
1195 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1196 my $st = stat($binary);
1198 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1199 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1200 $cachedmtime == $st->mtime;
1202 $kvm_user_version->{$binary} = 'unknown';
1203 $kvm_mtime->{$binary} = $st->mtime;
1207 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1208 $kvm_user_version->{$binary} = $2;
1212 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1215 return $kvm_user_version->{$binary};
1218 my sub extract_version
{
1219 my ($machine_type, $version) = @_;
1220 $version = kvm_user_version
() if !defined($version);
1221 return PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1224 sub kernel_has_vhost_net
{
1225 return -c
'/dev/vhost-net';
1230 return defined($confdesc->{$key});
1234 sub get_cdrom_path
{
1236 return $cdrom_path if defined($cdrom_path);
1238 $cdrom_path = first
{ -l
$_ } map { "/dev/cdrom$_" } ('', '1', '2');
1240 if (!defined($cdrom_path)) {
1241 log_warn
("no physical CD-ROM available, ignoring");
1249 my ($storecfg, $vmid, $cdrom) = @_;
1251 if ($cdrom eq 'cdrom') {
1252 return get_cdrom_path
();
1253 } elsif ($cdrom eq 'none') {
1255 } elsif ($cdrom =~ m
|^/|) {
1258 return PVE
::Storage
::path
($storecfg, $cdrom);
1262 # try to convert old style file names to volume IDs
1263 sub filename_to_volume_id
{
1264 my ($vmid, $file, $media) = @_;
1266 if (!($file eq 'none' || $file eq 'cdrom' ||
1267 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1269 return if $file =~ m
|/|;
1271 if ($media && $media eq 'cdrom') {
1272 $file = "local:iso/$file";
1274 $file = "local:$vmid/$file";
1281 sub verify_media_type
{
1282 my ($opt, $vtype, $media) = @_;
1287 if ($media eq 'disk') {
1289 } elsif ($media eq 'cdrom') {
1292 die "internal error";
1295 return if ($vtype eq $etype);
1297 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1300 sub cleanup_drive_path
{
1301 my ($opt, $storecfg, $drive) = @_;
1303 # try to convert filesystem paths to volume IDs
1305 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1306 ($drive->{file
} !~ m
|^/dev/.+|) &&
1307 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1308 ($drive->{file
} !~ m/^\d+$/)) {
1309 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1310 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1312 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1313 verify_media_type
($opt, $vtype, $drive->{media
});
1314 $drive->{file
} = $volid;
1317 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1320 sub parse_hotplug_features
{
1325 return $res if $data eq '0';
1327 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1329 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1330 if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1333 die "invalid hotplug feature '$feature'\n";
1339 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1340 sub pve_verify_hotplug_features
{
1341 my ($value, $noerr) = @_;
1343 return $value if parse_hotplug_features
($value);
1347 die "unable to parse hotplug option\n";
1350 sub assert_clipboard_config
{
1353 my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1357 && $vga->{'clipboard'} eq 'vnc'
1359 && $vga->{type
} !~ $clipboard_regex
1361 die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1365 sub print_tabletdevice_full
{
1366 my ($conf, $arch) = @_;
1368 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1370 # we use uhci for old VMs because tablet driver was buggy in older qemu
1372 if ($q35 || $arch eq 'aarch64') {
1378 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1381 sub print_keyboarddevice_full
{
1382 my ($conf, $arch) = @_;
1384 return if $arch ne 'aarch64';
1386 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1389 my sub get_drive_id
{
1391 return "$drive->{interface}$drive->{index}";
1394 sub print_drivedevice_full
{
1395 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1400 my $drive_id = get_drive_id
($drive);
1401 if ($drive->{interface
} eq 'virtio') {
1402 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1403 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1404 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1405 } elsif ($drive->{interface
} eq 'scsi') {
1407 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1408 my $unit = $drive->{index} % $maxdev;
1410 my $machine_version = extract_version
($machine_type, kvm_user_version
());
1411 my $device_type = PVE
::QemuServer
::Drive
::get_scsi_device_type
(
1412 $drive, $storecfg, $machine_version);
1414 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1415 $device = "scsi-$device_type,bus=$controller_prefix$controller.0,scsi-id=$unit";
1417 $device = "scsi-$device_type,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1418 .",lun=$drive->{index}";
1420 $device .= ",drive=drive-$drive_id,id=$drive_id";
1422 if ($drive->{ssd
} && ($device_type eq 'block' || $device_type eq 'hd')) {
1423 $device .= ",rotation_rate=1";
1425 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1427 # only scsi-hd and scsi-cd support passing vendor and product information
1428 if ($device_type eq 'hd' || $device_type eq 'cd') {
1429 if (my $vendor = $drive->{vendor
}) {
1430 $device .= ",vendor=$vendor";
1432 if (my $product = $drive->{product
}) {
1433 $device .= ",product=$product";
1437 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1438 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1439 my $controller = int($drive->{index} / $maxdev);
1440 my $unit = $drive->{index} % $maxdev;
1442 # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1443 # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1444 # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1445 # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1446 # were before. Move odd ones up by 2 where they don't clash.
1447 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) && $drive->{interface
} eq 'ide') {
1448 $controller += 2 * ($unit % 2);
1452 my $device_type = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1454 $device = "ide-$device_type";
1455 if ($drive->{interface
} eq 'ide') {
1456 $device .= ",bus=ide.$controller,unit=$unit";
1458 $device .= ",bus=ahci$controller.$unit";
1460 $device .= ",drive=drive-$drive_id,id=$drive_id";
1462 if ($device_type eq 'hd') {
1463 if (my $model = $drive->{model
}) {
1464 $model = URI
::Escape
::uri_unescape
($model);
1465 $device .= ",model=$model";
1467 if ($drive->{ssd
}) {
1468 $device .= ",rotation_rate=1";
1471 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1472 } elsif ($drive->{interface
} eq 'usb') {
1474 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1476 die "unsupported interface type";
1479 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1481 if (my $serial = $drive->{serial
}) {
1482 $serial = URI
::Escape
::uri_unescape
($serial);
1483 $device .= ",serial=$serial";
1490 sub get_initiator_name
{
1493 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1494 while (defined(my $line = <$fh>)) {
1495 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1504 my sub storage_allows_io_uring_default
{
1505 my ($scfg, $cache_direct) = @_;
1507 # io_uring with cache mode writeback or writethrough on krbd will hang...
1508 return if $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1510 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1511 # sometimes, just plain disable...
1512 return if $scfg && $scfg->{type
} eq 'lvm';
1514 # io_uring causes problems when used with CIFS since kernel 5.15
1515 # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1516 return if $scfg && $scfg->{type
} eq 'cifs';
1521 my sub drive_uses_cache_direct
{
1522 my ($drive, $scfg) = @_;
1524 my $cache_direct = 0;
1526 if (my $cache = $drive->{cache
}) {
1527 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1528 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1532 return $cache_direct;
1535 sub print_drive_commandline_full
{
1536 my ($storecfg, $vmid, $drive, $live_restore_name, $io_uring) = @_;
1539 my $volid = $drive->{file
};
1540 my $format = $drive->{format
};
1541 my $drive_id = get_drive_id
($drive);
1543 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1544 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1546 if (drive_is_cdrom
($drive)) {
1547 $path = get_iso_path
($storecfg, $vmid, $volid);
1548 die "$drive_id: cannot back cdrom drive with a live restore image\n" if $live_restore_name;
1551 $path = PVE
::Storage
::path
($storecfg, $volid);
1552 $format //= qemu_img_format
($scfg, $volname);
1559 my $is_rbd = $path =~ m/^rbd:/;
1562 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1563 foreach my $o (@qemu_drive_options) {
1564 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1567 # snapshot only accepts on|off
1568 if (defined($drive->{snapshot
})) {
1569 my $v = $drive->{snapshot
} ?
'on' : 'off';
1570 $opts .= ",snapshot=$v";
1573 if (defined($drive->{ro
})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1574 $opts .= ",readonly=" . ($drive->{ro
} ?
'on' : 'off');
1577 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1578 my ($dir, $qmpname) = @$type;
1579 if (my $v = $drive->{"mbps$dir"}) {
1580 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1582 if (my $v = $drive->{"mbps${dir}_max"}) {
1583 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1585 if (my $v = $drive->{"bps${dir}_max_length"}) {
1586 $opts .= ",throttling.bps$qmpname-max-length=$v";
1588 if (my $v = $drive->{"iops${dir}"}) {
1589 $opts .= ",throttling.iops$qmpname=$v";
1591 if (my $v = $drive->{"iops${dir}_max"}) {
1592 $opts .= ",throttling.iops$qmpname-max=$v";
1594 if (my $v = $drive->{"iops${dir}_max_length"}) {
1595 $opts .= ",throttling.iops$qmpname-max-length=$v";
1599 if ($live_restore_name) {
1600 $format = "rbd" if $is_rbd;
1601 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1603 $opts .= ",format=alloc-track,file.driver=$format";
1605 $opts .= ",format=$format";
1608 my $cache_direct = drive_uses_cache_direct
($drive, $scfg);
1610 $opts .= ",cache=none" if !$drive->{cache
} && $cache_direct;
1612 if (!$drive->{aio
}) {
1613 if ($io_uring && storage_allows_io_uring_default
($scfg, $cache_direct)) {
1614 # io_uring supports all cache modes
1615 $opts .= ",aio=io_uring";
1617 # aio native works only with O_DIRECT
1619 $opts .= ",aio=native";
1621 $opts .= ",aio=threads";
1626 if (!drive_is_cdrom
($drive)) {
1628 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1629 $detectzeroes = 'off';
1630 } elsif ($drive->{discard
}) {
1631 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1633 # This used to be our default with discard not being specified:
1634 $detectzeroes = 'on';
1637 # note: 'detect-zeroes' works per blockdev and we want it to persist
1638 # after the alloc-track is removed, so put it on 'file' directly
1639 my $dz_param = $live_restore_name ?
"file.detect-zeroes" : "detect-zeroes";
1640 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1643 if ($live_restore_name) {
1644 $opts .= ",backing=$live_restore_name";
1645 $opts .= ",auto-remove=on";
1648 # my $file_param = $live_restore_name ? "file.file.filename" : "file";
1649 my $file_param = "file";
1650 if ($live_restore_name) {
1651 # non-rbd drivers require the underlying file to be a seperate block
1652 # node, so add a second .file indirection
1653 $file_param .= ".file" if !$is_rbd;
1654 $file_param .= ".filename";
1656 my $pathinfo = $path ?
"$file_param=$path," : '';
1658 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1661 sub print_pbs_blockdev
{
1662 my ($pbs_conf, $pbs_name) = @_;
1663 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1664 $blockdev .= ",repository=$pbs_conf->{repository}";
1665 $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace
};
1666 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1667 $blockdev .= ",archive=$pbs_conf->{archive}";
1668 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1672 sub print_netdevice_full
{
1673 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1675 my $device = $net->{model
};
1676 if ($net->{model
} eq 'virtio') {
1677 $device = 'virtio-net-pci';
1680 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1681 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1682 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1683 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1684 # and out of each queue plus one config interrupt and control vector queue
1685 my $vectors = $net->{queues
} * 2 + 2;
1686 $tmpstr .= ",vectors=$vectors,mq=on";
1687 if (min_version
($machine_version, 7, 1)) {
1688 $tmpstr .= ",packed=on";
1692 if (min_version
($machine_version, 7, 1) && $net->{model
} eq 'virtio'){
1693 $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1696 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1698 if (my $mtu = $net->{mtu
}) {
1699 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1700 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1703 } elsif ($mtu < 576) {
1704 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1705 } elsif ($mtu > $bridge_mtu) {
1706 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1708 $tmpstr .= ",host_mtu=$mtu";
1710 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1714 if ($use_old_bios_files) {
1716 if ($device eq 'virtio-net-pci') {
1717 $romfile = 'pxe-virtio.rom';
1718 } elsif ($device eq 'e1000') {
1719 $romfile = 'pxe-e1000.rom';
1720 } elsif ($device eq 'e1000e') {
1721 $romfile = 'pxe-e1000e.rom';
1722 } elsif ($device eq 'ne2k') {
1723 $romfile = 'pxe-ne2k_pci.rom';
1724 } elsif ($device eq 'pcnet') {
1725 $romfile = 'pxe-pcnet.rom';
1726 } elsif ($device eq 'rtl8139') {
1727 $romfile = 'pxe-rtl8139.rom';
1729 $tmpstr .= ",romfile=$romfile" if $romfile;
1735 sub print_netdev_full
{
1736 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1739 if ($netid =~ m/^net(\d+)$/) {
1743 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1745 my $ifname = "tap${vmid}i$i";
1747 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1748 die "interface name '$ifname' is too long (max 15 character)\n"
1749 if length($ifname) >= 16;
1751 my $vhostparam = '';
1752 if (is_native_arch
($arch)) {
1753 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1756 my $vmname = $conf->{name
} || "vm$vmid";
1759 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1761 if ($net->{bridge
}) {
1762 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1763 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1765 $netdev = "type=user,id=$netid,hostname=$vmname";
1768 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1774 'cirrus' => 'cirrus-vga',
1776 'vmware' => 'vmware-svga',
1777 'virtio' => 'virtio-vga',
1778 'virtio-gl' => 'virtio-vga-gl',
1781 sub print_vga_device
{
1782 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1784 my $type = $vga_map->{$vga->{type
}};
1785 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1786 $type = 'virtio-gpu';
1788 my $vgamem_mb = $vga->{memory
};
1790 my $max_outputs = '';
1792 $type = $id ?
'qxl' : 'qxl-vga';
1794 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1795 # set max outputs so linux can have up to 4 qxl displays with one device
1796 if (min_version
($machine_version, 4, 1)) {
1797 $max_outputs = ",max_outputs=4";
1802 die "no device-type for $vga->{type}\n" if !$type;
1806 if ($vga->{type
} =~ /^virtio/) {
1807 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1808 $memory = ",max_hostmem=$bytes";
1810 # from https://www.spice-space.org/multiple-monitors.html
1811 $memory = ",vgamem_mb=$vga->{memory}";
1812 my $ram = $vgamem_mb * 4;
1813 my $vram = $vgamem_mb * 2;
1814 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1816 $memory = ",vgamem_mb=$vga->{memory}";
1818 } elsif ($qxlnum && $id) {
1819 $memory = ",ram_size=67108864,vram_size=33554432";
1823 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1824 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1827 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1828 my $vgaid = "vga" . ($id // '');
1830 if ($q35 && $vgaid eq 'vga') {
1831 # the first display uses pcie.0 bus on q35 machines
1832 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1834 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1837 if ($vga->{type
} eq 'virtio-gl') {
1838 my $base = '/usr/lib/x86_64-linux-gnu/lib';
1839 die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1840 if !-e
"${base}EGL.so.1" || !-e
"${base}GL.so.1";
1842 die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1843 if !PVE
::Tools
::dir_glob_regex
('/dev/dri/', "renderD.*");
1846 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1849 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1851 my ($data, $disable_mac_autogen) = @_;
1853 my $res = eval { parse_property_string
($net_fmt, $data) };
1858 if (!defined($res->{macaddr
}) && !$disable_mac_autogen) {
1859 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1860 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1865 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1866 sub parse_ipconfig
{
1869 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1875 if ($res->{gw
} && !$res->{ip
}) {
1876 warn 'gateway specified without specifying an IP address';
1879 if ($res->{gw6
} && !$res->{ip6
}) {
1880 warn 'IPv6 gateway specified without specifying an IPv6 address';
1883 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1884 warn 'gateway specified together with DHCP';
1887 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1889 warn "IPv6 gateway specified together with $res->{ip6} address";
1893 if (!$res->{ip
} && !$res->{ip6
}) {
1894 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1903 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1906 sub add_random_macs
{
1907 my ($settings) = @_;
1909 foreach my $opt (keys %$settings) {
1910 next if $opt !~ m/^net(\d+)$/;
1911 my $net = parse_net
($settings->{$opt});
1913 $settings->{$opt} = print_net
($net);
1917 sub vm_is_volid_owner
{
1918 my ($storecfg, $vmid, $volid) = @_;
1920 if ($volid !~ m
|^/|) {
1922 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1923 if ($owner && ($owner == $vmid)) {
1931 sub vmconfig_register_unused_drive
{
1932 my ($storecfg, $vmid, $conf, $drive) = @_;
1934 if (drive_is_cloudinit
($drive)) {
1935 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1937 delete $conf->{cloudinit
};
1938 } elsif (!drive_is_cdrom
($drive)) {
1939 my $volid = $drive->{file
};
1940 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1941 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1946 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1950 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1951 format_description
=> 'UUID',
1952 description
=> "Set SMBIOS1 UUID.",
1957 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1958 format_description
=> 'Base64 encoded string',
1959 description
=> "Set SMBIOS1 version.",
1964 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1965 format_description
=> 'Base64 encoded string',
1966 description
=> "Set SMBIOS1 serial number.",
1971 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1972 format_description
=> 'Base64 encoded string',
1973 description
=> "Set SMBIOS1 manufacturer.",
1978 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1979 format_description
=> 'Base64 encoded string',
1980 description
=> "Set SMBIOS1 product ID.",
1985 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1986 format_description
=> 'Base64 encoded string',
1987 description
=> "Set SMBIOS1 SKU string.",
1992 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1993 format_description
=> 'Base64 encoded string',
1994 description
=> "Set SMBIOS1 family string.",
1999 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2007 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2014 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2017 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2019 sub parse_watchdog
{
2024 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2029 sub parse_guest_agent
{
2032 return {} if !defined($conf->{agent
});
2034 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2037 # if the agent is disabled ignore the other potentially set properties
2038 return {} if !$res->{enabled
};
2043 my ($conf, $key) = @_;
2044 return undef if !defined($conf->{agent
});
2046 my $agent = parse_guest_agent
($conf);
2047 return $agent->{$key};
2053 return {} if !$value;
2054 my $res = eval { parse_property_string
($vga_fmt, $value) };
2064 my $res = eval { parse_property_string
($rng_fmt, $value) };
2069 sub parse_meta_info
{
2074 my $res = eval { parse_property_string
($meta_info_fmt, $value) };
2079 sub new_meta_info_string
{
2080 my () = @_; # for now do not allow to override any value
2082 return PVE
::JSONSchema
::print_property_string
(
2084 'creation-qemu' => kvm_user_version
(),
2085 ctime
=> "". int(time()),
2091 sub qemu_created_version_fixups
{
2092 my ($conf, $forcemachine, $kvmver) = @_;
2094 my $meta = parse_meta_info
($conf->{meta
}) // {};
2095 my $forced_vers = PVE
::QemuServer
::Machine
::extract_version
($forcemachine);
2097 # check if we need to apply some handling for VMs that always use the latest machine version but
2098 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2099 # would be required (we do not want to pin machine version for non-windows OS type)
2100 my $machine_conf = PVE
::QemuServer
::Machine
::parse_machine
($conf->{machine
});
2102 (!defined($machine_conf->{type
}) || $machine_conf->{type
} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2103 && (!defined($meta->{'creation-qemu'}) || !min_version
($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2104 && (!$forced_vers || min_version
($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2105 && min_version
($kvmver, 6, 1) # only need to apply the change since 6.1
2107 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2108 if ($q35 && $conf->{ostype
} && $conf->{ostype
} eq 'l26') {
2109 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2110 # and thus with the predictable interface naming of systemd
2111 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2117 # add JSON properties for create and set function
2118 sub json_config_properties
{
2119 my ($prop, $with_disk_alloc) = @_;
2121 my $skip_json_config_opts = {
2125 runningmachine
=> 1,
2130 foreach my $opt (keys %$confdesc) {
2131 next if $skip_json_config_opts->{$opt};
2133 if ($with_disk_alloc && is_valid_drivename
($opt)) {
2134 $prop->{$opt} = $PVE::QemuServer
::Drive
::drivedesc_hash_with_alloc-
>{$opt};
2136 $prop->{$opt} = $confdesc->{$opt};
2143 # Properties that we can read from an OVF file
2144 sub json_ovf_properties
{
2147 for my $device (PVE
::QemuServer
::Drive
::valid_drive_names
()) {
2148 $prop->{$device} = {
2150 format
=> 'pve-volume-id-or-absolute-path',
2151 description
=> "Disk image that gets imported to $device",
2158 description
=> "The number of CPU cores.",
2163 description
=> "Amount of RAM for the VM in MB.",
2168 description
=> "Name of the VM.",
2175 # return copy of $confdesc_cloudinit to generate documentation
2176 sub cloudinit_config_properties
{
2178 return dclone
($confdesc_cloudinit);
2181 sub cloudinit_pending_properties
{
2183 map { $_ => 1 } keys $confdesc_cloudinit->%*,
2186 $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2191 my ($key, $value) = @_;
2193 die "unknown setting '$key'\n" if !$confdesc->{$key};
2195 my $type = $confdesc->{$key}->{type
};
2197 if (!defined($value)) {
2198 die "got undefined value\n";
2201 if ($value =~ m/[\n\r]/) {
2202 die "property contains a line feed\n";
2205 if ($type eq 'boolean') {
2206 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2207 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2208 die "type check ('boolean') failed - got '$value'\n";
2209 } elsif ($type eq 'integer') {
2210 return int($1) if $value =~ m/^(\d+)$/;
2211 die "type check ('integer') failed - got '$value'\n";
2212 } elsif ($type eq 'number') {
2213 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2214 die "type check ('number') failed - got '$value'\n";
2215 } elsif ($type eq 'string') {
2216 if (my $fmt = $confdesc->{$key}->{format
}) {
2217 PVE
::JSONSchema
::check_format
($fmt, $value);
2220 $value =~ s/^\"(.*)\"$/$1/;
2223 die "internal error"
2228 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2230 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2232 if (!$skiplock && !PVE
::QemuConfig-
>has_lock($conf, 'suspended')) {
2233 PVE
::QemuConfig-
>check_lock($conf);
2236 if ($conf->{template
}) {
2237 # check if any base image is still used by a linked clone
2238 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2239 my ($ds, $drive) = @_;
2240 return if drive_is_cdrom
($drive);
2242 my $volid = $drive->{file
};
2243 return if !$volid || $volid =~ m
|^/|;
2245 die "base volume '$volid' is still in use by linked cloned\n"
2246 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2252 my $remove_owned_drive = sub {
2253 my ($ds, $drive) = @_;
2254 return if drive_is_cdrom
($drive, 1);
2256 my $volid = $drive->{file
};
2257 return if !$volid || $volid =~ m
|^/|;
2258 return if $volids->{$volid};
2260 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2261 return if !$path || !$owner || ($owner != $vmid);
2263 $volids->{$volid} = 1;
2264 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2265 warn "Could not remove disk '$volid', check manually: $@" if $@;
2268 # only remove disks owned by this VM (referenced in the config)
2269 my $include_opts = {
2270 include_unused
=> 1,
2271 extra_keys
=> ['vmstate'],
2273 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2275 for my $snap (values %{$conf->{snapshots
}}) {
2276 next if !defined($snap->{vmstate
});
2277 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2278 next if !defined($drive);
2279 $remove_owned_drive->('vmstate', $drive);
2282 PVE
::QemuConfig-
>foreach_volume_full($conf->{pending
}, $include_opts, $remove_owned_drive);
2284 if ($purge_unreferenced) { # also remove unreferenced disk
2285 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2286 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2287 my ($volid, $sid, $volname, $d) = @_;
2288 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2293 eval { delete_ifaces_ipams_ips
($conf, $vmid)};
2296 if (defined $replacement_conf) {
2297 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2299 PVE
::QemuConfig-
>destroy_config($vmid);
2303 sub parse_vm_config
{
2304 my ($filename, $raw, $strict) = @_;
2306 return if !defined($raw);
2309 digest
=> Digest
::SHA
::sha1_hex
($raw),
2315 my $handle_error = sub {
2325 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2326 || die "got strange filename '$filename'";
2332 my $finish_description = sub {
2333 if (defined($descr)) {
2335 $conf->{description
} = $descr;
2341 my @lines = split(/\n/, $raw);
2342 foreach my $line (@lines) {
2343 next if $line =~ m/^\s*$/;
2345 if ($line =~ m/^\[PENDING\]\s*$/i) {
2346 $section = 'pending';
2347 $finish_description->();
2348 $conf = $res->{$section} = {};
2350 } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2351 $section = 'cloudinit';
2352 $finish_description->();
2353 $conf = $res->{$section} = {};
2356 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2358 $finish_description->();
2359 $conf = $res->{snapshots
}->{$section} = {};
2363 if ($line =~ m/^\#(.*)$/) {
2364 $descr = '' if !defined($descr);
2365 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2369 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2370 $descr = '' if !defined($descr);
2371 $descr .= PVE
::Tools
::decode_text
($2);
2372 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2373 $conf->{snapstate
} = $1;
2374 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2377 $conf->{$key} = $value;
2378 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2380 if ($section eq 'pending') {
2381 $conf->{delete} = $value; # we parse this later
2383 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2385 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2388 if ($section eq 'cloudinit') {
2389 # ignore validation only used for informative purpose
2390 $conf->{$key} = $value;
2393 eval { $value = check_type
($key, $value); };
2395 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2397 $key = 'ide2' if $key eq 'cdrom';
2398 my $fmt = $confdesc->{$key}->{format
};
2399 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2400 my $v = parse_drive
($key, $value);
2401 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2402 $v->{file
} = $volid;
2403 $value = print_drive
($v);
2405 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2410 $conf->{$key} = $value;
2413 $handle_error->("vm $vmid - unable to parse config: $line\n");
2417 $finish_description->();
2418 delete $res->{snapstate
}; # just to be sure
2423 sub write_vm_config
{
2424 my ($filename, $conf) = @_;
2426 delete $conf->{snapstate
}; # just to be sure
2428 if ($conf->{cdrom
}) {
2429 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2430 $conf->{ide2
} = $conf->{cdrom
};
2431 delete $conf->{cdrom
};
2434 # we do not use 'smp' any longer
2435 if ($conf->{sockets
}) {
2436 delete $conf->{smp
};
2437 } elsif ($conf->{smp
}) {
2438 $conf->{sockets
} = $conf->{smp
};
2439 delete $conf->{cores
};
2440 delete $conf->{smp
};
2443 my $used_volids = {};
2445 my $cleanup_config = sub {
2446 my ($cref, $pending, $snapname) = @_;
2448 foreach my $key (keys %$cref) {
2449 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2450 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2451 my $value = $cref->{$key};
2452 if ($key eq 'delete') {
2453 die "propertry 'delete' is only allowed in [PENDING]\n"
2455 # fixme: check syntax?
2458 eval { $value = check_type
($key, $value); };
2459 die "unable to parse value of '$key' - $@" if $@;
2461 $cref->{$key} = $value;
2463 if (!$snapname && is_valid_drivename
($key)) {
2464 my $drive = parse_drive
($key, $value);
2465 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2470 &$cleanup_config($conf);
2472 &$cleanup_config($conf->{pending
}, 1);
2474 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2475 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2476 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2479 # remove 'unusedX' settings if we re-add a volume
2480 foreach my $key (keys %$conf) {
2481 my $value = $conf->{$key};
2482 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2483 delete $conf->{$key};
2487 my $generate_raw_config = sub {
2488 my ($conf, $pending) = @_;
2492 # add description as comment to top of file
2493 if (defined(my $descr = $conf->{description
})) {
2495 foreach my $cl (split(/\n/, $descr)) {
2496 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2499 $raw .= "#\n" if $pending;
2503 foreach my $key (sort keys %$conf) {
2504 next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2505 $raw .= "$key: $conf->{$key}\n";
2510 my $raw = &$generate_raw_config($conf);
2512 if (scalar(keys %{$conf->{pending
}})){
2513 $raw .= "\n[PENDING]\n";
2514 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2517 if (scalar(keys %{$conf->{cloudinit
}}) && PVE
::QemuConfig-
>has_cloudinit($conf)){
2518 $raw .= "\n[special:cloudinit]\n";
2519 $raw .= &$generate_raw_config($conf->{cloudinit
});
2522 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2523 $raw .= "\n[$snapname]\n";
2524 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2534 # we use static defaults from our JSON schema configuration
2535 foreach my $key (keys %$confdesc) {
2536 if (defined(my $default = $confdesc->{$key}->{default})) {
2537 $res->{$key} = $default;
2545 my $vmlist = PVE
::Cluster
::get_vmlist
();
2547 return $res if !$vmlist || !$vmlist->{ids
};
2548 my $ids = $vmlist->{ids
};
2549 my $nodename = nodename
();
2551 foreach my $vmid (keys %$ids) {
2552 my $d = $ids->{$vmid};
2553 next if !$d->{node
} || $d->{node
} ne $nodename;
2554 next if !$d->{type
} || $d->{type
} ne 'qemu';
2555 $res->{$vmid}->{exists} = 1;
2560 # test if VM uses local resources (to prevent migration)
2561 sub check_local_resources
{
2562 my ($conf, $noerr) = @_;
2565 my $mapped_res = [];
2567 my $nodelist = PVE
::Cluster
::get_nodelist
();
2568 my $pci_map = PVE
::Mapping
::PCI
::config
();
2569 my $usb_map = PVE
::Mapping
::USB
::config
();
2571 my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2573 my $add_missing_mapping = sub {
2574 my ($type, $key, $id) = @_;
2575 for my $node (@$nodelist) {
2577 if ($type eq 'pci') {
2578 $entry = PVE
::Mapping
::PCI
::get_node_mapping
($pci_map, $id, $node);
2579 } elsif ($type eq 'usb') {
2580 $entry = PVE
::Mapping
::USB
::get_node_mapping
($usb_map, $id, $node);
2582 if (!scalar($entry->@*)) {
2583 push @{$missing_mappings_by_node->{$node}}, $key;
2588 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2589 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2591 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2593 foreach my $k (keys %$conf) {
2594 if ($k =~ m/^usb/) {
2595 my $entry = parse_property_string
('pve-qm-usb', $conf->{$k});
2596 next if $entry->{host
} && $entry->{host
} =~ m/^spice$/i;
2597 if ($entry->{mapping
}) {
2598 $add_missing_mapping->('usb', $k, $entry->{mapping
});
2599 push @$mapped_res, $k;
2602 if ($k =~ m/^hostpci/) {
2603 my $entry = parse_property_string
('pve-qm-hostpci', $conf->{$k});
2604 if ($entry->{mapping
}) {
2605 $add_missing_mapping->('pci', $k, $entry->{mapping
});
2606 push @$mapped_res, $k;
2609 # sockets are safe: they will recreated be on the target side post-migrate
2610 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2611 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2614 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2616 return wantarray ?
(\
@loc_res, $mapped_res, $missing_mappings_by_node) : \
@loc_res;
2619 # check if used storages are available on all nodes (use by migrate)
2620 sub check_storage_availability
{
2621 my ($storecfg, $conf, $node) = @_;
2623 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2624 my ($ds, $drive) = @_;
2626 my $volid = $drive->{file
};
2629 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2632 # check if storage is available on both nodes
2633 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2634 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2636 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2638 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2639 if !$scfg->{content
}->{$vtype};
2643 # list nodes where all VM images are available (used by has_feature API)
2645 my ($conf, $storecfg) = @_;
2647 my $nodelist = PVE
::Cluster
::get_nodelist
();
2648 my $nodehash = { map { $_ => 1 } @$nodelist };
2649 my $nodename = nodename
();
2651 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2652 my ($ds, $drive) = @_;
2654 my $volid = $drive->{file
};
2657 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2659 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2660 if ($scfg->{disable
}) {
2662 } elsif (my $avail = $scfg->{nodes
}) {
2663 foreach my $node (keys %$nodehash) {
2664 delete $nodehash->{$node} if !$avail->{$node};
2666 } elsif (!$scfg->{shared
}) {
2667 foreach my $node (keys %$nodehash) {
2668 delete $nodehash->{$node} if $node ne $nodename
2677 sub check_local_storage_availability
{
2678 my ($conf, $storecfg) = @_;
2680 my $nodelist = PVE
::Cluster
::get_nodelist
();
2681 my $nodehash = { map { $_ => {} } @$nodelist };
2683 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2684 my ($ds, $drive) = @_;
2686 my $volid = $drive->{file
};
2689 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2691 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2693 if ($scfg->{disable
}) {
2694 foreach my $node (keys %$nodehash) {
2695 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2697 } elsif (my $avail = $scfg->{nodes
}) {
2698 foreach my $node (keys %$nodehash) {
2699 if (!$avail->{$node}) {
2700 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2707 foreach my $node (values %$nodehash) {
2708 if (my $unavail = $node->{unavailable_storages
}) {
2709 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2716 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2718 my ($vmid, $nocheck, $node) = @_;
2720 # $nocheck is set when called during a migration, in which case the config
2721 # file might still or already reside on the *other* node
2722 # - because rename has already happened, and current node is source
2723 # - because rename hasn't happened yet, and current node is target
2724 # - because rename has happened, current node is target, but hasn't yet
2726 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2727 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2732 my $vzlist = config_list
();
2734 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2736 while (defined(my $de = $fd->read)) {
2737 next if $de !~ m/^(\d+)\.pid$/;
2739 next if !defined($vzlist->{$vmid});
2740 if (my $pid = check_running
($vmid)) {
2741 $vzlist->{$vmid}->{pid
} = $pid;
2748 our $vmstatus_return_properties = {
2749 vmid
=> get_standard_option
('pve-vmid'),
2751 description
=> "QEMU process status.",
2753 enum
=> ['stopped', 'running'],
2756 description
=> "Maximum memory in bytes.",
2759 renderer
=> 'bytes',
2762 description
=> "Root disk size in bytes.",
2765 renderer
=> 'bytes',
2768 description
=> "VM name.",
2773 description
=> "VM run state from the 'query-status' QMP monitor command.",
2778 description
=> "PID of running qemu process.",
2783 description
=> "Uptime.",
2786 renderer
=> 'duration',
2789 description
=> "Maximum usable CPUs.",
2794 description
=> "The current config lock, if any.",
2799 description
=> "The current configured tags, if any",
2803 'running-machine' => {
2804 description
=> "The currently running machine type (if running).",
2809 description
=> "The currently running QEMU version (if running).",
2815 my $last_proc_pid_stat;
2817 # get VM status information
2818 # This must be fast and should not block ($full == false)
2819 # We only query KVM using QMP if $full == true (this can be slow)
2821 my ($opt_vmid, $full) = @_;
2825 my $storecfg = PVE
::Storage
::config
();
2827 my $list = vzlist
();
2828 my $defaults = load_defaults
();
2830 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2832 my $cpucount = $cpuinfo->{cpus
} || 1;
2834 foreach my $vmid (keys %$list) {
2835 next if $opt_vmid && ($vmid ne $opt_vmid);
2837 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2839 my $d = { vmid
=> int($vmid) };
2840 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2842 # fixme: better status?
2843 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2845 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2846 if (defined($size)) {
2847 $d->{disk
} = 0; # no info available
2848 $d->{maxdisk
} = $size;
2854 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2855 * ($conf->{cores
} || $defaults->{cores
});
2856 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2857 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2859 $d->{name
} = $conf->{name
} || "VM $vmid";
2860 $d->{maxmem
} = get_current_memory
($conf->{memory
})*(1024*1024);
2862 if ($conf->{balloon
}) {
2863 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2864 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2865 : $defaults->{shares
};
2876 $d->{diskwrite
} = 0;
2878 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2880 $d->{serial
} = 1 if conf_has_serial
($conf);
2881 $d->{lock} = $conf->{lock} if $conf->{lock};
2882 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2887 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2888 foreach my $dev (keys %$netdev) {
2889 next if $dev !~ m/^tap([1-9]\d*)i/;
2891 my $d = $res->{$vmid};
2894 $d->{netout
} += $netdev->{$dev}->{receive
};
2895 $d->{netin
} += $netdev->{$dev}->{transmit
};
2898 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2899 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2904 my $ctime = gettimeofday
;
2906 foreach my $vmid (keys %$list) {
2908 my $d = $res->{$vmid};
2909 my $pid = $d->{pid
};
2912 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2913 next if !$pstat; # not running
2915 my $used = $pstat->{utime} + $pstat->{stime
};
2917 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2919 if ($pstat->{vsize
}) {
2920 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2923 my $old = $last_proc_pid_stat->{$pid};
2925 $last_proc_pid_stat->{$pid} = {
2933 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2935 if ($dtime > 1000) {
2936 my $dutime = $used - $old->{used
};
2938 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2939 $last_proc_pid_stat->{$pid} = {
2945 $d->{cpu
} = $old->{cpu
};
2949 return $res if !$full;
2951 my $qmpclient = PVE
::QMPClient-
>new();
2953 my $ballooncb = sub {
2954 my ($vmid, $resp) = @_;
2956 my $info = $resp->{'return'};
2957 return if !$info->{max_mem
};
2959 my $d = $res->{$vmid};
2961 # use memory assigned to VM
2962 $d->{maxmem
} = $info->{max_mem
};
2963 $d->{balloon
} = $info->{actual
};
2965 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2966 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2967 $d->{freemem
} = $info->{free_mem
};
2970 $d->{ballooninfo
} = $info;
2973 my $blockstatscb = sub {
2974 my ($vmid, $resp) = @_;
2975 my $data = $resp->{'return'} || [];
2976 my $totalrdbytes = 0;
2977 my $totalwrbytes = 0;
2979 for my $blockstat (@$data) {
2980 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2981 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2983 $blockstat->{device
} =~ s/drive-//;
2984 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2986 $res->{$vmid}->{diskread
} = $totalrdbytes;
2987 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2990 my $machinecb = sub {
2991 my ($vmid, $resp) = @_;
2992 my $data = $resp->{'return'} || [];
2994 $res->{$vmid}->{'running-machine'} =
2995 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
2998 my $versioncb = sub {
2999 my ($vmid, $resp) = @_;
3000 my $data = $resp->{'return'} // {};
3001 my $version = 'unknown';
3003 if (my $v = $data->{qemu
}) {
3004 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
3007 $res->{$vmid}->{'running-qemu'} = $version;
3010 my $statuscb = sub {
3011 my ($vmid, $resp) = @_;
3013 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3014 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3015 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3016 # this fails if ballon driver is not loaded, so this must be
3017 # the last commnand (following command are aborted if this fails).
3018 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3020 my $status = 'unknown';
3021 if (!defined($status = $resp->{'return'}->{status
})) {
3022 warn "unable to get VM status\n";
3026 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3029 foreach my $vmid (keys %$list) {
3030 next if $opt_vmid && ($vmid ne $opt_vmid);
3031 next if !$res->{$vmid}->{pid
}; # not running
3032 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3035 $qmpclient->queue_execute(undef, 2);
3037 foreach my $vmid (keys %$list) {
3038 next if $opt_vmid && ($vmid ne $opt_vmid);
3039 next if !$res->{$vmid}->{pid
}; #not running
3041 # we can't use the $qmpclient since it might have already aborted on
3042 # 'query-balloon', but this might also fail for older versions...
3043 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
3044 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3047 foreach my $vmid (keys %$list) {
3048 next if $opt_vmid && ($vmid ne $opt_vmid);
3049 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3055 sub conf_has_serial
{
3058 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3059 if ($conf->{"serial$i"}) {
3067 sub conf_has_audio
{
3068 my ($conf, $id) = @_;
3071 my $audio = $conf->{"audio$id"};
3072 return if !defined($audio);
3074 my $audioproperties = parse_property_string
($audio_fmt, $audio);
3075 my $audiodriver = $audioproperties->{driver
} // 'spice';
3078 dev
=> $audioproperties->{device
},
3079 dev_id
=> "audiodev$id",
3080 backend
=> $audiodriver,
3081 backend_id
=> "$audiodriver-backend${id}",
3086 my ($audio, $audiopciaddr, $machine_version) = @_;
3090 my $id = $audio->{dev_id
};
3092 if (min_version
($machine_version, 4, 2)) {
3093 $audiodev = ",audiodev=$audio->{backend_id}";
3096 if ($audio->{dev
} eq 'AC97') {
3097 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3098 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3099 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3100 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3101 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3103 die "unkown audio device '$audio->{dev}', implement me!";
3106 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3114 socket => "/var/run/qemu-server/$vmid.swtpm",
3115 pid
=> "/var/run/qemu-server/$vmid.swtpm.pid",
3119 sub add_tpm_device
{
3120 my ($vmid, $devices, $conf) = @_;
3122 return if !$conf->{tpmstate0
};
3124 my $paths = get_tpm_paths
($vmid);
3126 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3127 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3128 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3132 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3134 return if !$tpmdrive;
3137 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
3138 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
3140 $state = PVE
::Storage
::map_volume
($storecfg, $tpm->{file
});
3142 $state = $tpm->{file
};
3145 my $paths = get_tpm_paths
($vmid);
3147 # during migration, we will get state from remote
3150 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3157 "--create-platform-cert",
3160 "/etc/swtpm_setup.conf", # do not use XDG configs
3162 "0", # force creation as root, error if not possible
3163 "--not-overwrite", # ignore existing state, do not modify
3166 push @$setup_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3167 # TPM 2.0 supports ECC crypto, use if possible
3168 push @$setup_cmd, "--ecc" if $tpm->{version
} eq 'v2.0';
3170 run_command
($setup_cmd, outfunc
=> sub {
3171 print "swtpm_setup: $1\n";
3175 # Used to distinguish different invocations in the log.
3176 my $log_prefix = "[id=" . int(time()) . "] ";
3178 my $emulator_cmd = [
3182 "backend-uri=file://$state,mode=0600",
3184 "type=unixio,path=$paths->{socket},mode=0600",
3186 "file=$paths->{pid}",
3187 "--terminate", # terminate on QEMU disconnect
3190 "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3192 push @$emulator_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3193 run_command
($emulator_cmd, outfunc
=> sub { print $1; });
3195 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3196 while (! -e
$paths->{pid
}) {
3197 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3201 # return untainted PID of swtpm daemon so it can be killed on error
3202 file_read_firstline
($paths->{pid
}) =~ m/(\d+)/;
3206 sub vga_conf_has_spice
{
3209 my $vgaconf = parse_vga
($vga);
3210 my $vgatype = $vgaconf->{type
};
3211 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3218 return $conf->{arch
} // get_host_arch
();
3221 my $default_machines = {
3226 sub get_installed_machine_version
{
3227 my ($kvmversion) = @_;
3228 $kvmversion = kvm_user_version
() if !defined($kvmversion);
3229 $kvmversion =~ m/^(\d+\.\d+)/;
3233 sub windows_get_pinned_machine_version
{
3234 my ($machine, $base_version, $kvmversion) = @_;
3236 my $pin_version = $base_version;
3237 if (!defined($base_version) ||
3238 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3240 $pin_version = get_installed_machine_version
($kvmversion);
3242 if (!$machine || $machine eq 'pc') {
3243 $machine = "pc-i440fx-$pin_version";
3244 } elsif ($machine eq 'q35') {
3245 $machine = "pc-q35-$pin_version";
3246 } elsif ($machine eq 'virt') {
3247 $machine = "virt-$pin_version";
3249 warn "unknown machine type '$machine', not touching that!\n";
3255 sub get_vm_machine
{
3256 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3258 my $machine_conf = PVE
::QemuServer
::Machine
::parse_machine
($conf->{machine
});
3259 my $machine = $forcemachine || $machine_conf->{type
};
3261 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3262 $kvmversion //= kvm_user_version
();
3263 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3264 # layout which confuses windows quite a bit and may result in various regressions..
3265 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3266 if (windows_version
($conf->{ostype
})) {
3267 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3270 $machine ||= $default_machines->{$arch};
3271 if ($add_pve_version) {
3272 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3273 $machine .= "+pve$pvever";
3277 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3278 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3279 $machine = $1 if $is_pxe;
3281 # for version-pinned machines that do not include a pve-version (e.g.
3282 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3283 $machine .= '+pve0';
3285 $machine .= '.pxe' if $is_pxe;
3291 sub get_ovmf_files
($$$) {
3292 my ($arch, $efidisk, $smm) = @_;
3294 my $types = $OVMF->{$arch}
3295 or die "no OVMF images known for architecture '$arch'\n";
3297 my $type = 'default';
3298 if ($arch eq 'x86_64') {
3299 if (defined($efidisk->{efitype
}) && $efidisk->{efitype
} eq '4m') {
3300 $type = $smm ?
"4m" : "4m-no-smm";
3301 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3303 # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3307 my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3308 die "EFI base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3309 die "EFI vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
3311 return ($ovmf_code, $ovmf_vars);
3315 aarch64
=> '/usr/bin/qemu-system-aarch64',
3316 x86_64
=> '/usr/bin/qemu-system-x86_64',
3318 sub get_command_for_arch
($) {
3320 return '/usr/bin/kvm' if is_native_arch
($arch);
3322 my $cmd = $Arch2Qemu->{$arch}
3323 or die "don't know how to emulate architecture '$arch'\n";
3327 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3328 # to use in a QEMU command line (-cpu element), first array_intersect the result
3329 # of query_supported_ with query_understood_. This is necessary because:
3331 # a) query_understood_ returns flags the host cannot use and
3332 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3333 # flags, but CPU settings - with most of them being flags. Those settings
3334 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3336 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3337 # expensive. If you need the value returned from this, you can get it much
3338 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3339 # $accel being 'kvm' or 'tcg'.
3341 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3342 # changes, automatically populating pmxcfs.
3344 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3345 # since kvm and tcg machines support different flags
3347 sub query_supported_cpu_flags
{
3350 $arch //= get_host_arch
();
3351 my $default_machine = $default_machines->{$arch};
3355 # FIXME: Once this is merged, the code below should work for ARM as well:
3356 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3357 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3360 my $kvm_supported = defined(kvm_version
());
3361 my $qemu_cmd = get_command_for_arch
($arch);
3363 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3365 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3366 my $query_supported_run_qemu = sub {
3372 '-machine', $default_machine,
3374 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3375 '-mon', 'chardev=qmp,mode=control',
3376 '-pidfile', $pidfile,
3381 push @$cmd, '-accel', 'tcg';
3384 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3385 die "QEMU flag querying VM exited with code " . $rc if $rc;
3388 my $cmd_result = mon_cmd
(
3390 'query-cpu-model-expansion',
3392 model
=> { name
=> 'host' }
3395 my $props = $cmd_result->{model
}->{props
};
3396 foreach my $prop (keys %$props) {
3397 next if $props->{$prop} ne '1';
3398 # QEMU returns some flags multiple times, with '_', '.' or '-'
3399 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3400 # We only keep those with underscores, to match /proc/cpuinfo
3401 $prop =~ s/\.|-/_/g;
3402 $flags->{$prop} = 1;
3407 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3408 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3412 return [ sort keys %$flags ];
3415 # We need to query QEMU twice, since KVM and TCG have different supported flags
3416 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3417 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3418 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3420 if ($kvm_supported) {
3421 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3422 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3429 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3430 my $understood_cpu_flag_dir = "/usr/share/kvm";
3431 sub query_understood_cpu_flags
{
3432 my $arch = get_host_arch
();
3433 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3435 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3438 my $raw = file_get_contents
($filepath);
3439 $raw =~ s/^\s+|\s+$//g;
3440 my @flags = split(/\s+/, $raw);
3445 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3446 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3447 my sub should_disable_smm
{
3448 my ($conf, $vga, $machine) = @_;
3450 return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3452 return (!defined($conf->{bios
}) || $conf->{bios
} eq 'seabios') &&
3453 $vga->{type
} && $vga->{type
} =~ m/^(serial\d+|none)$/;
3456 my sub print_ovmf_drive_commandlines
{
3457 my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3459 my $d = $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
3461 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch, $d, $q35);
3463 my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3465 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3466 my ($path, $format) = $d->@{'file', 'format'};
3468 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3469 if (!defined($format)) {
3470 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3471 $format = qemu_img_format
($scfg, $volname);
3473 } elsif (!defined($format)) {
3474 die "efidisk format must be specified\n";
3476 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3477 if ($path =~ m/^rbd:/) {
3478 $var_drive_str .= ',cache=writeback';
3479 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3481 $var_drive_str .= ",format=$format,file=$path";
3483 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3484 $var_drive_str .= ',readonly=on' if drive_is_read_only
($conf, $d);
3486 log_warn
("no efidisk configured! Using temporary efivars disk.");
3487 my $path = "/tmp/$vmid-ovmf.fd";
3488 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3489 $var_drive_str .= ",format=raw,file=$path";
3490 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $version_guard->(4, 1, 2);
3493 return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3496 sub config_to_command
{
3497 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3498 $live_restore_backing) = @_;
3500 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3503 my $ostype = $conf->{ostype
};
3504 my $winversion = windows_version
($ostype);
3505 my $kvm = $conf->{kvm
};
3506 my $nodename = nodename
();
3508 my $machine_conf = PVE
::QemuServer
::Machine
::parse_machine
($conf->{machine
});
3510 my $arch = get_vm_arch
($conf);
3511 my $kvm_binary = get_command_for_arch
($arch);
3512 my $kvmver = kvm_user_version
($kvm_binary);
3514 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3515 $kvmver //= "undefined";
3516 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3519 my $add_pve_version = min_version
($kvmver, 4, 1);
3521 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3522 my $machine_version = extract_version
($machine_type, $kvmver);
3523 $kvm //= 1 if is_native_arch
($arch);
3525 $machine_version =~ m/(\d+)\.(\d+)/;
3526 my ($machine_major, $machine_minor) = ($1, $2);
3528 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3529 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3530 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3531 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3532 ." please upgrade node '$nodename'\n"
3533 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3534 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3535 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3536 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3537 ." node '$nodename'\n";
3540 # if a specific +pve version is required for a feature, use $version_guard
3541 # instead of min_version to allow machines to be run with the minimum
3543 my $required_pve_version = 0;
3544 my $version_guard = sub {
3545 my ($major, $minor, $pve) = @_;
3546 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3547 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3548 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3549 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3553 if ($kvm && !defined kvm_version
()) {
3554 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3555 ." or enable in BIOS.\n";
3558 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3559 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3560 my $use_old_bios_files = undef;
3561 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3564 if ($conf->{affinity
}) {
3565 push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity
};
3568 push @$cmd, $kvm_binary;
3570 push @$cmd, '-id', $vmid;
3572 my $vmname = $conf->{name
} || "vm$vmid";
3574 push @$cmd, '-name', "$vmname,debug-threads=on";
3576 push @$cmd, '-no-shutdown';
3580 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3581 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3582 push @$cmd, '-mon', "chardev=qmp,mode=control";
3584 if (min_version
($machine_version, 2, 12)) {
3585 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3586 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3589 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3591 push @$cmd, '-daemonize';
3593 if ($conf->{smbios1
}) {
3594 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3595 if ($smbios_conf->{base64
}) {
3596 # Do not pass base64 flag to qemu
3597 delete $smbios_conf->{base64
};
3598 my $smbios_string = "";
3599 foreach my $key (keys %$smbios_conf) {
3601 if ($key eq "uuid") {
3602 $value = $smbios_conf->{uuid
}
3604 $value = decode_base64
($smbios_conf->{$key});
3606 # qemu accepts any binary data, only commas need escaping by double comma
3608 $smbios_string .= "," . $key . "=" . $value if $value;
3610 push @$cmd, '-smbios', "type=1" . $smbios_string;
3612 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3616 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3617 die "OVMF (UEFI) BIOS is not supported on 32-bit CPU types\n"
3618 if !$forcecpu && get_cpu_bitness
($conf->{cpu
}, $arch) == 32;
3620 my ($code_drive_str, $var_drive_str) =
3621 print_ovmf_drive_commandlines
($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3622 push $cmd->@*, '-drive', $code_drive_str;
3623 push $cmd->@*, '-drive', $var_drive_str;
3626 if ($q35) { # tell QEMU to load q35 config early
3627 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3628 if (min_version
($machine_version, 4, 0)) {
3629 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3631 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3635 if (defined(my $fixups = qemu_created_version_fixups
($conf, $forcemachine, $kvmver))) {
3636 push @$cmd, $fixups->@*;
3639 if ($conf->{vmgenid
}) {
3640 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3643 # add usb controllers
3644 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3645 $conf, $bridges, $arch, $machine_type, $machine_version);
3646 push @$devices, @usbcontrollers if @usbcontrollers;
3647 my $vga = parse_vga
($conf->{vga
});
3649 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3650 $vga->{type
} = 'qxl' if $qxlnum;
3652 if (!$vga->{type
}) {
3653 if ($arch eq 'aarch64') {
3654 $vga->{type
} = 'virtio';
3655 } elsif (min_version
($machine_version, 2, 9)) {
3656 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3658 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3662 # enable absolute mouse coordinates (needed by vnc)
3663 my $tablet = $conf->{tablet
};
3664 if (!defined($tablet)) {
3665 $tablet = $defaults->{tablet
};
3666 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3667 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3671 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3672 my $kbd = print_keyboarddevice_full
($conf, $arch);
3673 push @$devices, '-device', $kbd if defined($kbd);
3676 my $bootorder = device_bootorder
($conf);
3678 # host pci device passthrough
3679 my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3680 $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3683 my $usb_dev_features = {};
3684 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3686 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3687 $conf, $usb_dev_features, $bootorder, $machine_version);
3688 push @$devices, @usbdevices if @usbdevices;
3691 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3692 my $path = $conf->{"serial$i"} or next;
3693 if ($path eq 'socket') {
3694 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3695 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3696 # On aarch64, serial0 is the UART device. QEMU only allows
3697 # connecting UART devices via the '-serial' command line, as
3698 # the device has a fixed slot on the hardware...
3699 if ($arch eq 'aarch64' && $i == 0) {
3700 push @$devices, '-serial', "chardev:serial$i";
3702 push @$devices, '-device', "isa-serial,chardev=serial$i";
3705 die "no such serial device\n" if ! -c
$path;
3706 push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3707 push @$devices, '-device', "isa-serial,chardev=serial$i";
3712 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3713 if (my $path = $conf->{"parallel$i"}) {
3714 die "no such parallel device\n" if ! -c
$path;
3715 my $devtype = $path =~ m!^/dev/usb/lp! ?
'serial' : 'parallel';
3716 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3717 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3721 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3722 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3723 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3724 push @$devices, @$audio_devs;
3727 # Add a TPM only if the VM is not a template,
3728 # to support backing up template VMs even if the TPM disk is write-protected.
3729 add_tpm_device
($vmid, $devices, $conf) if (!PVE
::QemuConfig-
>is_template($conf));
3732 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3733 $sockets = $conf->{sockets
} if $conf->{sockets
};
3735 my $cores = $conf->{cores
} || 1;
3737 my $maxcpus = $sockets * $cores;
3739 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3741 my $allowed_vcpus = $cpuinfo->{cpus
};
3743 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3745 if ($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3746 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3747 for (my $i = 2; $i <= $vcpus; $i++) {
3748 my $cpustr = print_cpu_device
($conf, $arch, $i);
3749 push @$cmd, '-device', $cpustr;
3754 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3756 push @$cmd, '-nodefaults';
3758 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3760 push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3762 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3764 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3765 push @$devices, '-device', print_vga_device
(
3766 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3768 push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type
} eq 'virtio-gl'; # VIRGL
3770 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3771 push @$cmd, '-vnc', "unix:$socket,password=on";
3773 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3774 push @$cmd, '-nographic';
3778 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3779 my $useLocaltime = $conf->{localtime};
3781 if ($winversion >= 5) { # windows
3782 $useLocaltime = 1 if !defined($conf->{localtime});
3784 # use time drift fix when acpi is enabled
3785 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3786 $tdf = 1 if !defined($conf->{tdf
});
3790 if ($winversion >= 6) {
3791 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3792 push @$machineFlags, 'hpet=off';
3795 push @$rtcFlags, 'driftfix=slew' if $tdf;
3797 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3798 push @$rtcFlags, "base=$conf->{startdate}";
3799 } elsif ($useLocaltime) {
3800 push @$rtcFlags, 'base=localtime';
3804 push @$cmd, '-cpu', $forcecpu;
3806 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3809 PVE
::QemuServer
::Memory
::config
(
3810 $conf, $vmid, $sockets, $cores, $hotplug_features->{memory
}, $cmd);
3812 push @$cmd, '-S' if $conf->{freeze
};
3814 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3816 my $guest_agent = parse_guest_agent
($conf);
3818 if ($guest_agent->{enabled
}) {
3819 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3820 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3822 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3823 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3824 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3825 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3826 } elsif ($guest_agent->{type
} eq 'isa') {
3827 push @$devices, '-device', "isa-serial,chardev=qga0";
3831 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3832 if ($rng && $version_guard->(4, 1, 2)) {
3833 check_rng_source
($rng->{source
});
3835 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3836 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3837 my $limiter_str = "";
3839 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3842 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3843 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3844 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3849 assert_clipboard_config
($vga);
3850 my $is_spice = $qxlnum || $vga->{type
} =~ /^virtio/;
3852 if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3855 for (my $i = 1; $i < $qxlnum; $i++){
3856 push @$devices, '-device', print_vga_device
(
3857 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3860 # assume other OS works like Linux
3861 my ($ram, $vram) = ("134217728", "67108864");
3862 if ($vga->{memory
}) {
3863 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3864 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3866 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3867 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3871 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3873 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3874 if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3875 push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3877 push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3879 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3882 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3883 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3884 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3886 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3887 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3889 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3890 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3891 if ($spice_enhancement->{foldersharing
}) {
3892 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3893 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3896 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3897 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3898 if $spice_enhancement->{videostreaming
};
3899 push @$devices, '-spice', "$spice_opts";
3903 # enable balloon by default, unless explicitly disabled
3904 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3905 my $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3906 my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3907 $ballooncmd .= ",free-page-reporting=on" if min_version
($machine_version, 6, 2);
3908 push @$devices, '-device', $ballooncmd;
3911 if ($conf->{watchdog
}) {
3912 my $wdopts = parse_watchdog
($conf->{watchdog
});
3913 my $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3914 my $watchdog = $wdopts->{model
} || 'i6300esb';
3915 push @$devices, '-device', "$watchdog$pciaddr";
3916 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3920 my $scsicontroller = {};
3921 my $ahcicontroller = {};
3922 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3924 # Add iscsi initiator name if available
3925 if (my $initiator = get_initiator_name
()) {
3926 push @$devices, '-iscsi', "initiator-name=$initiator";
3929 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3930 my ($ds, $drive) = @_;
3932 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3933 check_volume_storage_type
($storecfg, $drive->{file
});
3934 push @$vollist, $drive->{file
};
3937 # ignore efidisk here, already added in bios/fw handling code above
3938 return if $drive->{interface
} eq 'efidisk';
3940 return if $drive->{interface
} eq 'tpmstate';
3942 $use_virtio = 1 if $ds =~ m/^virtio/;
3944 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3946 if ($drive->{interface
} eq 'virtio'){
3947 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3950 if ($drive->{interface
} eq 'scsi') {
3952 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3954 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3955 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3957 my $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3958 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3961 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3962 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3963 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3964 } elsif ($drive->{iothread
}) {
3966 "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3971 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3972 $queues = ",num_queues=$drive->{queues}";
3975 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3976 if !$scsicontroller->{$controller};
3977 $scsicontroller->{$controller}=1;
3980 if ($drive->{interface
} eq 'sata') {
3981 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3982 my $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3983 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3984 if !$ahcicontroller->{$controller};
3985 $ahcicontroller->{$controller}=1;
3988 my $live_restore = $live_restore_backing->{$ds};
3989 my $live_blockdev_name = undef;
3990 if ($live_restore) {
3991 $live_blockdev_name = $live_restore->{name
};
3992 push @$devices, '-blockdev', $live_restore->{blockdev
};
3995 my $drive_cmd = print_drive_commandline_full
(
3996 $storecfg, $vmid, $drive, $live_blockdev_name, min_version
($kvmver, 6, 0));
3998 # extra protection for templates, but SATA and IDE don't support it..
3999 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
4001 push @$devices, '-drive',$drive_cmd;
4002 push @$devices, '-device', print_drivedevice_full
(
4003 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4006 for (my $i = 0; $i < $MAX_NETS; $i++) {
4007 my $netname = "net$i";
4009 next if !$conf->{$netname};
4010 my $d = parse_net
($conf->{$netname});
4012 # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4014 $use_virtio = 1 if $d->{model
} eq 'virtio';
4016 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
4018 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
4019 push @$devices, '-netdev', $netdevfull;
4021 my $netdevicefull = print_netdevice_full
(
4022 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4024 push @$devices, '-device', $netdevicefull;
4027 if ($conf->{ivshmem
}) {
4028 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4032 $bus = print_pcie_addr
("ivshmem");
4034 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4037 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4038 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4040 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4041 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4042 .",size=$ivshmem->{size}M";
4045 # pci.4 is nested in pci.1
4046 $bridges->{1} = 1 if $bridges->{4};
4048 if (!$q35) { # add pci bridges
4049 if (min_version
($machine_version, 2, 3)) {
4053 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4056 for my $k (sort {$b cmp $a} keys %$bridges) {
4057 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4060 if ($k == 2 && $legacy_igd) {
4063 my $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
4064 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4066 if ($q35) { # add after -readconfig pve-q35.cfg
4067 splice @$devices, 2, 0, '-device', $devstr;
4069 unshift @$devices, '-device', $devstr if $k > 0;
4074 push @$machineFlags, 'accel=tcg';
4077 push @$machineFlags, 'smm=off' if should_disable_smm
($conf, $vga, $machine_type);
4079 my $machine_type_min = $machine_type;
4080 if ($add_pve_version) {
4081 $machine_type_min =~ s/\+pve\d+$//;
4082 $machine_type_min .= "+pve$required_pve_version";
4084 push @$machineFlags, "type=${machine_type_min}";
4086 PVE
::QemuServer
::Machine
::assert_valid_machine_property
($conf, $machine_conf);
4088 if (my $viommu = $machine_conf->{viommu
}) {
4089 if ($viommu eq 'intel') {
4090 unshift @$devices, '-device', 'intel-iommu,intremap=on,caching-mode=on';
4091 push @$machineFlags, 'kernel-irqchip=split';
4092 } elsif ($viommu eq 'virtio') {
4093 push @$devices, '-device', 'virtio-iommu-pci';
4097 push @$cmd, @$devices;
4098 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4099 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4100 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4102 if (my $vmstate = $conf->{vmstate
}) {
4103 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4104 push @$vollist, $vmstate;
4105 push @$cmd, '-loadstate', $statepath;
4106 print "activating and using '$vmstate' as vmstate\n";
4109 if (PVE
::QemuConfig-
>is_template($conf)) {
4110 # needed to workaround base volumes being read-only
4111 push @$cmd, '-snapshot';
4115 if ($conf->{args
}) {
4116 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4120 return wantarray ?
($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4123 sub check_rng_source
{
4126 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4127 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4130 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4131 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
4132 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4133 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4134 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4135 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4143 my $res = mon_cmd
($vmid, 'query-spice');
4145 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4148 sub vm_devices_list
{
4151 my $res = mon_cmd
($vmid, 'query-pci');
4152 my $devices_to_check = [];
4154 foreach my $pcibus (@$res) {
4155 push @$devices_to_check, @{$pcibus->{devices
}},
4158 while (@$devices_to_check) {
4160 for my $d (@$devices_to_check) {
4161 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4162 next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices
};
4164 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4165 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4167 $devices_to_check = $to_check;
4170 my $resblock = mon_cmd
($vmid, 'query-block');
4171 foreach my $block (@$resblock) {
4172 if($block->{device
} =~ m/^drive-(\S+)/){
4177 my $resmice = mon_cmd
($vmid, 'query-mice');
4178 foreach my $mice (@$resmice) {
4179 if ($mice->{name
} eq 'QEMU HID Tablet') {
4180 $devices->{tablet
} = 1;
4185 # for usb devices there is no query-usb
4186 # but we can iterate over the entries in
4187 # qom-list path=/machine/peripheral
4188 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4189 foreach my $per (@$resperipheral) {
4190 if ($per->{name
} =~ m/^usb(?:redirdev)?\d+$/) {
4191 $devices->{$per->{name
}} = 1;
4199 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4201 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4203 my $devices_list = vm_devices_list
($vmid);
4204 return 1 if defined($devices_list->{$deviceid});
4206 # add PCI bridge if we need it for the device
4207 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4209 if ($deviceid eq 'tablet') {
4210 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4211 } elsif ($deviceid eq 'keyboard') {
4212 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4213 } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4215 qemu_spice_usbredir_chardev_add
($vmid, "usbredirchardev$id");
4216 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_spice_usbdevice
($id, "xhci", $id + 1));
4217 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4218 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device, {}, $1 + 1));
4219 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4220 qemu_iothread_add
($vmid, $deviceid, $device);
4222 qemu_driveadd
($storecfg, $vmid, $device);
4223 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4225 qemu_deviceadd
($vmid, $devicefull);
4226 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4228 eval { qemu_drivedel
($vmid, $deviceid); };
4232 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4233 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4234 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4235 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4237 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4239 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4240 qemu_iothread_add
($vmid, $deviceid, $device);
4241 $devicefull .= ",iothread=iothread-$deviceid";
4244 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4245 $devicefull .= ",num_queues=$device->{queues}";
4248 qemu_deviceadd
($vmid, $devicefull);
4249 qemu_deviceaddverify
($vmid, $deviceid);
4250 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4251 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4252 qemu_driveadd
($storecfg, $vmid, $device);
4254 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4255 eval { qemu_deviceadd
($vmid, $devicefull); };
4257 eval { qemu_drivedel
($vmid, $deviceid); };
4261 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4262 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4264 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4265 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type);
4266 my $use_old_bios_files = undef;
4267 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4269 my $netdevicefull = print_netdevice_full
(
4270 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4271 qemu_deviceadd
($vmid, $netdevicefull);
4273 qemu_deviceaddverify
($vmid, $deviceid);
4274 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4277 eval { qemu_netdevdel
($vmid, $deviceid); };
4281 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4283 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4284 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4286 qemu_deviceadd
($vmid, $devicefull);
4287 qemu_deviceaddverify
($vmid, $deviceid);
4289 die "can't hotplug device '$deviceid'\n";
4295 # fixme: this should raise exceptions on error!
4296 sub vm_deviceunplug
{
4297 my ($vmid, $conf, $deviceid) = @_;
4299 my $devices_list = vm_devices_list
($vmid);
4300 return 1 if !defined($devices_list->{$deviceid});
4302 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4303 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4305 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4306 qemu_devicedel
($vmid, $deviceid);
4307 } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4308 qemu_devicedel
($vmid, $deviceid);
4309 qemu_devicedelverify
($vmid, $deviceid);
4310 } elsif ($deviceid =~ m/^usb\d+$/) {
4311 qemu_devicedel
($vmid, $deviceid);
4312 qemu_devicedelverify
($vmid, $deviceid);
4313 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4314 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4316 qemu_devicedel
($vmid, $deviceid);
4317 qemu_devicedelverify
($vmid, $deviceid);
4318 qemu_drivedel
($vmid, $deviceid);
4319 qemu_iothread_del
($vmid, $deviceid, $device);
4320 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4321 qemu_devicedel
($vmid, $deviceid);
4322 qemu_devicedelverify
($vmid, $deviceid);
4323 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4324 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4326 qemu_devicedel
($vmid, $deviceid);
4327 qemu_devicedelverify
($vmid, $deviceid);
4328 qemu_drivedel
($vmid, $deviceid);
4329 qemu_deletescsihw
($conf, $vmid, $deviceid);
4331 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4332 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4333 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4334 qemu_devicedel
($vmid, $deviceid);
4335 qemu_devicedelverify
($vmid, $deviceid);
4336 qemu_netdevdel
($vmid, $deviceid);
4338 die "can't unplug device '$deviceid'\n";
4344 sub qemu_spice_usbredir_chardev_add
{
4345 my ($vmid, $id) = @_;
4347 mon_cmd
($vmid, "chardev-add" , (
4358 sub qemu_iothread_add
{
4359 my ($vmid, $deviceid, $device) = @_;
4361 if ($device->{iothread
}) {
4362 my $iothreads = vm_iothreads_list
($vmid);
4363 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4367 sub qemu_iothread_del
{
4368 my ($vmid, $deviceid, $device) = @_;
4370 if ($device->{iothread
}) {
4371 my $iothreads = vm_iothreads_list
($vmid);
4372 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4377 my ($storecfg, $vmid, $device) = @_;
4379 my $kvmver = get_running_qemu_version
($vmid);
4380 my $io_uring = min_version
($kvmver, 6, 0);
4381 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4382 $drive =~ s/\\/\\\\/g;
4383 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4385 # If the command succeeds qemu prints: "OK
"
4386 return 1 if $ret =~ m/OK/s;
4388 die "adding drive failed
: $ret\n";
4392 my ($vmid, $deviceid) = @_;
4394 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4397 return 1 if $ret eq "";
4399 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4400 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4402 die "deleting drive
$deviceid failed
: $ret\n";
4405 sub qemu_deviceaddverify {
4406 my ($vmid, $deviceid) = @_;
4408 for (my $i = 0; $i <= 5; $i++) {
4409 my $devices_list = vm_devices_list($vmid);
4410 return 1 if defined($devices_list->{$deviceid});
4414 die "error on hotplug device
'$deviceid'\n";
4418 sub qemu_devicedelverify {
4419 my ($vmid, $deviceid) = @_;
4421 # need to verify that the device is correctly removed as device_del
4422 # is async and empty return is not reliable
4424 for (my $i = 0; $i <= 5; $i++) {
4425 my $devices_list = vm_devices_list($vmid);
4426 return 1 if !defined($devices_list->{$deviceid});
4430 die "error on hot-unplugging device
'$deviceid'\n";
4433 sub qemu_findorcreatescsihw {
4434 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4436 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4438 my $scsihwid="$controller_prefix$controller";
4439 my $devices_list = vm_devices_list($vmid);
4441 if (!defined($devices_list->{$scsihwid})) {
4442 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4448 sub qemu_deletescsihw {
4449 my ($conf, $vmid, $opt) = @_;
4451 my $device = parse_drive($opt, $conf->{$opt});
4453 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4454 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4458 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4460 my $devices_list = vm_devices_list($vmid);
4461 foreach my $opt (keys %{$devices_list}) {
4462 if (is_valid_drivename($opt)) {
4463 my $drive = parse_drive($opt, $conf->{$opt});
4464 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4470 my $scsihwid="scsihw
$controller";
4472 vm_deviceunplug($vmid, $conf, $scsihwid);
4477 sub qemu_add_pci_bridge {
4478 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4484 print_pci_addr($device, $bridges, $arch, $machine_type);
4486 while (my ($k, $v) = each %$bridges) {
4489 return 1 if !defined($bridgeid) || $bridgeid < 1;
4491 my $bridge = "pci
.$bridgeid";
4492 my $devices_list = vm_devices_list($vmid);
4494 if (!defined($devices_list->{$bridge})) {
4495 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4501 sub qemu_set_link_status {
4502 my ($vmid, $device, $up) = @_;
4504 mon_cmd($vmid, "set_link
", name => $device,
4505 up => $up ? JSON::true : JSON::false);
4508 sub qemu_netdevadd {
4509 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4511 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4512 my %options = split(/[=,]/, $netdev);
4514 if (defined(my $vhost = $options{vhost})) {
4515 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4518 if (defined(my $queues = $options{queues})) {
4519 $options{queues} = $queues + 0;
4522 mon_cmd($vmid, "netdev_add
", %options);
4526 sub qemu_netdevdel {
4527 my ($vmid, $deviceid) = @_;
4529 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4532 sub qemu_usb_hotplug {
4533 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4537 # remove the old one first
4538 vm_deviceunplug($vmid, $conf, $deviceid);
4540 # check if xhci controller is necessary and available
4541 my $devicelist = vm_devices_list($vmid);
4543 if (!$devicelist->{xhci}) {
4544 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4545 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4549 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4552 sub qemu_cpu_hotplug {
4553 my ($vmid, $conf, $vcpus) = @_;
4555 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4558 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4559 $sockets = $conf->{sockets} if $conf->{sockets};
4560 my $cores = $conf->{cores} || 1;
4561 my $maxcpus = $sockets * $cores;
4563 $vcpus = $maxcpus if !$vcpus;
4565 die "you can
't add more vcpus than maxcpus\n"
4566 if $vcpus > $maxcpus;
4568 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4570 if ($vcpus < $currentvcpus) {
4572 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4574 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4575 qemu_devicedel($vmid, "cpu$i");
4577 my $currentrunningvcpus = undef;
4579 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4580 last if scalar(@{$currentrunningvcpus}) == $i-1;
4581 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4585 #update conf after each succesfull cpu unplug
4586 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4587 PVE::QemuConfig->write_config($vmid, $conf);
4590 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4596 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4597 die "vcpus in running vm does not match its configuration\n"
4598 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4600 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4601 my $arch = get_vm_arch($conf);
4603 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4604 my $cpustr = print_cpu_device($conf, $arch, $i);
4605 qemu_deviceadd($vmid, $cpustr);
4608 my $currentrunningvcpus = undef;
4610 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4611 last if scalar(@{$currentrunningvcpus}) == $i;
4612 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4616 #update conf after each succesfull cpu hotplug
4617 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4618 PVE::QemuConfig->write_config($vmid, $conf);
4622 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4623 mon_cmd($vmid, "cpu-add", id => int($i));
4628 sub qemu_block_set_io_throttle {
4629 my ($vmid, $deviceid,
4630 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4631 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4632 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4633 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4635 return if !check_running($vmid) ;
4637 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4639 bps_rd => int($bps_rd),
4640 bps_wr => int($bps_wr),
4642 iops_rd => int($iops_rd),
4643 iops_wr => int($iops_wr),
4644 bps_max => int($bps_max),
4645 bps_rd_max => int($bps_rd_max),
4646 bps_wr_max => int($bps_wr_max),
4647 iops_max => int($iops_max),
4648 iops_rd_max => int($iops_rd_max),
4649 iops_wr_max => int($iops_wr_max),
4650 bps_max_length => int($bps_max_length),
4651 bps_rd_max_length => int($bps_rd_max_length),
4652 bps_wr_max_length => int($bps_wr_max_length),
4653 iops_max_length => int($iops_max_length),
4654 iops_rd_max_length => int($iops_rd_max_length),
4655 iops_wr_max_length => int($iops_wr_max_length),
4660 sub qemu_block_resize {
4661 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4663 my $running = check_running($vmid);
4665 PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4667 return if !$running;
4669 my $padding = (1024 - $size % 1024) % 1024;
4670 $size = $size + $padding;
4675 device => $deviceid,
4681 sub qemu_volume_snapshot {
4682 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4684 my $running = check_running($vmid);
4686 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4687 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4689 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4693 sub qemu_volume_snapshot_delete {
4694 my ($vmid, $storecfg, $volid, $snap) = @_;
4696 my $running = check_running($vmid);
4697 my $attached_deviceid;
4700 my $conf = PVE::QemuConfig->load_config($vmid);
4701 PVE::QemuConfig->foreach_volume($conf, sub {
4702 my ($ds, $drive) = @_;
4703 $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
4707 if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
4710 'blockdev-snapshot-delete-internal-sync
',
4711 device => $attached_deviceid,
4715 PVE::Storage::volume_snapshot_delete(
4716 $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
4720 sub set_migration_caps {
4721 my ($vmid, $savevm) = @_;
4723 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4725 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4726 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4731 "auto-converge" => 1,
4733 "x-rdma-pin-all" => 0,
4736 "dirty-bitmaps" => $dirty_bitmaps,
4739 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4741 for my $supported_capability (@$supported_capabilities) {
4743 capability => $supported_capability->{capability},
4744 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4748 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4752 my ($conf, $func, @param) = @_;
4756 my $test_volid = sub {
4757 my ($key, $drive, $snapname, $pending) = @_;
4759 my $volid = $drive->{file};
4762 $volhash->{$volid}->{cdrom} //= 1;
4763 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4765 my $replicate = $drive->{replicate} // 1;
4766 $volhash->{$volid}->{replicate} //= 0;
4767 $volhash->{$volid}->{replicate} = 1 if $replicate;
4769 $volhash->{$volid}->{shared} //= 0;
4770 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4772 $volhash->{$volid}->{is_unused} //= 0;
4773 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4775 $volhash->{$volid}->{is_attached} //= 0;
4776 $volhash->{$volid}->{is_attached} = 1
4777 if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4779 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4780 if defined($snapname);
4782 $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4784 my $size = $drive->{size};
4785 $volhash->{$volid}->{size} //= $size if $size;
4787 $volhash->{$volid}->{is_vmstate} //= 0;
4788 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4790 $volhash->{$volid}->{is_tpmstate} //= 0;
4791 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0
';
4793 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4796 my $include_opts = {
4797 extra_keys => ['vmstate
'],
4798 include_unused => 1,
4801 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4803 PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4804 if defined($conf->{pending}) && $conf->{pending}->%*;
4806 foreach my $snapname (keys %{$conf->{snapshots}}) {
4807 my $snap = $conf->{snapshots}->{$snapname};
4808 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4811 foreach my $volid (keys %$volhash) {
4812 &$func($volid, $volhash->{$volid}, @param);
4816 my $fast_plug_option = {
4820 'migrate_downtime
' => 1,
4821 'migrate_speed
' => 1,
4828 'vmstatestorage
' => 1,
4831 for my $opt (keys %$confdesc_cloudinit) {
4832 $fast_plug_option->{$opt} = 1;
4835 # hotplug changes in [PENDING]
4836 # $selection hash can be used to only apply specified options, for
4837 # example: { cores => 1 } (only apply changed 'cores
')
4838 # $errors ref is used to return error messages
4839 sub vmconfig_hotplug_pending {
4840 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4842 my $defaults = load_defaults();
4843 my $arch = get_vm_arch($conf);
4844 my $machine_type = get_vm_machine($conf, undef, $arch);
4846 # commit values which do not have any impact on running VM first
4847 # Note: those option cannot raise errors, we we do not care about
4848 # $selection and always apply them.
4850 my $add_error = sub {
4851 my ($opt, $msg) = @_;
4852 $errors->{$opt} = "hotplug problem - $msg";
4855 my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4857 my $cloudinit_record_changed = sub {
4858 my ($conf, $opt, $old, $new) = @_;
4859 return if !$cloudinit_pending_properties->{$opt};
4861 my $ci = ($conf->{cloudinit} //= {});
4863 my $recorded = $ci->{$opt};
4864 my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4866 if (defined($new)) {
4867 if (defined($old)) {
4868 # an existing value is being modified
4869 if (defined($recorded)) {
4870 # the value was already not in sync
4871 if ($new eq $recorded) {
4872 # a value is being reverted to the cloud-init state:
4874 delete $added{$opt};
4876 # the value was changed multiple times, do nothing
4878 } elsif ($added{$opt}) {
4879 # the value had been marked as added and is being changed, do nothing
4881 # the value is new, record it:
4885 # a new value is being added
4886 if (defined($recorded)) {
4887 # it was already not in sync
4888 if ($new eq $recorded) {
4889 # a value is being reverted to the cloud-init state:
4891 delete $added{$opt};
4893 # the value had temporarily been removed, do nothing
4895 } elsif ($added{$opt}) {
4896 # the value had been marked as added already, do nothing
4898 # the value is new, add it
4902 } elsif (!defined($old)) {
4903 # a non-existent value is being removed? ignore...
4905 # a value is being deleted
4906 if (defined($recorded)) {
4907 # a value was already recorded, just keep it
4908 } elsif ($added{$opt}) {
4909 # the value was marked as added, remove it
4910 delete $added{$opt};
4912 # a previously unrecorded value is being removed, record the old value:
4917 my $added = join(',', sort keys %added);
4918 $ci->{added} = $added if length($added);
4922 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4923 if ($fast_plug_option->{$opt}) {
4924 my $new = delete $conf->{pending}->{$opt};
4925 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4926 $conf->{$opt} = $new;
4932 PVE::QemuConfig->write_config($vmid, $conf);
4935 my $ostype = $conf->{ostype};
4936 my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4937 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4938 my $usb_hotplug = $hotplug_features->{usb}
4939 && min_version($version, 7, 1)
4940 && defined($ostype) && ($ostype eq 'l26
' || windows_version($ostype) > 7);
4942 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4943 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4945 foreach my $opt (sort keys %$pending_delete_hash) {
4946 next if $selection && !$selection->{$opt};
4947 my $force = $pending_delete_hash->{$opt}->{force};
4949 if ($opt eq 'hotplug
') {
4950 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4951 } elsif ($opt eq 'tablet
') {
4952 die "skip\n" if !$hotplug_features->{usb};
4953 if ($defaults->{tablet}) {
4954 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4955 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4956 if $arch eq 'aarch64
';
4958 vm_deviceunplug($vmid, $conf, 'tablet
');
4959 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4961 } elsif ($opt =~ m/^usb(\d+)$/) {
4963 die "skip\n" if !$usb_hotplug;
4964 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4965 vm_deviceunplug
($vmid, $conf, $opt);
4966 } elsif ($opt eq 'vcpus') {
4967 die "skip\n" if !$hotplug_features->{cpu
};
4968 qemu_cpu_hotplug
($vmid, $conf, undef);
4969 } elsif ($opt eq 'balloon') {
4970 # enable balloon device is not hotpluggable
4971 die "skip\n" if defined($conf->{balloon
}) && $conf->{balloon
} == 0;
4972 # here we reset the ballooning value to memory
4973 my $balloon = get_current_memory
($conf->{memory
});
4974 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
4975 } elsif ($fast_plug_option->{$opt}) {
4977 } elsif ($opt =~ m/^net(\d+)$/) {
4978 die "skip\n" if !$hotplug_features->{network
};
4979 vm_deviceunplug
($vmid, $conf, $opt);
4981 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
4982 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
});
4984 } elsif (is_valid_drivename
($opt)) {
4985 die "skip\n" if !$hotplug_features->{disk
} || $opt =~ m/(ide|sata)(\d+)/;
4986 vm_deviceunplug
($vmid, $conf, $opt);
4987 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
4988 } elsif ($opt =~ m/^memory$/) {
4989 die "skip\n" if !$hotplug_features->{memory
};
4990 PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf);
4991 } elsif ($opt eq 'cpuunits') {
4992 $cgroup->change_cpu_shares(undef);
4993 } elsif ($opt eq 'cpulimit') {
4994 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5000 &$add_error($opt, $err) if $err ne "skip\n";
5002 my $old = delete $conf->{$opt};
5003 $cloudinit_record_changed->($conf, $opt, $old, undef);
5004 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5009 foreach my $opt (keys %{$conf->{pending
}}) {
5010 next if $selection && !$selection->{$opt};
5011 my $value = $conf->{pending
}->{$opt};
5013 if ($opt eq 'hotplug') {
5014 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug
} =~ /memory/);
5015 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug
} =~ /cpu/);
5016 } elsif ($opt eq 'tablet') {
5017 die "skip\n" if !$hotplug_features->{usb
};
5019 vm_deviceplug
($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5020 vm_deviceplug
($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5021 if $arch eq 'aarch64';
5022 } elsif ($value == 0) {
5023 vm_deviceunplug
($vmid, $conf, 'tablet');
5024 vm_deviceunplug
($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5026 } elsif ($opt =~ m/^usb(\d+)$/) {
5028 die "skip\n" if !$usb_hotplug;
5029 my $d = eval { parse_property_string
('pve-qm-usb', $value) };
5031 if ($d->{host
} =~ m/^spice$/i) {
5032 $id = "usbredirdev$index";
5034 qemu_usb_hotplug
($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5035 } elsif ($opt eq 'vcpus') {
5036 die "skip\n" if !$hotplug_features->{cpu
};
5037 qemu_cpu_hotplug
($vmid, $conf, $value);
5038 } elsif ($opt eq 'balloon') {
5039 # enable/disable balloning device is not hotpluggable
5040 my $old_balloon_enabled = !!(!defined($conf->{balloon
}) || $conf->{balloon
});
5041 my $new_balloon_enabled = !!(!defined($conf->{pending
}->{balloon
}) || $conf->{pending
}->{balloon
});
5042 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5044 # allow manual ballooning if shares is set to zero
5045 if ((defined($conf->{shares
}) && ($conf->{shares
} == 0))) {
5046 my $memory = get_current_memory
($conf->{memory
});
5047 my $balloon = $conf->{pending
}->{balloon
} || $memory;
5048 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5050 } elsif ($opt =~ m/^net(\d+)$/) {
5051 # some changes can be done without hotplug
5052 vmconfig_update_net
($storecfg, $conf, $hotplug_features->{network
},
5053 $vmid, $opt, $value, $arch, $machine_type);
5054 } elsif (is_valid_drivename
($opt)) {
5055 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5056 # some changes can be done without hotplug
5057 my $drive = parse_drive
($opt, $value);
5058 if (drive_is_cloudinit
($drive)) {
5059 $cloudinit_opt = [$opt, $drive];
5060 # apply all the other changes first, then generate the cloudinit disk
5063 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5064 $vmid, $opt, $value, $arch, $machine_type);
5065 } elsif ($opt =~ m/^memory$/) { #dimms
5066 die "skip\n" if !$hotplug_features->{memory
};
5067 $value = PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $value);
5068 } elsif ($opt eq 'cpuunits') {
5069 my $new_cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{pending
}->{$opt}); #clamp
5070 $cgroup->change_cpu_shares($new_cpuunits);
5071 } elsif ($opt eq 'cpulimit') {
5072 my $cpulimit = $conf->{pending
}->{$opt} == 0 ?
-1 : int($conf->{pending
}->{$opt} * 100000);
5073 $cgroup->change_cpu_quota($cpulimit, 100000);
5074 } elsif ($opt eq 'agent') {
5075 vmconfig_update_agent
($conf, $opt, $value);
5077 die "skip\n"; # skip non-hot-pluggable options
5081 &$add_error($opt, $err) if $err ne "skip\n";
5083 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5084 $conf->{$opt} = $value;
5085 delete $conf->{pending
}->{$opt};
5089 if (defined($cloudinit_opt)) {
5090 my ($opt, $drive) = @$cloudinit_opt;
5091 my $value = $conf->{pending
}->{$opt};
5093 my $temp = {%$conf, $opt => $value};
5094 PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($temp, $vmid);
5095 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5096 $vmid, $opt, $value, $arch, $machine_type);
5099 &$add_error($opt, $err) if $err ne "skip\n";
5101 $conf->{$opt} = $value;
5102 delete $conf->{pending
}->{$opt};
5106 # unplug xhci controller if no usb device is left
5109 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
5110 next if !defined($conf->{"usb$i"});
5115 vm_deviceunplug
($vmid, $conf, 'xhci');
5119 PVE
::QemuConfig-
>write_config($vmid, $conf);
5121 if ($hotplug_features->{cloudinit
} && PVE
::QemuServer
::Cloudinit
::has_changes
($conf)) {
5122 PVE
::QemuServer
::vmconfig_update_cloudinit_drive
($storecfg, $conf, $vmid);
5126 sub try_deallocate_drive
{
5127 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5129 if (($force || $key =~ /^unused/) && !drive_is_cdrom
($drive, 1)) {
5130 my $volid = $drive->{file
};
5131 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
5132 my $sid = PVE
::Storage
::parse_volume_id
($volid);
5133 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5135 # check if the disk is really unused
5136 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5137 if PVE
::QemuServer
::Drive
::is_volume_in_use
($storecfg, $conf, $key, $volid);
5138 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5141 # If vm is not owner of this disk remove from config
5149 sub vmconfig_delete_or_detach_drive
{
5150 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5152 my $drive = parse_drive
($opt, $conf->{$opt});
5154 my $rpcenv = PVE
::RPCEnvironment
::get
();
5155 my $authuser = $rpcenv->get_user();
5158 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5159 try_deallocate_drive
($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5161 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $drive);
5167 sub vmconfig_apply_pending
{
5168 my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5170 return if !scalar(keys %{$conf->{pending
}});
5172 my $add_apply_error = sub {
5173 my ($opt, $msg) = @_;
5174 my $err_msg = "unable to apply pending change $opt : $msg";
5175 $errors->{$opt} = $err_msg;
5181 my $pending_delete_hash = PVE
::QemuConfig-
>parse_pending_delete($conf->{pending
}->{delete});
5182 foreach my $opt (sort keys %$pending_delete_hash) {
5183 my $force = $pending_delete_hash->{$opt}->{force
};
5185 if ($opt =~ m/^unused/) {
5186 die "internal error";
5187 } elsif (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5188 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5189 } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5191 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
5192 eval { PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
}) };
5198 $add_apply_error->($opt, $err);
5200 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5201 delete $conf->{$opt};
5205 PVE
::QemuConfig-
>cleanup_pending($conf);
5207 my $generate_cloudinit = $skip_cloud_init ?
0 : undef;
5209 foreach my $opt (keys %{$conf->{pending
}}) { # add/change
5210 next if $opt eq 'delete'; # just to be sure
5212 if (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5213 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, parse_drive
($opt, $conf->{$opt}))
5214 } elsif (defined($conf->{pending
}->{$opt}) && $opt =~ m/^net\d+$/) {
5215 return if !$have_sdn; # return from eval if SDN is not available
5217 my $new_net = PVE
::QemuServer
::parse_net
($conf->{pending
}->{$opt});
5218 if ($conf->{$opt}) {
5219 my $old_net = PVE
::QemuServer
::parse_net
($conf->{$opt});
5221 if (defined($old_net->{bridge
}) && defined($old_net->{macaddr
}) && (
5222 safe_string_ne
($old_net->{bridge
}, $new_net->{bridge
}) ||
5223 safe_string_ne
($old_net->{macaddr
}, $new_net->{macaddr
})
5225 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($old_net->{bridge
}, $old_net->{macaddr
}, $conf->{name
});
5228 #fixme: reuse ip if mac change && same bridge
5229 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($new_net->{bridge
}, $conf->{name
}, $new_net->{macaddr
}, $vmid, undef, 1);
5233 $add_apply_error->($opt, $err);
5236 if (is_valid_drivename
($opt)) {
5237 my $drive = parse_drive
($opt, $conf->{pending
}->{$opt});
5238 $generate_cloudinit //= 1 if drive_is_cloudinit
($drive);
5241 $conf->{$opt} = delete $conf->{pending
}->{$opt};
5245 # write all changes at once to avoid unnecessary i/o
5246 PVE
::QemuConfig-
>write_config($vmid, $conf);
5247 if ($generate_cloudinit) {
5248 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5249 # After successful generation and if there were changes to be applied, update the
5250 # config to drop the {cloudinit} entry.
5251 PVE
::QemuConfig-
>write_config($vmid, $conf);
5256 sub vmconfig_update_net
{
5257 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5259 my $newnet = parse_net
($value);
5261 if ($conf->{$opt}) {
5262 my $oldnet = parse_net
($conf->{$opt});
5264 if (safe_string_ne
($oldnet->{model
}, $newnet->{model
}) ||
5265 safe_string_ne
($oldnet->{macaddr
}, $newnet->{macaddr
}) ||
5266 safe_num_ne
($oldnet->{queues
}, $newnet->{queues
}) ||
5267 safe_num_ne
($oldnet->{mtu
}, $newnet->{mtu
}) ||
5268 !($newnet->{bridge
} && $oldnet->{bridge
})
5269 ) { # bridge/nat mode change
5271 # for non online change, we try to hot-unplug
5272 die "skip\n" if !$hotplug;
5273 vm_deviceunplug
($vmid, $conf, $opt);
5276 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($oldnet->{bridge
}, $oldnet->{macaddr
}, $conf->{name
});
5281 die "internal error" if $opt !~ m/net(\d+)/;
5282 my $iface = "tap${vmid}i$1";
5284 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5285 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
}) ||
5286 safe_string_ne
($oldnet->{trunks
}, $newnet->{trunks
}) ||
5287 safe_num_ne
($oldnet->{firewall
}, $newnet->{firewall
})
5289 PVE
::Network
::tap_unplug
($iface);
5291 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5292 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5293 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
})
5295 qemu_set_link_status
($vmid, $opt, 0);
5298 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
})) {
5300 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($oldnet->{bridge
}, $oldnet->{macaddr
}, $conf->{name
});
5301 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($newnet->{bridge
}, $conf->{name
}, $newnet->{macaddr
}, $vmid, undef, 1);
5306 PVE
::Network
::SDN
::Zones
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5308 PVE
::Network
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5311 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5312 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5313 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
})
5315 qemu_set_link_status
($vmid, $opt, 1);
5318 } elsif (safe_num_ne
($oldnet->{rate
}, $newnet->{rate
})) {
5319 # Rate can be applied on its own but any change above needs to
5320 # include the rate in tap_plug since OVS resets everything.
5321 PVE
::Network
::tap_rate_limit
($iface, $newnet->{rate
});
5324 if (safe_string_ne
($oldnet->{link_down
}, $newnet->{link_down
})) {
5325 qemu_set_link_status
($vmid, $opt, !$newnet->{link_down
});
5334 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($newnet->{bridge
}, $conf->{name
}, $newnet->{macaddr
}, $vmid, undef, 1);
5335 PVE
::Network
::SDN
::Vnets
::add_dhcp_mapping
($newnet->{bridge
}, $newnet->{macaddr
}, $vmid, $conf->{name
});
5337 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5343 sub vmconfig_update_agent
{
5344 my ($conf, $opt, $value) = @_;
5346 die "skip\n" if !$conf->{$opt};
5348 my $hotplug_options = { fstrim_cloned_disks
=> 1 };
5350 my $old_agent = parse_guest_agent
($conf);
5351 my $agent = parse_guest_agent
({$opt => $value});
5353 for my $option (keys %$agent) { # added/changed options
5354 next if defined($hotplug_options->{$option});
5355 die "skip\n" if safe_string_ne
($agent->{$option}, $old_agent->{$option});
5358 for my $option (keys %$old_agent) { # removed options
5359 next if defined($hotplug_options->{$option});
5360 die "skip\n" if safe_string_ne
($old_agent->{$option}, $agent->{$option});
5363 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5366 sub vmconfig_update_disk
{
5367 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5369 my $drive = parse_drive
($opt, $value);
5371 if ($conf->{$opt} && (my $old_drive = parse_drive
($opt, $conf->{$opt}))) {
5372 my $media = $drive->{media
} || 'disk';
5373 my $oldmedia = $old_drive->{media
} || 'disk';
5374 die "unable to change media type\n" if $media ne $oldmedia;
5376 if (!drive_is_cdrom
($old_drive)) {
5378 if ($drive->{file
} ne $old_drive->{file
}) {
5380 die "skip\n" if !$hotplug;
5382 # unplug and register as unused
5383 vm_deviceunplug
($vmid, $conf, $opt);
5384 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive)
5387 # update existing disk
5389 # skip non hotpluggable value
5390 if (safe_string_ne
($drive->{aio
}, $old_drive->{aio
}) ||
5391 safe_string_ne
($drive->{discard
}, $old_drive->{discard
}) ||
5392 safe_string_ne
($drive->{iothread
}, $old_drive->{iothread
}) ||
5393 safe_string_ne
($drive->{queues
}, $old_drive->{queues
}) ||
5394 safe_string_ne
($drive->{product
}, $old_drive->{product
}) ||
5395 safe_string_ne
($drive->{cache
}, $old_drive->{cache
}) ||
5396 safe_string_ne
($drive->{ssd
}, $old_drive->{ssd
}) ||
5397 safe_string_ne
($drive->{vendor
}, $old_drive->{vendor
}) ||
5398 safe_string_ne
($drive->{ro
}, $old_drive->{ro
})) {
5403 if (safe_num_ne
($drive->{mbps
}, $old_drive->{mbps
}) ||
5404 safe_num_ne
($drive->{mbps_rd
}, $old_drive->{mbps_rd
}) ||
5405 safe_num_ne
($drive->{mbps_wr
}, $old_drive->{mbps_wr
}) ||
5406 safe_num_ne
($drive->{iops
}, $old_drive->{iops
}) ||
5407 safe_num_ne
($drive->{iops_rd
}, $old_drive->{iops_rd
}) ||
5408 safe_num_ne
($drive->{iops_wr
}, $old_drive->{iops_wr
}) ||
5409 safe_num_ne
($drive->{mbps_max
}, $old_drive->{mbps_max
}) ||
5410 safe_num_ne
($drive->{mbps_rd_max
}, $old_drive->{mbps_rd_max
}) ||
5411 safe_num_ne
($drive->{mbps_wr_max
}, $old_drive->{mbps_wr_max
}) ||
5412 safe_num_ne
($drive->{iops_max
}, $old_drive->{iops_max
}) ||
5413 safe_num_ne
($drive->{iops_rd_max
}, $old_drive->{iops_rd_max
}) ||
5414 safe_num_ne
($drive->{iops_wr_max
}, $old_drive->{iops_wr_max
}) ||
5415 safe_num_ne
($drive->{bps_max_length
}, $old_drive->{bps_max_length
}) ||
5416 safe_num_ne
($drive->{bps_rd_max_length
}, $old_drive->{bps_rd_max_length
}) ||
5417 safe_num_ne
($drive->{bps_wr_max_length
}, $old_drive->{bps_wr_max_length
}) ||
5418 safe_num_ne
($drive->{iops_max_length
}, $old_drive->{iops_max_length
}) ||
5419 safe_num_ne
($drive->{iops_rd_max_length
}, $old_drive->{iops_rd_max_length
}) ||
5420 safe_num_ne
($drive->{iops_wr_max_length
}, $old_drive->{iops_wr_max_length
})) {
5422 qemu_block_set_io_throttle
(
5424 ($drive->{mbps
} || 0)*1024*1024,
5425 ($drive->{mbps_rd
} || 0)*1024*1024,
5426 ($drive->{mbps_wr
} || 0)*1024*1024,
5427 $drive->{iops
} || 0,
5428 $drive->{iops_rd
} || 0,
5429 $drive->{iops_wr
} || 0,
5430 ($drive->{mbps_max
} || 0)*1024*1024,
5431 ($drive->{mbps_rd_max
} || 0)*1024*1024,
5432 ($drive->{mbps_wr_max
} || 0)*1024*1024,
5433 $drive->{iops_max
} || 0,
5434 $drive->{iops_rd_max
} || 0,
5435 $drive->{iops_wr_max
} || 0,
5436 $drive->{bps_max_length
} || 1,
5437 $drive->{bps_rd_max_length
} || 1,
5438 $drive->{bps_wr_max_length
} || 1,
5439 $drive->{iops_max_length
} || 1,
5440 $drive->{iops_rd_max_length
} || 1,
5441 $drive->{iops_wr_max_length
} || 1,
5451 if ($drive->{file
} eq 'none') {
5452 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5453 if (drive_is_cloudinit
($old_drive)) {
5454 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive);
5457 my $path = get_iso_path
($storecfg, $vmid, $drive->{file
});
5459 # force eject if locked
5460 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5463 mon_cmd
($vmid, "blockdev-change-medium",
5464 id
=> "$opt", filename
=> "$path");
5472 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5474 PVE
::Storage
::activate_volumes
($storecfg, [$drive->{file
}]) if $drive->{file
} !~ m
|^/dev/.+|;
5475 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5478 sub vmconfig_update_cloudinit_drive
{
5479 my ($storecfg, $conf, $vmid) = @_;
5481 my $cloudinit_ds = undef;
5482 my $cloudinit_drive = undef;
5484 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5485 my ($ds, $drive) = @_;
5486 if (PVE
::QemuServer
::drive_is_cloudinit
($drive)) {
5487 $cloudinit_ds = $ds;
5488 $cloudinit_drive = $drive;
5492 return if !$cloudinit_drive;
5494 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5495 PVE
::QemuConfig-
>write_config($vmid, $conf);
5498 my $running = PVE
::QemuServer
::check_running
($vmid);
5501 my $path = PVE
::Storage
::path
($storecfg, $cloudinit_drive->{file
});
5503 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$cloudinit_ds");
5504 mon_cmd
($vmid, "blockdev-change-medium", id
=> "$cloudinit_ds", filename
=> "$path");
5509 # called in locked context by incoming migration
5510 sub vm_migrate_get_nbd_disks
{
5511 my ($storecfg, $conf, $replicated_volumes) = @_;
5513 my $local_volumes = {};
5514 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5515 my ($ds, $drive) = @_;
5517 return if drive_is_cdrom
($drive);
5518 return if $ds eq 'tpmstate0';
5520 my $volid = $drive->{file
};
5524 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid);
5526 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5527 return if $scfg->{shared
};
5529 my $format = qemu_img_format
($scfg, $volname);
5531 # replicated disks re-use existing state via bitmap
5532 my $use_existing = $replicated_volumes->{$volid} ?
1 : 0;
5533 $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5535 return $local_volumes;
5538 # called in locked context by incoming migration
5539 sub vm_migrate_alloc_nbd_disks
{
5540 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5543 foreach my $opt (sort keys %$source_volumes) {
5544 my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5546 if ($use_existing) {
5547 $nbd->{$opt}->{drivestr
} = print_drive
($drive);
5548 $nbd->{$opt}->{volid
} = $volid;
5549 $nbd->{$opt}->{replicated
} = 1;
5553 $storeid = PVE
::JSONSchema
::map_id
($storagemap, $storeid);
5555 # order of precedence, filtered by whether storage supports it:
5556 # 1. explicit requested format
5557 # 2. default format of storage
5558 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5559 $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5561 my $size = $drive->{size
} / 1024;
5562 my $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $format, undef, $size);
5563 my $newdrive = $drive;
5564 $newdrive->{format
} = $format;
5565 $newdrive->{file
} = $newvolid;
5566 my $drivestr = print_drive
($newdrive);
5567 $nbd->{$opt}->{drivestr
} = $drivestr;
5568 $nbd->{$opt}->{volid
} = $newvolid;
5574 # see vm_start_nolock for parameters, additionally:
5576 # storagemap = parsed storage map for allocating NBD disks
5578 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5580 return PVE
::QemuConfig-
>lock_config($vmid, sub {
5581 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migrate_opts->{migratedfrom
});
5583 die "you can't start a vm if it's a template\n"
5584 if !$params->{skiptemplate
} && PVE
::QemuConfig-
>is_template($conf);
5586 my $has_suspended_lock = PVE
::QemuConfig-
>has_lock($conf, 'suspended');
5587 my $has_backup_lock = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5589 my $running = check_running
($vmid, undef, $migrate_opts->{migratedfrom
});
5591 if ($has_backup_lock && $running) {
5592 # a backup is currently running, attempt to start the guest in the
5593 # existing QEMU instance
5594 return vm_resume
($vmid);
5597 PVE
::QemuConfig-
>check_lock($conf)
5598 if !($params->{skiplock
} || $has_suspended_lock);
5600 $params->{resume
} = $has_suspended_lock || defined($conf->{vmstate
});
5602 die "VM $vmid already running\n" if $running;
5604 if (my $storagemap = $migrate_opts->{storagemap
}) {
5605 my $replicated = $migrate_opts->{replicated_volumes
};
5606 my $disks = vm_migrate_get_nbd_disks
($storecfg, $conf, $replicated);
5607 $migrate_opts->{nbd
} = vm_migrate_alloc_nbd_disks
($storecfg, $vmid, $disks, $storagemap);
5609 foreach my $opt (keys %{$migrate_opts->{nbd
}}) {
5610 $conf->{$opt} = $migrate_opts->{nbd
}->{$opt}->{drivestr
};
5614 return vm_start_nolock
($storecfg, $vmid, $conf, $params, $migrate_opts);
5620 # statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5621 # skiplock => 0/1, skip checking for config lock
5622 # skiptemplate => 0/1, skip checking whether VM is template
5623 # forcemachine => to force QEMU machine (rollback/migration)
5624 # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5625 # timeout => in seconds
5626 # paused => start VM in paused state (backup)
5627 # resume => resume from hibernation
5628 # live-restore-backing => {
5630 # name => blockdev-name,
5631 # blockdev => "arg to the -blockdev command instantiating device named 'name'",
5636 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5637 # migratedfrom => source node
5638 # spice_ticket => used for spice migration, passed via tunnel/stdin
5639 # network => CIDR of migration network
5640 # type => secure/insecure - tunnel over encrypted connection or plain-text
5641 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5642 # replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5643 # offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5644 # contained in config
5645 sub vm_start_nolock
{
5646 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5648 my $statefile = $params->{statefile
};
5649 my $resume = $params->{resume
};
5651 my $migratedfrom = $migrate_opts->{migratedfrom
};
5652 my $migration_type = $migrate_opts->{type
};
5656 # clean up leftover reboot request files
5657 eval { clear_reboot_request
($vmid); };
5660 if (!$statefile && scalar(keys %{$conf->{pending
}})) {
5661 vmconfig_apply_pending
($vmid, $conf, $storecfg);
5662 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5665 # don't regenerate the ISO if the VM is started as part of a live migration
5666 # this way we can reuse the old ISO with the correct config
5667 if (!$migratedfrom) {
5668 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5669 # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5670 # $conf->{cloudinit}, so we could just not do this?
5671 # But we do it above, so for now let's be consistent.
5672 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5676 # override offline migrated volumes, conf is out of date still
5677 if (my $offline_volumes = $migrate_opts->{offline_volumes
}) {
5678 for my $key (sort keys $offline_volumes->%*) {
5679 my $parsed = parse_drive
($key, $conf->{$key});
5680 $parsed->{file
} = $offline_volumes->{$key};
5681 $conf->{$key} = print_drive
($parsed);
5685 my $defaults = load_defaults
();
5687 # set environment variable useful inside network script
5688 # for remote migration the config is available on the target node!
5689 if (!$migrate_opts->{remote_node
}) {
5690 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom;
5693 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5695 my $forcemachine = $params->{forcemachine
};
5696 my $forcecpu = $params->{forcecpu
};
5698 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5699 $forcemachine = $conf->{runningmachine
};
5700 $forcecpu = $conf->{runningcpu
};
5701 print "Resuming suspended VM\n";
5704 my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command
($storecfg, $vmid,
5705 $conf, $defaults, $forcemachine, $forcecpu, $params->{'live-restore-backing'});
5708 my $get_migration_ip = sub {
5709 my ($nodename) = @_;
5711 return $migration_ip if defined($migration_ip);
5713 my $cidr = $migrate_opts->{network
};
5715 if (!defined($cidr)) {
5716 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5717 $cidr = $dc_conf->{migration
}->{network
};
5720 if (defined($cidr)) {
5721 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5723 die "could not get IP: no address configured on local " .
5724 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5726 die "could not get IP: multiple addresses configured on local " .
5727 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5729 $migration_ip = @$ips[0];
5732 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5733 if !defined($migration_ip);
5735 return $migration_ip;
5739 if ($statefile eq 'tcp') {
5740 my $migrate = $res->{migrate
} = { proto
=> 'tcp' };
5741 $migrate->{addr
} = "localhost";
5742 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5743 my $nodename = nodename
();
5745 if (!defined($migration_type)) {
5746 if (defined($datacenterconf->{migration
}->{type
})) {
5747 $migration_type = $datacenterconf->{migration
}->{type
};
5749 $migration_type = 'secure';
5753 if ($migration_type eq 'insecure') {
5754 $migrate->{addr
} = $get_migration_ip->($nodename);
5755 $migrate->{addr
} = "[$migrate->{addr}]" if Net
::IP
::ip_is_ipv6
($migrate->{addr
});
5758 # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5760 push @$cmd, '-incoming', 'defer';
5763 } elsif ($statefile eq 'unix') {
5764 # should be default for secure migrations as a ssh TCP forward
5765 # tunnel is not deterministic reliable ready and fails regurarly
5766 # to set up in time, so use UNIX socket forwards
5767 my $migrate = $res->{migrate
} = { proto
=> 'unix' };
5768 $migrate->{addr
} = "/run/qemu-server/$vmid.migrate";
5769 unlink $migrate->{addr
};
5771 $migrate->{uri
} = "unix:$migrate->{addr}";
5772 push @$cmd, '-incoming', $migrate->{uri
};
5775 } elsif (-e
$statefile) {
5776 push @$cmd, '-loadstate', $statefile;
5778 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5779 push @$vollist, $statefile;
5780 push @$cmd, '-loadstate', $statepath;
5782 } elsif ($params->{paused
}) {
5786 my $memory = get_current_memory
($conf->{memory
});
5787 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $memory, $resume);
5789 my $pci_reserve_list = [];
5790 for my $device (values $pci_devices->%*) {
5791 next if $device->{mdev
}; # we don't reserve for mdev devices
5792 push $pci_reserve_list->@*, map { $_->{id
} } $device->{ids
}->@*;
5795 # reserve all PCI IDs before actually doing anything with them
5796 PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, $start_timeout);
5800 for my $id (sort keys %$pci_devices) {
5801 my $d = $pci_devices->{$id};
5802 my ($index) = ($id =~ m/^hostpci(\d+)$/);
5805 for my $dev ($d->{ids
}->@*) {
5806 my $info = eval { PVE
::QemuServer
::PCI
::prepare_pci_device
($vmid, $dev->{id
}, $index, $d->{mdev
}) };
5809 $chosen_mdev = $info;
5810 last if $chosen_mdev; # if successful, we're done
5816 next if !$d->{mdev
};
5817 die "could not create mediated device\n" if !defined($chosen_mdev);
5819 # nvidia grid needs the uuid of the mdev as qemu parameter
5820 if (!defined($uuid) && $chosen_mdev->{vendor
} =~ m/^(0x)?10de$/) {
5821 if (defined($conf->{smbios1
})) {
5822 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
5823 $uuid = $smbios_conf->{uuid
} if defined($smbios_conf->{uuid
});
5825 $uuid = PVE
::QemuServer
::PCI
::generate_mdev_uuid
($vmid, $index) if !defined($uuid);
5828 push @$cmd, '-uuid', $uuid if defined($uuid);
5831 eval { cleanup_pci_devices
($vmid, $conf) };
5836 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5839 my %silence_std_outs = (outfunc
=> sub {}, errfunc
=> sub {});
5840 eval { run_command
(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5841 eval { run_command
(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5842 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5843 # timeout should be more than enough here...
5844 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 20);
5846 my $cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{cpuunits
});
5849 timeout
=> $statefile ?
undef : $start_timeout,
5854 # when migrating, prefix QEMU output so other side can pick up any
5855 # errors that might occur and show the user
5856 if ($migratedfrom) {
5857 $run_params{quiet
} = 1;
5858 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5861 my %systemd_properties = (
5862 Slice
=> 'qemu.slice',
5863 KillMode
=> 'process',
5865 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5868 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5869 $systemd_properties{CPUWeight
} = $cpuunits;
5871 $systemd_properties{CPUShares
} = $cpuunits;
5874 if (my $cpulimit = $conf->{cpulimit
}) {
5875 $systemd_properties{CPUQuota
} = int($cpulimit * 100);
5877 $systemd_properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5879 my $run_qemu = sub {
5880 PVE
::Tools
::run_fork
sub {
5881 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5884 if ((my $tpm = $conf->{tpmstate0
}) && !PVE
::QemuConfig-
>is_template($conf)) {
5885 # start the TPM emulator so QEMU can connect on start
5886 $tpmpid = start_swtpm
($storecfg, $vmid, $tpm, $migratedfrom);
5889 my $exitcode = run_command
($cmd, %run_params);
5892 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5893 kill 'TERM', $tpmpid;
5895 die "QEMU exited with code $exitcode\n";
5900 if ($conf->{hugepages
}) {
5903 my $hotplug_features =
5904 parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
5905 my $hugepages_topology =
5906 PVE
::QemuServer
::Memory
::hugepages_topology
($conf, $hotplug_features->{memory
});
5908 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5910 PVE
::QemuServer
::Memory
::hugepages_mount
();
5911 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5913 eval { $run_qemu->() };
5915 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5916 if !$conf->{keephugepages
};
5920 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5921 if !$conf->{keephugepages
};
5923 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5926 eval { $run_qemu->() };
5930 # deactivate volumes if start fails
5931 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5933 eval { cleanup_pci_devices
($vmid, $conf) };
5936 die "start failed: $err";
5939 # re-reserve all PCI IDs now that we can know the actual VM PID
5940 my $pid = PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
5941 eval { PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, undef, $pid) };
5944 if (defined(my $migrate = $res->{migrate
})) {
5945 if ($migrate->{proto
} eq 'tcp') {
5946 my $nodename = nodename
();
5947 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5948 $migrate->{port
} = PVE
::Tools
::next_migrate_port
($pfamily);
5949 $migrate->{uri
} = "tcp:$migrate->{addr}:$migrate->{port}";
5950 mon_cmd
($vmid, "migrate-incoming", uri
=> $migrate->{uri
});
5952 print "migration listens on $migrate->{uri}\n";
5953 } elsif ($statefile) {
5954 eval { mon_cmd
($vmid, "cont"); };
5958 #start nbd server for storage migration
5959 if (my $nbd = $migrate_opts->{nbd
}) {
5960 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5962 my $migrate_storage_uri;
5963 # nbd_protocol_version > 0 for unix socket support
5964 if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5965 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5966 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
5967 $migrate_storage_uri = "nbd:unix:$socket_path";
5968 $res->{migrate
}->{unix_sockets
} = [$socket_path];
5970 my $nodename = nodename
();
5971 my $localip = $get_migration_ip->($nodename);
5972 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5973 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5975 mon_cmd
($vmid, "nbd-server-start", addr
=> {
5978 host
=> "${localip}",
5979 port
=> "${storage_migrate_port}",
5982 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5983 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5986 my $block_info = mon_cmd
($vmid, "query-block");
5987 $block_info = { map { $_->{device
} => $_ } $block_info->@* };
5989 foreach my $opt (sort keys %$nbd) {
5990 my $drivestr = $nbd->{$opt}->{drivestr
};
5991 my $volid = $nbd->{$opt}->{volid
};
5993 my $block_node = $block_info->{"drive-$opt"}->{inserted
}->{'node-name'};
5999 'node-name' => $block_node,
6000 writable
=> JSON
::true
,
6002 name
=> "drive-$opt", # NBD export name
6005 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6006 print "storage migration listens on $nbd_uri volume:$drivestr\n";
6007 print "re-using replicated volume: $opt - $volid\n"
6008 if $nbd->{$opt}->{replicated
};
6010 $res->{drives
}->{$opt} = $nbd->{$opt};
6011 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
6015 if ($migratedfrom) {
6017 set_migration_caps
($vmid);
6022 print "spice listens on port $spice_port\n";
6023 $res->{spice_port
} = $spice_port;
6024 if ($migrate_opts->{spice_ticket
}) {
6025 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
6026 $migrate_opts->{spice_ticket
});
6027 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
6032 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
6033 if !$statefile && $conf->{balloon
};
6035 foreach my $opt (keys %$conf) {
6036 next if $opt !~ m/^net\d+$/;
6037 my $nicconf = parse_net
($conf->{$opt});
6038 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
6040 add_nets_bridge_fdb
($conf, $vmid);
6043 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
6048 path
=> "machine/peripheral/balloon0",
6049 property
=> "guest-stats-polling-interval",
6053 log_warn
("could not set polling interval for ballooning - $@") if $@;
6057 print "Resumed VM, removing state\n";
6058 if (my $vmstate = $conf->{vmstate
}) {
6059 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6060 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6062 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6063 PVE
::QemuConfig-
>write_config($vmid, $conf);
6066 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
6068 my ($current_machine, $is_deprecated) =
6069 PVE
::QemuServer
::Machine
::get_current_qemu_machine
($vmid);
6070 if ($is_deprecated) {
6072 "current machine version '$current_machine' is deprecated - see the documentation and ".
6073 "change to a newer one",
6080 sub vm_commandline
{
6081 my ($storecfg, $vmid, $snapname) = @_;
6083 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6085 my ($forcemachine, $forcecpu);
6087 my $snapshot = $conf->{snapshots
}->{$snapname};
6088 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6090 # check for machine or CPU overrides in snapshot
6091 $forcemachine = $snapshot->{runningmachine
};
6092 $forcecpu = $snapshot->{runningcpu
};
6094 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
6099 my $defaults = load_defaults
();
6101 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6103 return PVE
::Tools
::cmd2string
($cmd);
6107 my ($vmid, $skiplock) = @_;
6109 PVE
::QemuConfig-
>lock_config($vmid, sub {
6111 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6113 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6115 mon_cmd
($vmid, "system_reset");
6119 sub get_vm_volumes
{
6123 foreach_volid
($conf, sub {
6124 my ($volid, $attr) = @_;
6126 return if $volid =~ m
|^/|;
6128 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6131 push @$vollist, $volid;
6137 sub cleanup_pci_devices
{
6138 my ($vmid, $conf) = @_;
6140 foreach my $key (keys %$conf) {
6141 next if $key !~ m/^hostpci(\d+)$/;
6142 my $hostpciindex = $1;
6143 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
6144 my $d = parse_hostpci
($conf->{$key});
6146 # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6147 # don't want to break ABI just for this two liner
6148 my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6150 # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6151 # out when we do it first. so wait for up to 10 seconds and then try it manually
6152 if ($d->{ids
}->[0]->[0]->{vendor
} =~ m/^(0x)?10de$/ && -e
$dev_sysfs_dir) {
6154 while (-e
$dev_sysfs_dir && $count < 10) {
6158 print "waited $count seconds for mediated device driver finishing clean up\n";
6161 if (-e
$dev_sysfs_dir) {
6162 print "actively clean up mediated device with UUID $uuid\n";
6163 PVE
::SysFSTools
::file_write
("$dev_sysfs_dir/remove", "1");
6167 PVE
::QemuServer
::PCI
::remove_pci_reservation
($vmid);
6170 sub vm_stop_cleanup
{
6171 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6176 my $vollist = get_vm_volumes
($conf);
6177 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6179 if (my $tpmdrive = $conf->{tpmstate0
}) {
6180 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
6181 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
6183 PVE
::Storage
::unmap_volume
($storecfg, $tpm->{file
});
6188 foreach my $ext (qw(mon qmp pid vnc qga)) {
6189 unlink "/var/run/qemu-server/${vmid}.$ext";
6192 if ($conf->{ivshmem
}) {
6193 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
6194 # just delete it for now, VMs which have this already open do not
6195 # are affected, but new VMs will get a separated one. If this
6196 # becomes an issue we either add some sort of ref-counting or just
6197 # add a "don't delete on stop" flag to the ivshmem format.
6198 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
6201 cleanup_pci_devices
($vmid, $conf);
6203 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
6205 warn $@ if $@; # avoid errors - just warn
6208 # call only in locked context
6210 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6212 my $pid = check_running
($vmid, $nocheck);
6217 $conf = PVE
::QemuConfig-
>load_config($vmid);
6218 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6219 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
6220 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
6221 $timeout = $opts->{down
} if $opts->{down
};
6223 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
6228 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
6229 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
6231 mon_cmd
($vmid, "system_powerdown");
6234 mon_cmd
($vmid, "quit");
6240 $timeout = 60 if !defined($timeout);
6243 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6248 if ($count >= $timeout) {
6250 warn "VM still running - terminating now with SIGTERM\n";
6253 die "VM quit/powerdown failed - got timeout\n";
6256 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6260 if (!check_running
($vmid, $nocheck)) {
6261 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6265 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6268 die "VM quit/powerdown failed\n";
6276 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6281 if ($count >= $timeout) {
6282 warn "VM still running - terminating now with SIGKILL\n";
6287 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6290 # Note: use $nocheck to skip tests if VM configuration file exists.
6291 # We need that when migration VMs to other nodes (files already moved)
6292 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6294 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6296 $force = 1 if !defined($force) && !$shutdown;
6299 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
6300 kill 15, $pid if $pid;
6301 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
6302 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
6306 PVE
::QemuConfig-
>lock_config($vmid, sub {
6307 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6312 my ($vmid, $timeout) = @_;
6314 PVE
::QemuConfig-
>lock_config($vmid, sub {
6317 # only reboot if running, as qmeventd starts it again on a stop event
6318 return if !check_running
($vmid);
6320 create_reboot_request
($vmid);
6322 my $storecfg = PVE
::Storage
::config
();
6323 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
6327 # avoid that the next normal shutdown will be confused for a reboot
6328 clear_reboot_request
($vmid);
6334 # note: if using the statestorage parameter, the caller has to check privileges
6336 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6343 PVE
::QemuConfig-
>lock_config($vmid, sub {
6345 $conf = PVE
::QemuConfig-
>load_config($vmid);
6347 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
6348 PVE
::QemuConfig-
>check_lock($conf)
6349 if !($skiplock || $is_backing_up);
6351 die "cannot suspend to disk during backup\n"
6352 if $is_backing_up && $includestate;
6354 if ($includestate) {
6355 $conf->{lock} = 'suspending';
6356 my $date = strftime
("%Y-%m-%d", localtime(time()));
6357 $storecfg = PVE
::Storage
::config
();
6358 if (!$statestorage) {
6359 $statestorage = find_vmstate_storage
($conf, $storecfg);
6360 # check permissions for the storage
6361 my $rpcenv = PVE
::RPCEnvironment
::get
();
6362 if ($rpcenv->{type
} ne 'cli') {
6363 my $authuser = $rpcenv->get_user();
6364 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6369 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
6370 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6371 $path = PVE
::Storage
::path
($storecfg, $vmstate);
6372 PVE
::QemuConfig-
>write_config($vmid, $conf);
6374 mon_cmd
($vmid, "stop");
6378 if ($includestate) {
6380 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
6383 set_migration_caps
($vmid, 1);
6384 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
6386 my $state = mon_cmd
($vmid, "query-savevm");
6387 if (!$state->{status
}) {
6388 die "savevm not active\n";
6389 } elsif ($state->{status
} eq 'active') {
6392 } elsif ($state->{status
} eq 'completed') {
6393 print "State saved, quitting\n";
6395 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
6396 die "query-savevm failed with error '$state->{error}'\n"
6398 die "query-savevm returned status '$state->{status}'\n";
6404 PVE
::QemuConfig-
>lock_config($vmid, sub {
6405 $conf = PVE
::QemuConfig-
>load_config($vmid);
6407 # cleanup, but leave suspending lock, to indicate something went wrong
6409 eval { mon_cmd
($vmid, "savevm-end"); };
6411 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6412 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6413 delete $conf->@{qw(vmstate runningmachine runningcpu)};
6414 PVE
::QemuConfig-
>write_config($vmid, $conf);
6420 die "lock changed unexpectedly\n"
6421 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
6423 mon_cmd
($vmid, "quit");
6424 $conf->{lock} = 'suspended';
6425 PVE
::QemuConfig-
>write_config($vmid, $conf);
6430 # $nocheck is set when called as part of a migration - in this context the
6431 # location of the config file (source or target node) is not deterministic,
6432 # since migration cannot wait for pmxcfs to process the rename
6434 my ($vmid, $skiplock, $nocheck) = @_;
6436 PVE
::QemuConfig-
>lock_config($vmid, sub {
6437 my $res = mon_cmd
($vmid, 'query-status');
6438 my $resume_cmd = 'cont';
6442 $conf = eval { PVE
::QemuConfig-
>load_config($vmid) }; # try on target node
6444 my $vmlist = PVE
::Cluster
::get_vmlist
();
6445 if (exists($vmlist->{ids
}->{$vmid})) {
6446 my $node = $vmlist->{ids
}->{$vmid}->{node
};
6447 $conf = eval { PVE
::QemuConfig-
>load_config($vmid, $node) }; # try on source node
6450 PVE
::Cluster
::cfs_update
(); # vmlist was wrong, invalidate cache
6451 $conf = PVE
::QemuConfig-
>load_config($vmid); # last try on target node again
6455 $conf = PVE
::QemuConfig-
>load_config($vmid);
6458 if ($res->{status
}) {
6459 return if $res->{status
} eq 'running'; # job done, go home
6460 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
6461 $reset = 1 if $res->{status
} eq 'shutdown';
6465 PVE
::QemuConfig-
>check_lock($conf)
6466 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
6470 # required if a VM shuts down during a backup and we get a resume
6471 # request before the backup finishes for example
6472 mon_cmd
($vmid, "system_reset");
6475 add_nets_bridge_fdb
($conf, $vmid) if $resume_cmd eq 'cont';
6477 mon_cmd
($vmid, $resume_cmd);
6482 my ($vmid, $skiplock, $key) = @_;
6484 PVE
::QemuConfig-
>lock_config($vmid, sub {
6486 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6488 # there is no qmp command, so we use the human monitor command
6489 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
6490 die $res if $res ne '';
6494 sub check_bridge_access
{
6495 my ($rpcenv, $authuser, $conf) = @_;
6497 return 1 if $authuser eq 'root@pam';
6499 for my $opt (sort keys $conf->%*) {
6500 next if $opt !~ m/^net\d+$/;
6501 my $net = parse_net
($conf->{$opt});
6502 my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6503 PVE
::GuestHelpers
::check_vnet_access
($rpcenv, $authuser, $bridge, $tag, $trunks);
6508 sub check_mapping_access
{
6509 my ($rpcenv, $user, $conf) = @_;
6511 for my $opt (keys $conf->%*) {
6512 if ($opt =~ m/^usb\d+$/) {
6513 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-usb', $conf->{$opt});
6514 if (my $host = $device->{host
}) {
6515 die "only root can set '$opt' config for real devices\n"
6516 if $host !~ m/^spice$/i && $user ne 'root@pam';
6517 } elsif ($device->{mapping
}) {
6518 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6520 die "either 'host' or 'mapping' must be set.\n";
6522 } elsif ($opt =~ m/^hostpci\d+$/) {
6523 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-hostpci', $conf->{$opt});
6524 if ($device->{host
}) {
6525 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6526 } elsif ($device->{mapping
}) {
6527 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6529 die "either 'host' or 'mapping' must be set.\n";
6535 sub check_restore_permissions
{
6536 my ($rpcenv, $user, $conf) = @_;
6538 check_bridge_access
($rpcenv, $user, $conf);
6539 check_mapping_access
($rpcenv, $user, $conf);
6541 # vzdump restore implementaion
6543 sub tar_archive_read_firstfile
{
6544 my $archive = shift;
6546 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
6548 # try to detect archive type first
6549 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6550 die "unable to open file '$archive'\n";
6551 my $firstfile = <$fh>;
6555 die "ERROR: archive contaions no data\n" if !$firstfile;
6561 sub tar_restore_cleanup
{
6562 my ($storecfg, $statfile) = @_;
6564 print STDERR
"starting cleanup\n";
6566 if (my $fd = IO
::File-
>new($statfile, "r")) {
6567 while (defined(my $line = <$fd>)) {
6568 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6571 if ($volid =~ m
|^/|) {
6572 unlink $volid || die 'unlink failed\n';
6574 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6576 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6578 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6580 print STDERR
"unable to parse line in statfile - $line";
6587 sub restore_file_archive
{
6588 my ($archive, $vmid, $user, $opts) = @_;
6590 return restore_vma_archive
($archive, $vmid, $user, $opts)
6593 my $info = PVE
::Storage
::archive_info
($archive);
6594 my $format = $opts->{format
} // $info->{format
};
6595 my $comp = $info->{compression
};
6597 # try to detect archive format
6598 if ($format eq 'tar') {
6599 return restore_tar_archive
($archive, $vmid, $user, $opts);
6601 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
6605 # hepler to remove disks that will not be used after restore
6606 my $restore_cleanup_oldconf = sub {
6607 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6609 my $kept_disks = {};
6611 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
6612 my ($ds, $drive) = @_;
6614 return if drive_is_cdrom
($drive, 1);
6616 my $volid = $drive->{file
};
6617 return if !$volid || $volid =~ m
|^/|;
6619 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
6620 return if !$path || !$owner || ($owner != $vmid);
6622 # Note: only delete disk we want to restore
6623 # other volumes will become unused
6624 if ($virtdev_hash->{$ds}) {
6625 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
6630 $kept_disks->{$volid} = 1;
6634 # after the restore we have no snapshots anymore
6635 for my $snapname (keys $oldconf->{snapshots
}->%*) {
6636 my $snap = $oldconf->{snapshots
}->{$snapname};
6637 if ($snap->{vmstate
}) {
6638 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
6644 for my $volid (keys $kept_disks->%*) {
6645 eval { PVE
::Storage
::volume_snapshot_delete
($storecfg, $volid, $snapname); };
6651 # Helper to parse vzdump backup device hints
6653 # $rpcenv: Environment, used to ckeck storage permissions
6654 # $user: User ID, to check storage permissions
6655 # $storecfg: Storage configuration
6656 # $fh: the file handle for reading the configuration
6657 # $devinfo: should contain device sizes for all backu-up'ed devices
6658 # $options: backup options (pool, default storage)
6660 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6661 my $parse_backup_hints = sub {
6662 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6664 my $check_storage = sub { # assert if an image can be allocate
6665 my ($storeid, $scfg) = @_;
6666 die "Content type 'images' is not available on storage '$storeid'\n"
6667 if !$scfg->{content
}->{images
};
6668 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6669 if $user ne 'root@pam';
6672 my $virtdev_hash = {};
6673 while (defined(my $line = <$fh>)) {
6674 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6675 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6676 die "archive does not contain data for drive '$virtdev'\n"
6677 if !$devinfo->{$devname};
6679 if (defined($options->{storage
})) {
6680 $storeid = $options->{storage
} || 'local';
6681 } elsif (!$storeid) {
6684 $format = 'raw' if !$format;
6685 $devinfo->{$devname}->{devname
} = $devname;
6686 $devinfo->{$devname}->{virtdev
} = $virtdev;
6687 $devinfo->{$devname}->{format
} = $format;
6688 $devinfo->{$devname}->{storeid
} = $storeid;
6690 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6691 $check_storage->($storeid, $scfg); # permission and content type check
6693 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6694 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6696 my $drive = parse_drive
($virtdev, $2);
6698 if (drive_is_cloudinit
($drive)) {
6699 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6700 $storeid = $options->{storage
} if defined ($options->{storage
});
6701 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6702 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6704 $check_storage->($storeid, $scfg); # permission and content type check
6706 $virtdev_hash->{$virtdev} = {
6708 storeid
=> $storeid,
6709 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6716 return $virtdev_hash;
6719 # Helper to allocate and activate all volumes required for a restore
6721 # $storecfg: Storage configuration
6722 # $virtdev_hash: as returned by parse_backup_hints()
6724 # Returns: { $virtdev => $volid }
6725 my $restore_allocate_devices = sub {
6726 my ($storecfg, $virtdev_hash, $vmid) = @_;
6729 foreach my $virtdev (sort keys %$virtdev_hash) {
6730 my $d = $virtdev_hash->{$virtdev};
6731 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6732 my $storeid = $d->{storeid
};
6733 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6735 # test if requested format is supported
6736 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6737 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6738 $d->{format
} = $defFormat if !$supported;
6741 if ($d->{is_cloudinit
}) {
6742 $name = "vm-$vmid-cloudinit";
6743 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6744 if ($scfg->{path
}) {
6745 $name .= ".$d->{format}";
6749 my $volid = PVE
::Storage
::vdisk_alloc
(
6750 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6752 print STDERR
"new volume ID is '$volid'\n";
6753 $d->{volid
} = $volid;
6755 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6757 $map->{$virtdev} = $volid;
6763 sub restore_update_config_line
{
6764 my ($cookie, $map, $line, $unique) = @_;
6766 return '' if $line =~ m/^\#qmdump\#/;
6767 return '' if $line =~ m/^\#vzdump\#/;
6768 return '' if $line =~ m/^lock:/;
6769 return '' if $line =~ m/^unused\d+:/;
6770 return '' if $line =~ m/^parent:/;
6774 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6775 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6776 # try to convert old 1.X settings
6777 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6778 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6779 my ($model, $macaddr) = split(/\=/, $devconfig);
6780 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6783 bridge
=> "vmbr$ind",
6784 macaddr
=> $macaddr,
6786 my $netstr = print_net
($net);
6788 $res .= "net$cookie->{netcount}: $netstr\n";
6789 $cookie->{netcount
}++;
6791 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6792 my ($id, $netstr) = ($1, $2);
6793 my $net = parse_net
($netstr);
6794 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6795 $netstr = print_net
($net);
6796 $res .= "$id: $netstr\n";
6797 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6800 my $di = parse_drive
($virtdev, $value);
6801 if (defined($di->{backup
}) && !$di->{backup
}) {
6803 } elsif ($map->{$virtdev}) {
6804 delete $di->{format
}; # format can change on restore
6805 $di->{file
} = $map->{$virtdev};
6806 $value = print_drive
($di);
6807 $res .= "$virtdev: $value\n";
6811 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6813 if ($vmgenid ne '0') {
6814 # always generate a new vmgenid if there was a valid one setup
6815 $vmgenid = generate_uuid
();
6817 $res .= "vmgenid: $vmgenid\n";
6818 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6819 my ($uuid, $uuid_str);
6820 UUID
::generate
($uuid);
6821 UUID
::unparse
($uuid, $uuid_str);
6822 my $smbios1 = parse_smbios1
($2);
6823 $smbios1->{uuid
} = $uuid_str;
6824 $res .= $1.print_smbios1
($smbios1)."\n";
6832 my $restore_deactivate_volumes = sub {
6833 my ($storecfg, $virtdev_hash) = @_;
6836 for my $dev (values $virtdev_hash->%*) {
6837 push $vollist->@*, $dev->{volid
} if $dev->{volid
};
6840 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
6841 print STDERR
$@ if $@;
6844 my $restore_destroy_volumes = sub {
6845 my ($storecfg, $virtdev_hash) = @_;
6847 for my $dev (values $virtdev_hash->%*) {
6848 my $volid = $dev->{volid
} or next;
6850 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6851 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6853 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6857 sub restore_merge_config
{
6858 my ($filename, $backup_conf_raw, $override_conf) = @_;
6860 my $backup_conf = parse_vm_config
($filename, $backup_conf_raw);
6861 for my $key (keys $override_conf->%*) {
6862 $backup_conf->{$key} = $override_conf->{$key};
6865 return $backup_conf;
6869 my ($cfg, $vmid) = @_;
6871 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6873 my $volid_hash = {};
6874 foreach my $storeid (keys %$info) {
6875 foreach my $item (@{$info->{$storeid}}) {
6876 next if !($item->{volid
} && $item->{size
});
6877 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6878 $volid_hash->{$item->{volid
}} = $item;
6885 sub update_disk_config
{
6886 my ($vmid, $conf, $volid_hash) = @_;
6889 my $prefix = "VM $vmid";
6891 # used and unused disks
6892 my $referenced = {};
6894 # Note: it is allowed to define multiple storages with same path (alias), so
6895 # we need to check both 'volid' and real 'path' (two different volid can point
6896 # to the same path).
6898 my $referencedpath = {};
6901 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6902 my ($opt, $drive) = @_;
6904 my $volid = $drive->{file
};
6906 my $volume = $volid_hash->{$volid};
6908 # mark volid as "in-use" for next step
6909 $referenced->{$volid} = 1;
6910 if ($volume && (my $path = $volume->{path
})) {
6911 $referencedpath->{$path} = 1;
6914 return if drive_is_cdrom
($drive);
6917 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6918 if (defined($updated)) {
6920 $conf->{$opt} = print_drive
($updated);
6921 print "$prefix ($opt): $msg\n";
6925 # remove 'unusedX' entry if volume is used
6926 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6927 my ($opt, $drive) = @_;
6929 my $volid = $drive->{file
};
6933 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6934 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6935 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6937 delete $conf->{$opt};
6940 $referenced->{$volid} = 1;
6941 $referencedpath->{$path} = 1 if $path;
6944 foreach my $volid (sort keys %$volid_hash) {
6945 next if $volid =~ m/vm-$vmid-state-/;
6946 next if $referenced->{$volid};
6947 my $path = $volid_hash->{$volid}->{path
};
6948 next if !$path; # just to be sure
6949 next if $referencedpath->{$path};
6951 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6952 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6953 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6960 my ($vmid, $nolock, $dryrun) = @_;
6962 my $cfg = PVE
::Storage
::config
();
6964 print "rescan volumes...\n";
6965 my $volid_hash = scan_volids
($cfg, $vmid);
6967 my $updatefn = sub {
6970 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6972 PVE
::QemuConfig-
>check_lock($conf);
6975 foreach my $volid (keys %$volid_hash) {
6976 my $info = $volid_hash->{$volid};
6977 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6980 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6982 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6985 if (defined($vmid)) {
6989 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6992 my $vmlist = config_list
();
6993 foreach my $vmid (keys %$vmlist) {
6997 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
7003 sub restore_proxmox_backup_archive
{
7004 my ($archive, $vmid, $user, $options) = @_;
7006 my $storecfg = PVE
::Storage
::config
();
7008 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
7009 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7011 my $fingerprint = $scfg->{fingerprint
};
7012 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
7014 my $repo = PVE
::PBSClient
::get_repository
($scfg);
7015 my $namespace = $scfg->{namespace
};
7017 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7018 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
7019 local $ENV{PBS_PASSWORD
} = $password;
7020 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
7022 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7023 PVE
::Storage
::parse_volname
($storecfg, $archive);
7025 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7027 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7029 my $tmpdir = "/var/tmp/vzdumptmp$$";
7033 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7034 # disable interrupts (always do cleanups)
7038 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7040 # Note: $oldconf is undef if VM does not exists
7041 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7042 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7043 my $new_conf_raw = '';
7045 my $rpcenv = PVE
::RPCEnvironment
::get
();
7046 my $devinfo = {}; # info about drives included in backup
7047 my $virtdev_hash = {}; # info about allocated drives
7055 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7057 my $cfgfn = "$tmpdir/qemu-server.conf";
7058 my $firewall_config_fn = "$tmpdir/fw.conf";
7059 my $index_fn = "$tmpdir/index.json";
7061 my $cmd = "restore";
7063 my $param = [$pbs_backup_name, "index.json", $index_fn];
7064 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7065 my $index = PVE
::Tools
::file_get_contents
($index_fn);
7066 $index = decode_json
($index);
7068 foreach my $info (@{$index->{files
}}) {
7069 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
7071 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
7072 $devinfo->{$devname}->{size
} = $1;
7074 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7079 my $is_qemu_server_backup = scalar(
7080 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
7082 if (!$is_qemu_server_backup) {
7083 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7085 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
7087 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7088 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7090 if ($has_firewall_config) {
7091 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7092 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7094 my $pve_firewall_dir = '/etc/pve/firewall';
7095 mkdir $pve_firewall_dir; # make sure the dir exists
7096 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7099 my $fh = IO
::File-
>new($cfgfn, "r") ||
7100 die "unable to read qemu-server.conf - $!\n";
7102 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7104 # fixme: rate limit?
7106 # create empty/temp config
7107 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
7109 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7112 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7114 foreach my $virtdev (sort keys %$virtdev_hash) {
7115 my $d = $virtdev_hash->{$virtdev};
7116 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7118 # this fails if storage is unavailable
7119 my $volid = $d->{volid
};
7120 my $path = PVE
::Storage
::path
($storecfg, $volid);
7122 # for live-restore we only want to preload the efidisk and TPM state
7123 next if $options->{live
} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7126 if (defined(my $ns = $scfg->{namespace
})) {
7127 @ns_arg = ('--ns', $ns);
7130 my $pbs_restore_cmd = [
7131 '/usr/bin/pbs-restore',
7132 '--repository', $repo,
7135 "$d->{devname}.img.fidx",
7140 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
7141 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
7143 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
7144 push @$pbs_restore_cmd, '--skip-zero';
7147 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
7148 print "restore proxmox backup image: $dbg_cmdstring\n";
7149 run_command
($pbs_restore_cmd);
7152 $fh->seek(0, 0) || die "seek failed - $!\n";
7154 my $cookie = { netcount
=> 0 };
7155 while (defined(my $line = <$fh>)) {
7156 $new_conf_raw .= restore_update_config_line
(
7168 if ($err || !$options->{live
}) {
7169 $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7175 $restore_destroy_volumes->($storecfg, $virtdev_hash);
7179 if ($options->{live
}) {
7180 # keep lock during live-restore
7181 $new_conf_raw .= "\nlock: create";
7184 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $options->{override_conf
});
7185 check_restore_permissions
($rpcenv, $user, $new_conf);
7186 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7188 eval { rescan
($vmid, 1); };
7191 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
7193 if ($options->{live
}) {
7199 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
7201 my $conf = PVE
::QemuConfig-
>load_config($vmid);
7202 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
7204 # these special drives are already restored before start
7205 delete $devinfo->{'drive-efidisk0'};
7206 delete $devinfo->{'drive-tpmstate0-backup'};
7210 keyfile
=> $keyfile,
7211 snapshot
=> $pbs_backup_name,
7212 namespace
=> $namespace,
7214 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7216 PVE
::QemuConfig-
>remove_lock($vmid, "create");
7220 sub pbs_live_restore
{
7221 my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7223 print "starting VM for live-restore\n";
7224 print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7226 my $live_restore_backing = {};
7227 for my $ds (keys %$restored_disks) {
7228 $ds =~ m/^drive-(.*)$/;
7232 repository
=> $opts->{repo
},
7233 snapshot
=> $opts->{snapshot
},
7234 archive
=> "$ds.img.fidx",
7236 $pbs_conf->{keyfile
} = $opts->{keyfile
} if -e
$opts->{keyfile
};
7237 $pbs_conf->{namespace
} = $opts->{namespace
} if defined($opts->{namespace
});
7239 my $drive = parse_drive
($confname, $conf->{$confname});
7240 print "restoring '$ds' to '$drive->{file}'\n";
7242 my $pbs_name = "drive-${confname}-pbs";
7243 $live_restore_backing->{$confname} = {
7245 blockdev
=> print_pbs_blockdev
($pbs_conf, $pbs_name),
7249 my $drives_streamed = 0;
7251 # make sure HA doesn't interrupt our restore by stopping the VM
7252 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
7253 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7256 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7257 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7258 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'live-restore-backing' => $live_restore_backing}, {});
7260 my $qmeventd_fd = register_qmeventd_handle
($vmid);
7262 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7263 # this will effectively collapse the backing image chain consisting of
7264 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7265 # removes itself once all backing images vanish with 'auto-remove=on')
7267 for my $ds (sort keys %$restored_disks) {
7268 my $job_id = "restore-$ds";
7269 mon_cmd
($vmid, 'block-stream',
7270 'job-id' => $job_id,
7273 $jobs->{$job_id} = {};
7276 mon_cmd
($vmid, 'cont');
7277 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
7279 print "restore-drive jobs finished successfully, removing all tracking block devices"
7280 ." to disconnect from Proxmox Backup Server\n";
7282 for my $ds (sort keys %$restored_disks) {
7283 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7286 close($qmeventd_fd);
7292 warn "An error occurred during live-restore: $err\n";
7293 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
7294 die "live-restore failed\n";
7298 # Inspired by pbs live-restore, this restores with the disks being available as files.
7299 # Theoretically this can also be used to quick-start a full-clone vm if the
7300 # disks are all available as files.
7302 # The mapping should provide a path by config entry, such as
7303 # `{ scsi0 => { format => <qcow2|raw|...>, path => "/path/to/file", sata1 => ... } }`
7305 # This is used when doing a `create` call with the `--live-import` parameter,
7306 # where the disks get an `import-from=` property. The non-live part is
7307 # therefore already handled in the `$create_disks()` call happening in the
7309 sub live_import_from_files
{
7310 my ($mapping, $vmid, $conf, $restore_options) = @_;
7312 my $live_restore_backing = {};
7313 for my $dev (keys %$mapping) {
7314 die "disk not support for live-restoring: '$dev'\n"
7315 if !is_valid_drivename
($dev) || $dev =~ /^(?:efidisk|tpmstate)/;
7317 die "mapping contains disk '$dev' which does not exist in the config\n"
7318 if !exists($conf->{$dev});
7320 my $info = $mapping->{$dev};
7321 my ($format, $path) = $info->@{qw(format path)};
7322 die "missing path for '$dev' mapping\n" if !$path;
7323 die "missing format for '$dev' mapping\n" if !$format;
7324 die "invalid format '$format' for '$dev' mapping\n"
7325 if !grep { $format eq $_ } qw(raw qcow2 vmdk);
7327 $live_restore_backing->{$dev} = {
7328 name
=> "drive-$dev-restore",
7329 blockdev
=> "driver=$format,node-name=drive-$dev-restore"
7331 . ",file.driver=file,file.filename=$path"
7335 my $storecfg = PVE
::Storage
::config
();
7338 # make sure HA doesn't interrupt our restore by stopping the VM
7339 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
7340 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7343 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'live-restore-backing' => $live_restore_backing}, {});
7345 # prevent shutdowns from qmeventd when the VM powers off from the inside
7346 my $qmeventd_fd = register_qmeventd_handle
($vmid);
7348 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7349 # this will effectively collapse the backing image chain consisting of
7350 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7351 # removes itself once all backing images vanish with 'auto-remove=on')
7353 for my $ds (sort keys %$live_restore_backing) {
7354 my $job_id = "restore-$ds";
7355 mon_cmd
($vmid, 'block-stream',
7356 'job-id' => $job_id,
7357 device
=> "drive-$ds",
7359 $jobs->{$job_id} = {};
7362 mon_cmd
($vmid, 'cont');
7363 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
7365 print "restore-drive jobs finished successfully, removing all tracking block devices\n";
7367 for my $ds (sort keys %$live_restore_backing) {
7368 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "drive-$ds-restore");
7371 close($qmeventd_fd);
7377 warn "An error occurred during live-restore: $err\n";
7378 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
7379 die "live-restore failed\n";
7382 PVE
::QemuConfig-
>remove_lock($vmid, "import");
7385 sub restore_vma_archive
{
7386 my ($archive, $vmid, $user, $opts, $comp) = @_;
7388 my $readfrom = $archive;
7390 my $cfg = PVE
::Storage
::config
();
7392 my $bwlimit = $opts->{bwlimit
};
7394 my $dbg_cmdstring = '';
7395 my $add_pipe = sub {
7397 push @$commands, $cmd;
7398 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7399 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
7404 if ($archive eq '-') {
7407 # If we use a backup from a PVE defined storage we also consider that
7408 # storage's rate limit:
7409 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
7410 if (defined($volid)) {
7411 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
7412 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
7414 print STDERR
"applying read rate limit: $readlimit\n";
7415 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7416 $add_pipe->($cstream);
7422 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
7423 my $cmd = $info->{decompressor
};
7424 push @$cmd, $readfrom;
7428 my $tmpdir = "/var/tmp/vzdumptmp$$";
7431 # disable interrupts (always do cleanups)
7435 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
7437 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7438 POSIX
::mkfifo
($mapfifo, 0600);
7440 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7442 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7444 my $devinfo = {}; # info about drives included in backup
7445 my $virtdev_hash = {}; # info about allocated drives
7447 my $rpcenv = PVE
::RPCEnvironment
::get
();
7449 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7451 # Note: $oldconf is undef if VM does not exist
7452 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7453 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7454 my $new_conf_raw = '';
7458 my $print_devmap = sub {
7459 my $cfgfn = "$tmpdir/qemu-server.conf";
7461 # we can read the config - that is already extracted
7462 my $fh = IO
::File-
>new($cfgfn, "r") ||
7463 die "unable to read qemu-server.conf - $!\n";
7465 my $fwcfgfn = "$tmpdir/qemu-server.fw";
7467 my $pve_firewall_dir = '/etc/pve/firewall';
7468 mkdir $pve_firewall_dir; # make sure the dir exists
7469 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7472 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7474 foreach my $info (values %{$virtdev_hash}) {
7475 my $storeid = $info->{storeid
};
7476 next if defined($storage_limits{$storeid});
7478 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
7479 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7480 $storage_limits{$storeid} = $limit * 1024;
7483 foreach my $devname (keys %$devinfo) {
7484 die "found no device mapping information for device '$devname'\n"
7485 if !$devinfo->{$devname}->{virtdev
};
7488 # create empty/temp config
7490 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
7491 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7495 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7497 # print restore information to $fifofh
7498 foreach my $virtdev (sort keys %$virtdev_hash) {
7499 my $d = $virtdev_hash->{$virtdev};
7500 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7502 my $storeid = $d->{storeid
};
7503 my $volid = $d->{volid
};
7506 if (my $limit = $storage_limits{$storeid}) {
7507 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7510 my $write_zeros = 1;
7511 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
7515 my $path = PVE
::Storage
::path
($cfg, $volid);
7517 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7519 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7522 $fh->seek(0, 0) || die "seek failed - $!\n";
7524 my $cookie = { netcount
=> 0 };
7525 while (defined(my $line = <$fh>)) {
7526 $new_conf_raw .= restore_update_config_line
(
7545 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7546 local $SIG{ALRM
} = sub { die "got timeout\n"; };
7548 $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7555 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7556 my ($dev_id, $size, $devname) = ($1, $2, $3);
7557 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
7558 } elsif ($line =~ m/^CTIME: /) {
7559 # we correctly received the vma config, so we can disable
7560 # the timeout now for disk allocation
7561 alarm($oldtimeout || 0);
7562 $oldtimeout = undef;
7564 print $fifofh "done\n";
7570 print "restore vma archive: $dbg_cmdstring\n";
7571 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
7575 alarm($oldtimeout) if $oldtimeout;
7577 $restore_deactivate_volumes->($cfg, $virtdev_hash);
7579 close($fifofh) if $fifofh;
7584 $restore_destroy_volumes->($cfg, $virtdev_hash);
7588 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $opts->{override_conf
});
7589 check_restore_permissions
($rpcenv, $user, $new_conf);
7590 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7592 eval { rescan
($vmid, 1); };
7595 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
7598 sub restore_tar_archive
{
7599 my ($archive, $vmid, $user, $opts) = @_;
7601 if (scalar(keys $opts->{override_conf
}->%*) > 0) {
7602 my $keystring = join(' ', keys $opts->{override_conf
}->%*);
7603 die "cannot pass along options ($keystring) when restoring from tar archive\n";
7606 if ($archive ne '-') {
7607 my $firstfile = tar_archive_read_firstfile
($archive);
7608 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7609 if $firstfile ne 'qemu-server.conf';
7612 my $storecfg = PVE
::Storage
::config
();
7614 # avoid zombie disks when restoring over an existing VM -> cleanup first
7615 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7616 # skiplock=1 because qmrestore has set the 'create' lock itself already
7617 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
7618 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
7620 my $tocmd = "/usr/lib/qemu-server/qmextract";
7622 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
7623 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
7624 $tocmd .= ' --prealloc' if $opts->{prealloc
};
7625 $tocmd .= ' --info' if $opts->{info
};
7627 # tar option "xf" does not autodetect compression when read from STDIN,
7628 # so we pipe to zcat
7629 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
7630 PVE
::Tools
::shellquote
("--to-command=$tocmd");
7632 my $tmpdir = "/var/tmp/vzdumptmp$$";
7635 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
7636 local $ENV{VZDUMP_VMID
} = $vmid;
7637 local $ENV{VZDUMP_USER
} = $user;
7639 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7640 my $new_conf_raw = '';
7642 # disable interrupts (always do cleanups)
7646 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7654 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7656 if ($archive eq '-') {
7657 print "extracting archive from STDIN\n";
7658 run_command
($cmd, input
=> "<&STDIN");
7660 print "extracting archive '$archive'\n";
7664 return if $opts->{info
};
7668 my $statfile = "$tmpdir/qmrestore.stat";
7669 if (my $fd = IO
::File-
>new($statfile, "r")) {
7670 while (defined (my $line = <$fd>)) {
7671 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7672 $map->{$1} = $2 if $1;
7674 print STDERR
"unable to parse line in statfile - $line\n";
7680 my $confsrc = "$tmpdir/qemu-server.conf";
7682 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7684 my $cookie = { netcount
=> 0 };
7685 while (defined (my $line = <$srcfd>)) {
7686 $new_conf_raw .= restore_update_config_line
(
7697 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
7703 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7705 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7707 eval { rescan
($vmid, 1); };
7711 sub foreach_storage_used_by_vm
{
7712 my ($conf, $func) = @_;
7716 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7717 my ($ds, $drive) = @_;
7718 return if drive_is_cdrom
($drive);
7720 my $volid = $drive->{file
};
7722 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
7723 $sidhash->{$sid} = $sid if $sid;
7726 foreach my $sid (sort keys %$sidhash) {
7731 my $qemu_snap_storage = {
7734 sub do_snapshots_with_qemu
{
7735 my ($storecfg, $volid, $deviceid) = @_;
7737 return if $deviceid =~ m/tpmstate0/;
7739 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
7740 my $scfg = $storecfg->{ids
}->{$storage_name};
7741 die "could not find storage '$storage_name'\n" if !defined($scfg);
7743 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
7747 if ($volid =~ m/\.(qcow2|qed)$/){
7754 sub qga_check_running
{
7755 my ($vmid, $nowarn) = @_;
7757 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
7759 warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7765 sub template_create
{
7766 my ($vmid, $conf, $disk) = @_;
7768 my $storecfg = PVE
::Storage
::config
();
7770 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7771 my ($ds, $drive) = @_;
7773 return if drive_is_cdrom
($drive);
7774 return if $disk && $ds ne $disk;
7776 my $volid = $drive->{file
};
7777 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
7779 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
7780 $drive->{file
} = $voliddst;
7781 $conf->{$ds} = print_drive
($drive);
7782 PVE
::QemuConfig-
>write_config($vmid, $conf);
7786 sub convert_iscsi_path
{
7789 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
7794 my $initiator_name = get_initiator_name
();
7796 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7797 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7800 die "cannot convert iscsi path '$path', unkown format\n";
7803 sub qemu_img_convert
{
7804 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7806 my $storecfg = PVE
::Storage
::config
();
7807 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
7808 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
7810 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7814 my $src_is_iscsi = 0;
7818 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
7819 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7820 $src_format = qemu_img_format
($src_scfg, $src_volname);
7821 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
7822 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
7823 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
7824 } elsif (-f
$src_volid || -b
$src_volid) {
7825 $src_path = $src_volid;
7826 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7831 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7833 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7834 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7835 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7836 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7839 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7840 push @$cmd, '-l', "snapshot.name=$snapname"
7841 if $snapname && $src_format && $src_format eq "qcow2";
7842 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7843 push @$cmd, '-T', $cachemode if defined($cachemode);
7844 push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7846 if ($src_is_iscsi) {
7847 push @$cmd, '--image-opts';
7848 $src_path = convert_iscsi_path
($src_path);
7849 } elsif ($src_format) {
7850 push @$cmd, '-f', $src_format;
7853 if ($dst_is_iscsi) {
7854 push @$cmd, '--target-image-opts';
7855 $dst_path = convert_iscsi_path
($dst_path);
7857 push @$cmd, '-O', $dst_format;
7860 push @$cmd, $src_path;
7862 if (!$dst_is_iscsi && $is_zero_initialized) {
7863 push @$cmd, "zeroinit:$dst_path";
7865 push @$cmd, $dst_path;
7870 if($line =~ m/\((\S+)\/100\
%\)/){
7872 my $transferred = int($size * $percent / 100);
7873 my $total_h = render_bytes
($size, 1);
7874 my $transferred_h = render_bytes
($transferred, 1);
7876 print "transferred $transferred_h of $total_h ($percent%)\n";
7881 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7883 die "copy failed: $err" if $err;
7886 sub qemu_img_format
{
7887 my ($scfg, $volname) = @_;
7889 # FIXME: this entire function is kind of weird given that `parse_volname`
7890 # also already gives us a format?
7891 my $is_path_storage = $scfg->{path
} || $scfg->{type
} eq 'esxi';
7893 if ($is_path_storage && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7900 sub qemu_drive_mirror
{
7901 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7903 $jobs = {} if !$jobs;
7907 $jobs->{"drive-$drive"} = {};
7909 if ($dst_volid =~ /^nbd:/) {
7910 $qemu_target = $dst_volid;
7913 my $storecfg = PVE
::Storage
::config
();
7914 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7916 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7918 $format = qemu_img_format
($dst_scfg, $dst_volname);
7920 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7922 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7925 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7926 $opts->{format
} = $format if $format;
7928 if (defined($src_bitmap)) {
7929 $opts->{sync
} = 'incremental';
7930 $opts->{bitmap
} = $src_bitmap;
7931 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7934 if (defined($bwlimit)) {
7935 $opts->{speed
} = $bwlimit * 1024;
7936 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7938 print "drive mirror is starting for drive-$drive\n";
7941 # if a job already runs for this device we get an error, catch it for cleanup
7942 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7944 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7946 die "mirroring error: $err\n";
7949 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7952 # $completion can be either
7953 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7954 # 'cancel': wait until all jobs are ready, block-job-cancel them
7955 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7956 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7957 sub qemu_drive_mirror_monitor
{
7958 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7960 $completion //= 'complete';
7964 my $err_complete = 0;
7966 my $starttime = time ();
7968 die "block job ('$op') timed out\n" if $err_complete > 300;
7970 my $stats = mon_cmd
($vmid, "query-block-jobs");
7973 my $running_jobs = {};
7974 for my $stat (@$stats) {
7975 next if $stat->{type
} ne $op;
7976 $running_jobs->{$stat->{device
}} = $stat;
7979 my $readycounter = 0;
7981 for my $job_id (sort keys %$jobs) {
7982 my $job = $running_jobs->{$job_id};
7984 my $vanished = !defined($job);
7985 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7986 if($complete || ($vanished && $completion eq 'auto')) {
7987 print "$job_id: $op-job finished\n";
7988 delete $jobs->{$job_id};
7992 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7994 my $busy = $job->{busy
};
7995 my $ready = $job->{ready
};
7996 if (my $total = $job->{len
}) {
7997 my $transferred = $job->{offset
} || 0;
7998 my $remaining = $total - $transferred;
7999 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
8001 my $duration = $ctime - $starttime;
8002 my $total_h = render_bytes
($total, 1);
8003 my $transferred_h = render_bytes
($transferred, 1);
8005 my $status = sprintf(
8006 "transferred $transferred_h of $total_h ($percent%%) in %s",
8007 render_duration
($duration),
8012 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
8014 $status .= ", ready";
8017 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
8018 $jobs->{$job_id}->{ready
} = $ready;
8021 $readycounter++ if $job->{ready
};
8024 last if scalar(keys %$jobs) == 0;
8026 if ($readycounter == scalar(keys %$jobs)) {
8027 print "all '$op' jobs are ready\n";
8029 # do the complete later (or has already been done)
8030 last if $completion eq 'skip' || $completion eq 'auto';
8032 if ($vmiddst && $vmiddst != $vmid) {
8033 my $agent_running = $qga && qga_check_running
($vmid);
8034 if ($agent_running) {
8035 print "freeze filesystem\n";
8036 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
8039 print "suspend vm\n";
8040 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
8044 # if we clone a disk for a new target vm, we don't switch the disk
8045 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
8047 if ($agent_running) {
8048 print "unfreeze filesystem\n";
8049 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
8052 print "resume vm\n";
8053 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
8060 for my $job_id (sort keys %$jobs) {
8061 # try to switch the disk if source and destination are on the same guest
8062 print "$job_id: Completing block job_id...\n";
8065 if ($completion eq 'complete') {
8066 $op = 'block-job-complete';
8067 } elsif ($completion eq 'cancel') {
8068 $op = 'block-job-cancel';
8070 die "invalid completion value: $completion\n";
8072 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
8073 if ($@ =~ m/cannot be completed/) {
8074 print "$job_id: block job cannot be completed, trying again.\n";
8077 print "$job_id: Completed successfully.\n";
8078 $jobs->{$job_id}->{complete
} = 1;
8089 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
8090 die "block job ($op) error: $err";
8094 sub qemu_blockjobs_cancel
{
8095 my ($vmid, $jobs) = @_;
8097 foreach my $job (keys %$jobs) {
8098 print "$job: Cancelling block job\n";
8099 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
8100 $jobs->{$job}->{cancel
} = 1;
8104 my $stats = mon_cmd
($vmid, "query-block-jobs");
8106 my $running_jobs = {};
8107 foreach my $stat (@$stats) {
8108 $running_jobs->{$stat->{device
}} = $stat;
8111 foreach my $job (keys %$jobs) {
8113 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
8114 print "$job: Done.\n";
8115 delete $jobs->{$job};
8119 last if scalar(keys %$jobs) == 0;
8125 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8126 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8127 my sub clone_disk_check_io_uring
{
8128 my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8130 return if !$use_drive_mirror;
8132 # Don't complain when not changing storage.
8133 # Assume if it works for the source, it'll work for the target too.
8134 return if $src_storeid eq $dst_storeid;
8136 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
8137 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
8139 my $cache_direct = drive_uses_cache_direct
($src_drive);
8141 my $src_uses_io_uring;
8142 if ($src_drive->{aio
}) {
8143 $src_uses_io_uring = $src_drive->{aio
} eq 'io_uring';
8145 $src_uses_io_uring = storage_allows_io_uring_default
($src_scfg, $cache_direct);
8148 die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8149 if $src_uses_io_uring && !storage_allows_io_uring_default
($dst_scfg, $cache_direct);
8153 my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8155 my ($vmid, $running) = $source->@{qw(vmid running)};
8156 my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8158 my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8159 my ($storage, $format) = $dest->@{qw(storage format)};
8161 my $unused = defined($src_drivename) && $src_drivename =~ /^unused/;
8162 my $use_drive_mirror = $full && $running && $src_drivename && !$snapname && !$unused;
8164 if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8165 die "cloning from/to EFI disk requires EFI disk\n"
8166 if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8167 die "cloning from/to TPM state requires TPM state\n"
8168 if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8170 # This would lead to two device nodes in QEMU pointing to the same backing image!
8171 die "cannot change drive name when cloning disk from/to the same VM\n"
8172 if $use_drive_mirror && $vmid == $newvmid;
8175 die "cannot move TPM state while VM is running\n"
8176 if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8180 print "create " . ($full ?
'full' : 'linked') . " clone of drive ";
8181 print "$src_drivename " if $src_drivename;
8182 print "($drive->{file})\n";
8185 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
8186 push @$newvollist, $newvolid;
8188 my ($src_storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
8189 my $storeid = $storage || $src_storeid;
8191 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
8195 if (drive_is_cloudinit
($drive)) {
8196 $name = "vm-$newvmid-cloudinit";
8197 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8198 if ($scfg->{path
}) {
8199 $name .= ".$dst_format";
8202 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
8203 } elsif ($dst_drivename eq 'efidisk0') {
8204 $size = $efisize or die "internal error - need to specify EFI disk size\n";
8205 } elsif ($dst_drivename eq 'tpmstate0') {
8206 $dst_format = 'raw';
8207 $size = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8209 clone_disk_check_io_uring
($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8211 $size = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
8213 $newvolid = PVE
::Storage
::vdisk_alloc
(
8214 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8216 push @$newvollist, $newvolid;
8218 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
8220 if (drive_is_cloudinit
($drive)) {
8221 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8222 # if this is the case, we have to complete any block-jobs still there from
8223 # previous drive-mirrors
8224 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8225 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
8230 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
8231 if ($use_drive_mirror) {
8232 qemu_drive_mirror
($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8233 $completion, $qga, $bwlimit);
8235 if ($dst_drivename eq 'efidisk0') {
8236 # the relevant data on the efidisk may be smaller than the source
8237 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8238 # that is given by the OVMF_VARS.fd
8239 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
8240 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
8242 my $src_format = (PVE
::Storage
::parse_volname
($storecfg, $drive->{file
}))[6];
8244 # better for Ceph if block size is not too small, see bug #3324
8247 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8249 if ($src_format eq 'qcow2' && $snapname) {
8250 die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8251 if !min_version
(kvm_user_version
(), 6, 2);
8252 push $cmd->@*, '-l', $snapname;
8254 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8257 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8263 my $size = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
8265 my $disk = dclone
($drive);
8266 delete $disk->{format
};
8267 $disk->{file
} = $newvolid;
8268 $disk->{size
} = $size if defined($size) && !$unused;
8273 sub get_running_qemu_version
{
8275 my $res = mon_cmd
($vmid, "query-version");
8276 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8279 sub qemu_use_old_bios_files
{
8280 my ($machine_type) = @_;
8282 return if !$machine_type;
8284 my $use_old_bios_files = undef;
8286 if ($machine_type =~ m/^(\S+)\.pxe$/) {
8288 $use_old_bios_files = 1;
8290 my $version = extract_version
($machine_type, kvm_user_version
());
8291 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8292 # load new efi bios files on migration. So this hack is required to allow
8293 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8294 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8295 $use_old_bios_files = !min_version
($version, 2, 4);
8298 return ($use_old_bios_files, $machine_type);
8301 sub get_efivars_size
{
8302 my ($conf, $efidisk) = @_;
8304 my $arch = get_vm_arch
($conf);
8305 $efidisk //= $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
8306 my $smm = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
8307 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8308 return -s
$ovmf_vars;
8311 sub update_efidisk_size
{
8314 return if !defined($conf->{efidisk0
});
8316 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
8317 $disk->{size
} = get_efivars_size
($conf);
8318 $conf->{efidisk0
} = print_drive
($disk);
8323 sub update_tpmstate_size
{
8326 my $disk = PVE
::QemuServer
::parse_drive
('tpmstate0', $conf->{tpmstate0
});
8327 $disk->{size
} = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8328 $conf->{tpmstate0
} = print_drive
($disk);
8331 sub create_efidisk
($$$$$$$) {
8332 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8334 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8336 my $vars_size_b = -s
$ovmf_vars;
8337 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
8338 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8339 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
8341 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
8342 my $size = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
8344 return ($volid, $size/1024);
8347 sub vm_iothreads_list
{
8350 my $res = mon_cmd
($vmid, 'query-iothreads');
8353 foreach my $iothread (@$res) {
8354 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
8361 my ($conf, $drive) = @_;
8365 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
8367 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
8373 my $controller = int($drive->{index} / $maxdev);
8374 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
8378 return ($maxdev, $controller, $controller_prefix);
8381 sub resolve_dst_disk_format
{
8382 my ($storecfg, $storeid, $src_volname, $format) = @_;
8383 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
8386 # if no target format is specified, use the source disk format as hint
8388 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8389 $format = qemu_img_format
($scfg, $src_volname);
8395 # test if requested format is supported - else use default
8396 my $supported = grep { $_ eq $format } @$validFormats;
8397 $format = $defFormat if !$supported;
8401 # NOTE: if this logic changes, please update docs & possibly gui logic
8402 sub find_vmstate_storage
{
8403 my ($conf, $storecfg) = @_;
8405 # first, return storage from conf if set
8406 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
8408 my ($target, $shared, $local);
8410 foreach_storage_used_by_vm
($conf, sub {
8412 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
8413 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
8414 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
8417 # second, use shared storage where VM has at least one disk
8418 # third, use local storage where VM has at least one disk
8419 # fall back to local storage
8420 $target = $shared // $local // 'local';
8426 my ($uuid, $uuid_str);
8427 UUID
::generate
($uuid);
8428 UUID
::unparse
($uuid, $uuid_str);
8432 sub generate_smbios1_uuid
{
8433 return "uuid=".generate_uuid
();
8439 mon_cmd
($vmid, 'nbd-server-stop', timeout
=> 25);
8442 sub create_reboot_request
{
8444 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8445 or die "failed to create reboot trigger file: $!\n";
8449 sub clear_reboot_request
{
8451 my $path = "/run/qemu-server/$vmid.reboot";
8454 $res = unlink($path);
8455 die "could not remove reboot request for $vmid: $!"
8456 if !$res && $! != POSIX
::ENOENT
;
8461 sub bootorder_from_legacy
{
8462 my ($conf, $bootcfg) = @_;
8464 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
8465 my $bootindex_hash = {};
8467 foreach my $o (split(//, $boot)) {
8468 $bootindex_hash->{$o} = $i*100;
8474 PVE
::QemuConfig-
>foreach_volume($conf, sub {
8475 my ($ds, $drive) = @_;
8477 if (drive_is_cdrom
($drive, 1)) {
8478 if ($bootindex_hash->{d
}) {
8479 $bootorder->{$ds} = $bootindex_hash->{d
};
8480 $bootindex_hash->{d
} += 1;
8482 } elsif ($bootindex_hash->{c
}) {
8483 $bootorder->{$ds} = $bootindex_hash->{c
}
8484 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
8485 $bootindex_hash->{c
} += 1;
8489 if ($bootindex_hash->{n
}) {
8490 for (my $i = 0; $i < $MAX_NETS; $i++) {
8491 my $netname = "net$i";
8492 next if !$conf->{$netname};
8493 $bootorder->{$netname} = $bootindex_hash->{n
};
8494 $bootindex_hash->{n
} += 1;
8501 # Generate default device list for 'boot: order=' property. Matches legacy
8502 # default boot order, but with explicit device names. This is important, since
8503 # the fallback for when neither 'order' nor the old format is specified relies
8504 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8505 sub get_default_bootdevices
{
8511 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
8512 push @ret, $first if $first;
8515 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
8516 push @ret, $first if $first;
8519 for (my $i = 0; $i < $MAX_NETS; $i++) {
8520 my $netname = "net$i";
8521 next if !$conf->{$netname};
8522 push @ret, $netname;
8529 sub device_bootorder
{
8532 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
8534 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
8537 if (!defined($boot) || $boot->{legacy
}) {
8538 $bootorder = bootorder_from_legacy
($conf, $boot);
8539 } elsif ($boot->{order
}) {
8540 my $i = 100; # start at 100 to allow user to insert devices before us with -args
8541 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
8542 $bootorder->{$dev} = $i++;
8549 sub register_qmeventd_handle
{
8553 my $peer = "/var/run/qmeventd.sock";
8558 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
8560 if ($! != EINTR
&& $! != EAGAIN
) {
8561 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8564 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8565 . "after $count retries\n";
8570 # send handshake to mark VM as backing up
8571 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
8573 # return handle to be closed later when inhibit is no longer required
8577 # bash completion helper
8579 sub complete_backup_archives
{
8580 my ($cmdname, $pname, $cvalue) = @_;
8582 my $cfg = PVE
::Storage
::config
();
8586 if ($cvalue =~ m/^([^:]+):/) {
8590 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
8593 foreach my $id (keys %$data) {
8594 foreach my $item (@{$data->{$id}}) {
8595 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8596 push @$res, $item->{volid
} if defined($item->{volid
});
8603 my $complete_vmid_full = sub {
8606 my $idlist = vmstatus
();
8610 foreach my $id (keys %$idlist) {
8611 my $d = $idlist->{$id};
8612 if (defined($running)) {
8613 next if $d->{template
};
8614 next if $running && $d->{status
} ne 'running';
8615 next if !$running && $d->{status
} eq 'running';
8624 return &$complete_vmid_full();
8627 sub complete_vmid_stopped
{
8628 return &$complete_vmid_full(0);
8631 sub complete_vmid_running
{
8632 return &$complete_vmid_full(1);
8635 sub complete_storage
{
8637 my $cfg = PVE
::Storage
::config
();
8638 my $ids = $cfg->{ids
};
8641 foreach my $sid (keys %$ids) {
8642 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
8643 next if !$ids->{$sid}->{content
}->{images
};
8650 sub complete_migration_storage
{
8651 my ($cmd, $param, $current_value, $all_args) = @_;
8653 my $targetnode = @$all_args[1];
8655 my $cfg = PVE
::Storage
::config
();
8656 my $ids = $cfg->{ids
};
8659 foreach my $sid (keys %$ids) {
8660 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
8661 next if !$ids->{$sid}->{content
}->{images
};
8669 my ($vmid, $include_suspended) = @_;
8670 my $qmpstatus = eval {
8671 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
8672 mon_cmd
($vmid, "query-status");
8675 return $qmpstatus && (
8676 $qmpstatus->{status
} eq "paused" ||
8677 $qmpstatus->{status
} eq "prelaunch" ||
8678 ($include_suspended && $qmpstatus->{status
} eq "suspended")
8682 sub check_volume_storage_type
{
8683 my ($storecfg, $vol) = @_;
8685 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
8686 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8687 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
8689 die "storage '$storeid' does not support content-type '$vtype'\n"
8690 if !$scfg->{content
}->{$vtype};
8695 sub add_nets_bridge_fdb
{
8696 my ($conf, $vmid) = @_;
8698 for my $opt (keys %$conf) {
8699 next if $opt !~ m/^net(\d+)$/;
8700 my $iface = "tap${vmid}i$1";
8701 # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8702 my $net = parse_net
($conf->{$opt}, 1) or next;
8704 my $mac = $net->{macaddr
};
8706 log_warn
("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8707 if !file_read_firstline
("/sys/class/net/$iface/brport/learning");
8711 my $bridge = $net->{bridge
};
8713 log_warn
("Interface '$iface' not attached to any bridge.");
8717 PVE
::Network
::SDN
::Zones
::add_bridge_fdb
($iface, $mac, $bridge);
8718 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8719 PVE
::Network
::add_bridge_fdb
($iface, $mac);
8724 sub del_nets_bridge_fdb
{
8725 my ($conf, $vmid) = @_;
8727 for my $opt (keys %$conf) {
8728 next if $opt !~ m/^net(\d+)$/;
8729 my $iface = "tap${vmid}i$1";
8731 my $net = parse_net
($conf->{$opt}) or next;
8732 my $mac = $net->{macaddr
} or next;
8734 my $bridge = $net->{bridge
};
8736 PVE
::Network
::SDN
::Zones
::del_bridge_fdb
($iface, $mac, $bridge);
8737 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8738 PVE
::Network
::del_bridge_fdb
($iface, $mac);
8743 sub create_ifaces_ipams_ips
{
8744 my ($conf, $vmid) = @_;
8746 return if !$have_sdn;
8748 foreach my $opt (keys %$conf) {
8749 if ($opt =~ m/^net(\d+)$/) {
8750 my $value = $conf->{$opt};
8751 my $net = PVE
::QemuServer
::parse_net
($value);
8752 eval { PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($net->{bridge
}, $conf->{name
}, $net->{macaddr
}, $vmid, undef, 1) };
8758 sub delete_ifaces_ipams_ips
{
8759 my ($conf, $vmid) = @_;
8761 return if !$have_sdn;
8763 foreach my $opt (keys %$conf) {
8764 if ($opt =~ m/^net(\d+)$/) {
8765 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
8766 eval { PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
}) };