1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
22 use List
::Util
qw(first);
25 use Storable
qw(dclone);
26 use Time
::HiRes
qw(gettimeofday usleep);
30 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
33 use PVE
::DataCenterConfig
;
34 use PVE
::Exception
qw(raise raise_param_exc);
35 use PVE
::Format
qw(render_duration render_bytes);
36 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
38 use PVE
::Mapping
::PCI
;
39 use PVE
::Mapping
::USB
;
41 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
44 use PVE
::RESTEnvironment
qw(log_warn);
45 use PVE
::RPCEnvironment
;
49 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
53 use PVE
::QemuServer
::Helpers
qw(config_aware_timeout min_version windows_version);
54 use PVE
::QemuServer
::Cloudinit
;
55 use PVE
::QemuServer
::CGroup
;
56 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch);
57 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
58 use PVE
::QemuServer
::Machine
;
59 use PVE
::QemuServer
::Memory
qw(get_current_memory);
60 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
61 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
62 use PVE
::QemuServer
::QMPHelpers
qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
63 use PVE
::QemuServer
::USB
;
67 require PVE
::Network
::SDN
::Zones
;
68 require PVE
::Network
::SDN
::Vnets
;
72 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
76 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
77 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
80 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
81 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
84 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
85 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
88 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
89 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
91 # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
92 # anymore. how can we deperacate this sanely without breaking existing instances, or using
93 # older backups and snapshot?
95 "$EDK2_FW_BASE/OVMF_CODE.fd",
96 "$EDK2_FW_BASE/OVMF_VARS.fd",
101 "$EDK2_FW_BASE/AAVMF_CODE.fd",
102 "$EDK2_FW_BASE/AAVMF_VARS.fd",
107 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
109 # Note about locking: we use flock on the config file protect against concurent actions.
110 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
111 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
112 # But you can ignore this kind of lock with the --skiplock flag.
120 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
121 description
=> "Some command save/restore state from this location.",
127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
130 $nodename_cache //= PVE
::INotify
::nodename
();
131 return $nodename_cache;
138 enum
=> [qw(i6300esb ib700)],
139 description
=> "Watchdog type to emulate.",
140 default => 'i6300esb',
145 enum
=> [qw(reset shutdown poweroff pause debug none)],
146 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
150 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
154 description
=> "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
159 fstrim_cloned_disks
=> {
160 description
=> "Run fstrim after moving a disk or migrating the VM.",
165 'freeze-fs-on-backup' => {
166 description
=> "Freeze/thaw guest filesystems on backup for consistency.",
172 description
=> "Select the agent type",
176 enum
=> [qw(virtio isa)],
182 description
=> "Select the VGA type.",
187 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
190 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
197 description
=> 'Enable a specific clipboard. If not set, depending on the display type the'
198 .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
209 description
=> "The size of the file in MB.",
213 pattern
=> '[a-zA-Z0-9\-]+',
215 format_description
=> 'string',
216 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
223 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
224 description
=> "Configure an audio device."
228 enum
=> ['spice', 'none'],
231 description
=> "Driver backend for the audio device."
235 my $spice_enhancements_fmt = {
240 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
244 enum
=> ['off', 'all', 'filter'],
247 description
=> "Enable video streaming. Uses compression for detected video streams."
254 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
256 description
=> "The file on the host to gather entropy from. In most cases '/dev/urandom'"
257 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
258 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
259 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
260 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
261 ." a hardware RNG from the host.",
265 description
=> "Maximum bytes of entropy allowed to get injected into the guest every"
266 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
267 ." `0` to disable limiting (potentially dangerous!).",
270 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
271 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
272 # reading from /dev/urandom
277 description
=> "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
278 ." the guest to retrieve another 'max_bytes' of entropy.",
284 my $meta_info_fmt = {
287 description
=> "The guest creation timestamp as UNIX epoch time",
293 description
=> "The QEMU (machine) version from the time this VM was created.",
294 pattern
=> '\d+(\.\d+)+',
303 description
=> "Specifies whether a VM will be started during system bootup.",
309 description
=> "Automatic restart after crash (currently ignored).",
314 type
=> 'string', format
=> 'pve-hotplug-features',
315 description
=> "Selectively enable hotplug features. This is a comma separated list of"
316 ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
317 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
318 ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
320 default => 'network,disk,usb',
325 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
331 description
=> "Lock/unlock the VM.",
332 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
337 description
=> "Limit of CPU usage.",
338 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
339 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
347 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
348 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
349 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
350 ." weights of all the other running VMs.",
353 default => 'cgroup v1: 1024, cgroup v2: 100',
358 description
=> "Memory properties.",
359 format
=> $PVE::QemuServer
::Memory
::memory_fmt
364 description
=> "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
370 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the"
371 ." more memory this VM gets. Number is relative to weights of all other running VMs."
372 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
380 description
=> "Keyboard layout for VNC server. This option is generally not required and"
381 ." is often better handled from within the guest OS.",
382 enum
=> PVE
::Tools
::kvmkeymaplist
(),
387 type
=> 'string', format
=> 'dns-name',
388 description
=> "Set a name for the VM. Only used on the configuration web interface.",
393 description
=> "SCSI controller model",
394 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
400 description
=> "Description for the VM. Shown in the web-interface VM's summary."
401 ." This is saved as comment inside the configuration file.",
402 maxLength
=> 1024 * 8,
407 # NOTE: When extending, also consider extending `%guest_types` in `Import/ESXi.pm`.
408 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
409 description
=> "Specify guest operating system.",
410 verbose_description
=> <<EODESC,
411 Specify guest operating system. This is used to enable special
412 optimization/features for specific operating systems:
415 other;; unspecified OS
416 wxp;; Microsoft Windows XP
417 w2k;; Microsoft Windows 2000
418 w2k3;; Microsoft Windows 2003
419 w2k8;; Microsoft Windows 2008
420 wvista;; Microsoft Windows Vista
421 win7;; Microsoft Windows 7
422 win8;; Microsoft Windows 8/2012/2012r2
423 win10;; Microsoft Windows 10/2016/2019
424 win11;; Microsoft Windows 11/2022/2025
425 l24;; Linux 2.4 Kernel
426 l26;; Linux 2.6 - 6.X Kernel
427 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
432 type
=> 'string', format
=> 'pve-qm-boot',
433 description
=> "Specify guest boot order. Use the 'order=' sub-property as usage with no"
434 ." key or 'legacy=' is deprecated.",
438 type
=> 'string', format
=> 'pve-qm-bootdisk',
439 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
440 pattern
=> '(ide|sata|scsi|virtio)\d+',
445 description
=> "The number of CPUs. Please use option -sockets instead.",
452 description
=> "The number of CPU sockets.",
459 description
=> "The number of cores per socket.",
466 description
=> "Enable/disable NUMA.",
472 description
=> "Enable/disable hugepages memory.",
473 enum
=> [qw(any 2 1024)],
479 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
480 ." after VM shutdown and can be used for subsequent starts.",
485 description
=> "Number of hotplugged vcpus.",
492 description
=> "Enable/disable ACPI.",
497 description
=> "Enable/disable communication with the QEMU Guest Agent and its properties.",
499 format
=> $agent_fmt,
504 description
=> "Enable/disable KVM hardware virtualization.",
510 description
=> "Enable/disable time drift fix.",
516 description
=> "Set the real time clock (RTC) to local time. This is enabled by default if"
517 ." the `ostype` indicates a Microsoft Windows OS.",
522 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
526 type
=> 'string', format
=> $vga_fmt,
527 description
=> "Configure the VGA hardware.",
528 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
529 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
530 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
531 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
532 ." display server. For win* OS you can select how many independent displays you want,"
533 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
534 ." using a serial device as terminal.",
538 type
=> 'string', format
=> 'pve-qm-watchdog',
539 description
=> "Create a virtual hardware watchdog device.",
540 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
541 ." action), the watchdog must be periodically polled by an agent inside the guest or"
542 ." else the watchdog will reset the guest (or execute the respective action specified)",
547 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
548 description
=> "Set the initial date of the real time clock. Valid format for date are:"
549 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
550 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
553 startup
=> get_standard_option
('pve-startup-order'),
557 description
=> "Enable/disable Template.",
563 description
=> "Arbitrary arguments passed to kvm.",
564 verbose_description
=> <<EODESCR,
565 Arbitrary arguments passed to kvm, for example:
567 args: -no-reboot -smbios 'type=0,vendor=FOO'
569 NOTE: this option is for experts only.
576 description
=> "Enable/disable the USB tablet device.",
577 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
578 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
579 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
580 ." may consider disabling this to save some context switches. This is turned off by"
581 ." default if you use spice (`qm set <vmid> --vga qxl`).",
586 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
590 migrate_downtime
=> {
593 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
599 type
=> 'string', format
=> 'pve-qm-ide',
600 typetext
=> '<volume>',
601 description
=> "This is an alias for option -ide2",
605 description
=> "Emulated CPU type.",
607 format
=> 'pve-vm-cpu-conf',
609 parent
=> get_standard_option
('pve-snapshot-name', {
611 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
615 description
=> "Timestamp for snapshots.",
621 type
=> 'string', format
=> 'pve-volume-id',
622 description
=> "Reference to a volume which stores the VM state. This is used internally"
625 vmstatestorage
=> get_standard_option
('pve-storage-id', {
626 description
=> "Default storage for VM state volumes/files.",
629 runningmachine
=> get_standard_option
('pve-qemu-machine', {
630 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
634 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
635 ." internally for snapshots.",
638 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
639 format_description
=> 'QEMU -cpu parameter'
641 machine
=> get_standard_option
('pve-qemu-machine'),
643 description
=> "Virtual processor architecture. Defaults to the host.",
646 enum
=> [qw(x86_64 aarch64)],
649 description
=> "Specify SMBIOS type 1 fields.",
650 type
=> 'string', format
=> 'pve-qm-smbios1',
657 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
658 ." remove disk operations.",
664 enum
=> [ qw(seabios ovmf) ],
665 description
=> "Select BIOS implementation.",
666 default => 'seabios',
670 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
671 format_description
=> 'UUID',
672 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
673 ." to disable explicitly.",
674 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
675 ." value identifier to the guest OS. This allows to notify the guest operating system"
676 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
677 ." execution or creation from a template). The guest operating system notices the"
678 ." change, and is then able to react as appropriate by marking its copies of"
679 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
680 ."Note that auto-creation only works when done through API/CLI create or update methods"
681 .", but not when manually editing the config file.",
682 default => "1 (autogenerated)",
687 format
=> 'pve-volume-id',
689 description
=> "Script that will be executed during various steps in the vms lifetime.",
693 format
=> $ivshmem_fmt,
694 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
700 format
=> $audio_fmt,
701 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
704 spice_enhancements
=> {
706 format
=> $spice_enhancements_fmt,
707 description
=> "Configure additional enhancements for SPICE.",
711 type
=> 'string', format
=> 'pve-tag-list',
712 description
=> 'Tags of the VM. This is only meta information.',
718 description
=> "Configure a VirtIO-based Random Number Generator.",
723 format
=> $meta_info_fmt,
724 description
=> "Some (read-only) meta-information about this guest.",
728 type
=> 'string', format
=> 'pve-cpuset',
729 description
=> "List of host cores used to execute guest processes, for example: 0,5,8-11",
738 description
=> 'Specify a custom file containing all meta data passed to the VM via"
739 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
740 format
=> 'pve-volume-id',
741 format_description
=> 'volume',
746 description
=> 'To pass a custom file containing all network data to the VM via cloud-init.',
747 format
=> 'pve-volume-id',
748 format_description
=> 'volume',
753 description
=> 'To pass a custom file containing all user data to the VM via cloud-init.',
754 format
=> 'pve-volume-id',
755 format_description
=> 'volume',
760 description
=> 'To pass a custom file containing all vendor data to the VM via cloud-init.',
761 format
=> 'pve-volume-id',
762 format_description
=> 'volume',
765 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
767 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
768 my $confdesc_cloudinit = {
772 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
773 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
774 .' and `configdrive2` for windows.',
775 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
780 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
781 ." image's configured default user.",
786 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
787 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
788 .' support hashed passwords.',
793 description
=> 'cloud-init: do an automatic package upgrade after the first boot.',
799 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
801 format
=> 'pve-qm-cicustom',
806 description
=> 'cloud-init: Sets DNS search domains for a container. Create will'
807 .' automatically use the setting from the host if neither searchdomain nor nameserver'
812 type
=> 'string', format
=> 'address-list',
813 description
=> 'cloud-init: Sets DNS server IP address for a container. Create will'
814 .' automatically use the setting from the host if neither searchdomain nor nameserver'
820 format
=> 'urlencoded',
821 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
825 # what about other qemu settings ?
827 #machine => 'string',
840 ##soundhw => 'string',
842 while (my ($k, $v) = each %$confdesc) {
843 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
847 my $MAX_SERIAL_PORTS = 4;
848 my $MAX_PARALLEL_PORTS = 3;
850 for (my $i = 0; $i < $PVE::QemuServer
::Memory
::MAX_NUMA
; $i++) {
851 $confdesc->{"numa$i"} = $PVE::QemuServer
::Memory
::numadesc
;
854 my $nic_model_list = [
870 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
872 my $net_fmt_bridge_descr = <<__EOD__;
873 Bridge to attach the network device to. The Proxmox VE standard bridge
876 If you do not specify a bridge, we create a kvm user (NATed) network
877 device, which provides DHCP and DNS services. The following addresses
884 The DHCP server assign addresses to the guest starting from 10.0.2.15.
888 macaddr
=> get_standard_option
('mac-addr', {
889 description
=> "MAC address. That address must be unique withing your network. This is"
890 ." automatically generated if not specified.",
894 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
895 ." very low CPU overhead. If your guest does not support this driver, it is usually"
896 ." best to use 'e1000'.",
897 enum
=> $nic_model_list,
900 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
901 bridge
=> get_standard_option
('pve-bridge-id', {
902 description
=> $net_fmt_bridge_descr,
907 minimum
=> 0, maximum
=> 64,
908 description
=> 'Number of packet queues to be used on the device.',
914 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
919 minimum
=> 1, maximum
=> 4094,
920 description
=> 'VLAN tag to apply to packets on this interface.',
925 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
926 description
=> 'VLAN trunks to pass through this interface.',
927 format_description
=> 'vlanid[;vlanid...]',
932 description
=> 'Whether this interface should be protected by the firewall.',
937 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
942 minimum
=> 1, maximum
=> 65520,
943 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
950 type
=> 'string', format
=> $net_fmt,
951 description
=> "Specify network devices.",
954 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
959 format
=> 'pve-ipv4-config',
960 format_description
=> 'IPv4Format/CIDR',
961 description
=> 'IPv4 address in CIDR format.',
968 format_description
=> 'GatewayIPv4',
969 description
=> 'Default gateway for IPv4 traffic.',
975 format
=> 'pve-ipv6-config',
976 format_description
=> 'IPv6Format/CIDR',
977 description
=> 'IPv6 address in CIDR format.',
984 format_description
=> 'GatewayIPv6',
985 description
=> 'Default gateway for IPv6 traffic.',
990 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
993 type
=> 'string', format
=> 'pve-qm-ipconfig',
994 description
=> <<'EODESCR',
995 cloud-init: Specify IP addresses and gateways for the corresponding interface.
997 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
999 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1000 gateway should be provided.
1001 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1002 cloud-init 19.4 or newer.
1004 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1008 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
1010 for (my $i = 0; $i < $MAX_NETS; $i++) {
1011 $confdesc->{"net$i"} = $netdesc;
1012 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1015 foreach my $key (keys %$confdesc_cloudinit) {
1016 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1019 PVE
::JSONSchema
::register_format
('pve-cpuset', \
&pve_verify_cpuset
);
1020 sub pve_verify_cpuset
{
1021 my ($set_text, $noerr) = @_;
1023 my ($count, $members) = eval { PVE
::CpuSet
::parse_cpuset
($set_text) };
1027 die "unable to parse cpuset option\n";
1030 return PVE
::CpuSet-
>new($members)->short_string();
1033 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1034 sub verify_volume_id_or_qm_path
{
1035 my ($volid, $noerr) = @_;
1037 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1039 return verify_volume_id_or_absolute_path
($volid, $noerr);
1042 PVE
::JSONSchema
::register_format
('pve-volume-id-or-absolute-path', \
&verify_volume_id_or_absolute_path
);
1043 sub verify_volume_id_or_absolute_path
{
1044 my ($volid, $noerr) = @_;
1046 return $volid if $volid =~ m
|^/|;
1048 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1059 pattern
=> '(/dev/.+|socket)',
1060 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1061 verbose_description
=> <<EODESCR,
1062 Create a serial device inside the VM (n is 0 to 3), and pass through a
1063 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1064 host side (use 'qm terminal' to open a terminal connection).
1066 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1067 use with special care.
1069 CAUTION: Experimental! User reported problems with this option.
1076 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1077 description
=> "Map host parallel devices (n is 0 to 2).",
1078 verbose_description
=> <<EODESCR,
1079 Map host parallel devices (n is 0 to 2).
1081 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1082 machines - use with special care.
1084 CAUTION: Experimental! User reported problems with this option.
1088 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1089 $confdesc->{"parallel$i"} = $paralleldesc;
1092 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1093 $confdesc->{"serial$i"} = $serialdesc;
1096 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1097 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1100 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1101 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1104 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
1105 $confdesc->{"usb$i"} = $PVE::QemuServer
::USB
::usbdesc
;
1113 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1114 . " Deprecated, use 'order=' instead.",
1115 pattern
=> '[acdn]{1,4}',
1116 format_description
=> "[acdn]{1,4}",
1118 # note: this is also the fallback if boot: is not given at all
1124 format
=> 'pve-qm-bootdev-list',
1125 format_description
=> "device[;device...]",
1126 description
=> <<EODESC,
1127 The guest will attempt to boot from devices in the order they appear here.
1129 Disks, optical drives and passed-through storage USB devices will be directly
1130 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1131 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1133 Note that only devices in this list will be marked as bootable and thus loaded
1134 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1135 (e.g. software-raid), you need to specify all of them here.
1137 Overrides the deprecated 'legacy=[acdn]*' value when given.
1141 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1143 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1144 sub verify_bootdev
{
1145 my ($dev, $noerr) = @_;
1147 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1148 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && !$special;
1152 return 0 if $dev !~ m/^$base\d+$/;
1153 return 0 if !$confdesc->{$dev};
1157 return $dev if $check->("net");
1158 return $dev if $check->("usb");
1159 return $dev if $check->("hostpci");
1162 die "invalid boot device '$dev'\n";
1165 sub print_bootorder
{
1167 return "" if !@$devs;
1168 my $data = { order
=> join(';', @$devs) };
1169 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1172 my $kvm_api_version = 0;
1175 return $kvm_api_version if $kvm_api_version;
1177 open my $fh, '<', '/dev/kvm' or return;
1179 # 0xae00 => KVM_GET_API_VERSION
1180 $kvm_api_version = ioctl($fh, 0xae00, 0);
1183 return $kvm_api_version;
1186 my $kvm_user_version = {};
1189 sub kvm_user_version
{
1192 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1193 my $st = stat($binary);
1195 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1196 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1197 $cachedmtime == $st->mtime;
1199 $kvm_user_version->{$binary} = 'unknown';
1200 $kvm_mtime->{$binary} = $st->mtime;
1204 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1205 $kvm_user_version->{$binary} = $2;
1209 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1212 return $kvm_user_version->{$binary};
1215 my sub extract_version
{
1216 my ($machine_type, $version) = @_;
1217 $version = kvm_user_version
() if !defined($version);
1218 return PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1221 sub kernel_has_vhost_net
{
1222 return -c
'/dev/vhost-net';
1227 return defined($confdesc->{$key});
1231 sub get_cdrom_path
{
1233 return $cdrom_path if defined($cdrom_path);
1235 $cdrom_path = first
{ -l
$_ } map { "/dev/cdrom$_" } ('', '1', '2');
1237 if (!defined($cdrom_path)) {
1238 log_warn
("no physical CD-ROM available, ignoring");
1246 my ($storecfg, $vmid, $cdrom) = @_;
1248 if ($cdrom eq 'cdrom') {
1249 return get_cdrom_path
();
1250 } elsif ($cdrom eq 'none') {
1252 } elsif ($cdrom =~ m
|^/|) {
1255 return PVE
::Storage
::path
($storecfg, $cdrom);
1259 # try to convert old style file names to volume IDs
1260 sub filename_to_volume_id
{
1261 my ($vmid, $file, $media) = @_;
1263 if (!($file eq 'none' || $file eq 'cdrom' ||
1264 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1266 return if $file =~ m
|/|;
1268 if ($media && $media eq 'cdrom') {
1269 $file = "local:iso/$file";
1271 $file = "local:$vmid/$file";
1278 sub verify_media_type
{
1279 my ($opt, $vtype, $media) = @_;
1284 if ($media eq 'disk') {
1286 } elsif ($media eq 'cdrom') {
1289 die "internal error";
1292 return if ($vtype eq $etype);
1294 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1297 sub cleanup_drive_path
{
1298 my ($opt, $storecfg, $drive) = @_;
1300 # try to convert filesystem paths to volume IDs
1302 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1303 ($drive->{file
} !~ m
|^/dev/.+|) &&
1304 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1305 ($drive->{file
} !~ m/^\d+$/)) {
1306 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1307 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1309 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1310 verify_media_type
($opt, $vtype, $drive->{media
});
1311 $drive->{file
} = $volid;
1314 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1317 sub parse_hotplug_features
{
1322 return $res if $data eq '0';
1324 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1326 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1327 if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1330 die "invalid hotplug feature '$feature'\n";
1336 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1337 sub pve_verify_hotplug_features
{
1338 my ($value, $noerr) = @_;
1340 return $value if parse_hotplug_features
($value);
1344 die "unable to parse hotplug option\n";
1347 sub assert_clipboard_config
{
1350 my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1354 && $vga->{'clipboard'} eq 'vnc'
1356 && $vga->{type
} !~ $clipboard_regex
1358 die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1362 sub print_tabletdevice_full
{
1363 my ($conf, $arch) = @_;
1365 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1367 # we use uhci for old VMs because tablet driver was buggy in older qemu
1369 if ($q35 || $arch eq 'aarch64') {
1375 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1378 sub print_keyboarddevice_full
{
1379 my ($conf, $arch) = @_;
1381 return if $arch ne 'aarch64';
1383 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1386 my sub get_drive_id
{
1388 return "$drive->{interface}$drive->{index}";
1391 sub print_drivedevice_full
{
1392 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1397 my $drive_id = get_drive_id
($drive);
1398 if ($drive->{interface
} eq 'virtio') {
1399 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1400 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1401 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1402 } elsif ($drive->{interface
} eq 'scsi') {
1404 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1405 my $unit = $drive->{index} % $maxdev;
1407 my $machine_version = extract_version
($machine_type, kvm_user_version
());
1408 my $device_type = PVE
::QemuServer
::Drive
::get_scsi_device_type
(
1409 $drive, $storecfg, $machine_version);
1411 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1412 $device = "scsi-$device_type,bus=$controller_prefix$controller.0,scsi-id=$unit";
1414 $device = "scsi-$device_type,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1415 .",lun=$drive->{index}";
1417 $device .= ",drive=drive-$drive_id,id=$drive_id";
1419 if ($drive->{ssd
} && ($device_type eq 'block' || $device_type eq 'hd')) {
1420 $device .= ",rotation_rate=1";
1422 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1424 # only scsi-hd and scsi-cd support passing vendor and product information
1425 if ($device_type eq 'hd' || $device_type eq 'cd') {
1426 if (my $vendor = $drive->{vendor
}) {
1427 $device .= ",vendor=$vendor";
1429 if (my $product = $drive->{product
}) {
1430 $device .= ",product=$product";
1434 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1435 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1436 my $controller = int($drive->{index} / $maxdev);
1437 my $unit = $drive->{index} % $maxdev;
1439 # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1440 # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1441 # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1442 # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1443 # were before. Move odd ones up by 2 where they don't clash.
1444 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) && $drive->{interface
} eq 'ide') {
1445 $controller += 2 * ($unit % 2);
1449 my $device_type = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1451 $device = "ide-$device_type";
1452 if ($drive->{interface
} eq 'ide') {
1453 $device .= ",bus=ide.$controller,unit=$unit";
1455 $device .= ",bus=ahci$controller.$unit";
1457 $device .= ",drive=drive-$drive_id,id=$drive_id";
1459 if ($device_type eq 'hd') {
1460 if (my $model = $drive->{model
}) {
1461 $model = URI
::Escape
::uri_unescape
($model);
1462 $device .= ",model=$model";
1464 if ($drive->{ssd
}) {
1465 $device .= ",rotation_rate=1";
1468 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1469 } elsif ($drive->{interface
} eq 'usb') {
1471 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1473 die "unsupported interface type";
1476 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1478 if (my $serial = $drive->{serial
}) {
1479 $serial = URI
::Escape
::uri_unescape
($serial);
1480 $device .= ",serial=$serial";
1487 sub get_initiator_name
{
1490 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1491 while (defined(my $line = <$fh>)) {
1492 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1501 my sub storage_allows_io_uring_default
{
1502 my ($scfg, $cache_direct) = @_;
1504 # io_uring with cache mode writeback or writethrough on krbd will hang...
1505 return if $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1507 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1508 # sometimes, just plain disable...
1509 return if $scfg && $scfg->{type
} eq 'lvm';
1511 # io_uring causes problems when used with CIFS since kernel 5.15
1512 # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1513 return if $scfg && $scfg->{type
} eq 'cifs';
1518 my sub drive_uses_cache_direct
{
1519 my ($drive, $scfg) = @_;
1521 my $cache_direct = 0;
1523 if (my $cache = $drive->{cache
}) {
1524 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1525 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1529 return $cache_direct;
1532 sub print_drive_commandline_full
{
1533 my ($storecfg, $vmid, $drive, $live_restore_name, $io_uring) = @_;
1536 my $volid = $drive->{file
};
1537 my $format = $drive->{format
};
1538 my $drive_id = get_drive_id
($drive);
1540 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1541 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1543 if (drive_is_cdrom
($drive)) {
1544 $path = get_iso_path
($storecfg, $vmid, $volid);
1545 die "$drive_id: cannot back cdrom drive with a live restore image\n" if $live_restore_name;
1548 $path = PVE
::Storage
::path
($storecfg, $volid);
1549 $format //= qemu_img_format
($scfg, $volname);
1556 my $is_rbd = $path =~ m/^rbd:/;
1559 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1560 foreach my $o (@qemu_drive_options) {
1561 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1564 # snapshot only accepts on|off
1565 if (defined($drive->{snapshot
})) {
1566 my $v = $drive->{snapshot
} ?
'on' : 'off';
1567 $opts .= ",snapshot=$v";
1570 if (defined($drive->{ro
})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1571 $opts .= ",readonly=" . ($drive->{ro
} ?
'on' : 'off');
1574 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1575 my ($dir, $qmpname) = @$type;
1576 if (my $v = $drive->{"mbps$dir"}) {
1577 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1579 if (my $v = $drive->{"mbps${dir}_max"}) {
1580 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1582 if (my $v = $drive->{"bps${dir}_max_length"}) {
1583 $opts .= ",throttling.bps$qmpname-max-length=$v";
1585 if (my $v = $drive->{"iops${dir}"}) {
1586 $opts .= ",throttling.iops$qmpname=$v";
1588 if (my $v = $drive->{"iops${dir}_max"}) {
1589 $opts .= ",throttling.iops$qmpname-max=$v";
1591 if (my $v = $drive->{"iops${dir}_max_length"}) {
1592 $opts .= ",throttling.iops$qmpname-max-length=$v";
1596 if ($live_restore_name) {
1597 $format = "rbd" if $is_rbd;
1598 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1600 $opts .= ",format=alloc-track,file.driver=$format";
1602 $opts .= ",format=$format";
1605 my $cache_direct = drive_uses_cache_direct
($drive, $scfg);
1607 $opts .= ",cache=none" if !$drive->{cache
} && $cache_direct;
1609 if (!$drive->{aio
}) {
1610 if ($io_uring && storage_allows_io_uring_default
($scfg, $cache_direct)) {
1611 # io_uring supports all cache modes
1612 $opts .= ",aio=io_uring";
1614 # aio native works only with O_DIRECT
1616 $opts .= ",aio=native";
1618 $opts .= ",aio=threads";
1623 if (!drive_is_cdrom
($drive)) {
1625 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1626 $detectzeroes = 'off';
1627 } elsif ($drive->{discard
}) {
1628 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1630 # This used to be our default with discard not being specified:
1631 $detectzeroes = 'on';
1634 # note: 'detect-zeroes' works per blockdev and we want it to persist
1635 # after the alloc-track is removed, so put it on 'file' directly
1636 my $dz_param = $live_restore_name ?
"file.detect-zeroes" : "detect-zeroes";
1637 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1640 if ($live_restore_name) {
1641 $opts .= ",backing=$live_restore_name";
1642 $opts .= ",auto-remove=on";
1645 # my $file_param = $live_restore_name ? "file.file.filename" : "file";
1646 my $file_param = "file";
1647 if ($live_restore_name) {
1648 # non-rbd drivers require the underlying file to be a seperate block
1649 # node, so add a second .file indirection
1650 $file_param .= ".file" if !$is_rbd;
1651 $file_param .= ".filename";
1653 my $pathinfo = $path ?
"$file_param=$path," : '';
1655 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1658 sub print_pbs_blockdev
{
1659 my ($pbs_conf, $pbs_name) = @_;
1660 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1661 $blockdev .= ",repository=$pbs_conf->{repository}";
1662 $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace
};
1663 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1664 $blockdev .= ",archive=$pbs_conf->{archive}";
1665 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1669 sub print_netdevice_full
{
1670 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1672 my $device = $net->{model
};
1673 if ($net->{model
} eq 'virtio') {
1674 $device = 'virtio-net-pci';
1677 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1678 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1679 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1680 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1681 # and out of each queue plus one config interrupt and control vector queue
1682 my $vectors = $net->{queues
} * 2 + 2;
1683 $tmpstr .= ",vectors=$vectors,mq=on";
1684 if (min_version
($machine_version, 7, 1)) {
1685 $tmpstr .= ",packed=on";
1689 if (min_version
($machine_version, 7, 1) && $net->{model
} eq 'virtio'){
1690 $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1693 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1695 if (my $mtu = $net->{mtu
}) {
1696 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1697 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1700 } elsif ($mtu < 576) {
1701 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1702 } elsif ($mtu > $bridge_mtu) {
1703 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1705 $tmpstr .= ",host_mtu=$mtu";
1707 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1711 if ($use_old_bios_files) {
1713 if ($device eq 'virtio-net-pci') {
1714 $romfile = 'pxe-virtio.rom';
1715 } elsif ($device eq 'e1000') {
1716 $romfile = 'pxe-e1000.rom';
1717 } elsif ($device eq 'e1000e') {
1718 $romfile = 'pxe-e1000e.rom';
1719 } elsif ($device eq 'ne2k') {
1720 $romfile = 'pxe-ne2k_pci.rom';
1721 } elsif ($device eq 'pcnet') {
1722 $romfile = 'pxe-pcnet.rom';
1723 } elsif ($device eq 'rtl8139') {
1724 $romfile = 'pxe-rtl8139.rom';
1726 $tmpstr .= ",romfile=$romfile" if $romfile;
1732 sub print_netdev_full
{
1733 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1736 if ($netid =~ m/^net(\d+)$/) {
1740 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1742 my $ifname = "tap${vmid}i$i";
1744 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1745 die "interface name '$ifname' is too long (max 15 character)\n"
1746 if length($ifname) >= 16;
1748 my $vhostparam = '';
1749 if (is_native_arch
($arch)) {
1750 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1753 my $vmname = $conf->{name
} || "vm$vmid";
1756 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1758 if ($net->{bridge
}) {
1759 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1760 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1762 $netdev = "type=user,id=$netid,hostname=$vmname";
1765 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1771 'cirrus' => 'cirrus-vga',
1773 'vmware' => 'vmware-svga',
1774 'virtio' => 'virtio-vga',
1775 'virtio-gl' => 'virtio-vga-gl',
1778 sub print_vga_device
{
1779 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1781 my $type = $vga_map->{$vga->{type
}};
1782 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1783 $type = 'virtio-gpu';
1785 my $vgamem_mb = $vga->{memory
};
1787 my $max_outputs = '';
1789 $type = $id ?
'qxl' : 'qxl-vga';
1791 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1792 # set max outputs so linux can have up to 4 qxl displays with one device
1793 if (min_version
($machine_version, 4, 1)) {
1794 $max_outputs = ",max_outputs=4";
1799 die "no device-type for $vga->{type}\n" if !$type;
1803 if ($vga->{type
} =~ /^virtio/) {
1804 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1805 $memory = ",max_hostmem=$bytes";
1807 # from https://www.spice-space.org/multiple-monitors.html
1808 $memory = ",vgamem_mb=$vga->{memory}";
1809 my $ram = $vgamem_mb * 4;
1810 my $vram = $vgamem_mb * 2;
1811 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1813 $memory = ",vgamem_mb=$vga->{memory}";
1815 } elsif ($qxlnum && $id) {
1816 $memory = ",ram_size=67108864,vram_size=33554432";
1820 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1821 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1824 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1825 my $vgaid = "vga" . ($id // '');
1827 if ($q35 && $vgaid eq 'vga') {
1828 # the first display uses pcie.0 bus on q35 machines
1829 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1831 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1834 if ($vga->{type
} eq 'virtio-gl') {
1835 my $base = '/usr/lib/x86_64-linux-gnu/lib';
1836 die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1837 if !-e
"${base}EGL.so.1" || !-e
"${base}GL.so.1";
1839 die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1840 if !PVE
::Tools
::dir_glob_regex
('/dev/dri/', "renderD.*");
1843 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1846 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1848 my ($data, $disable_mac_autogen) = @_;
1850 my $res = eval { parse_property_string
($net_fmt, $data) };
1855 if (!defined($res->{macaddr
}) && !$disable_mac_autogen) {
1856 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1857 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1862 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1863 sub parse_ipconfig
{
1866 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1872 if ($res->{gw
} && !$res->{ip
}) {
1873 warn 'gateway specified without specifying an IP address';
1876 if ($res->{gw6
} && !$res->{ip6
}) {
1877 warn 'IPv6 gateway specified without specifying an IPv6 address';
1880 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1881 warn 'gateway specified together with DHCP';
1884 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1886 warn "IPv6 gateway specified together with $res->{ip6} address";
1890 if (!$res->{ip
} && !$res->{ip6
}) {
1891 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1900 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1903 sub add_random_macs
{
1904 my ($settings) = @_;
1906 foreach my $opt (keys %$settings) {
1907 next if $opt !~ m/^net(\d+)$/;
1908 my $net = parse_net
($settings->{$opt});
1910 $settings->{$opt} = print_net
($net);
1914 sub vm_is_volid_owner
{
1915 my ($storecfg, $vmid, $volid) = @_;
1917 if ($volid !~ m
|^/|) {
1919 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1920 if ($owner && ($owner == $vmid)) {
1928 sub vmconfig_register_unused_drive
{
1929 my ($storecfg, $vmid, $conf, $drive) = @_;
1931 if (drive_is_cloudinit
($drive)) {
1932 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1934 delete $conf->{cloudinit
};
1935 } elsif (!drive_is_cdrom
($drive)) {
1936 my $volid = $drive->{file
};
1937 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1938 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1943 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1947 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1948 format_description
=> 'UUID',
1949 description
=> "Set SMBIOS1 UUID.",
1954 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1955 format_description
=> 'Base64 encoded string',
1956 description
=> "Set SMBIOS1 version.",
1961 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1962 format_description
=> 'Base64 encoded string',
1963 description
=> "Set SMBIOS1 serial number.",
1968 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1969 format_description
=> 'Base64 encoded string',
1970 description
=> "Set SMBIOS1 manufacturer.",
1975 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1976 format_description
=> 'Base64 encoded string',
1977 description
=> "Set SMBIOS1 product ID.",
1982 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1983 format_description
=> 'Base64 encoded string',
1984 description
=> "Set SMBIOS1 SKU string.",
1989 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1990 format_description
=> 'Base64 encoded string',
1991 description
=> "Set SMBIOS1 family string.",
1996 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2004 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2011 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2014 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2016 sub parse_watchdog
{
2021 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2026 sub parse_guest_agent
{
2029 return {} if !defined($conf->{agent
});
2031 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2034 # if the agent is disabled ignore the other potentially set properties
2035 return {} if !$res->{enabled
};
2040 my ($conf, $key) = @_;
2041 return undef if !defined($conf->{agent
});
2043 my $agent = parse_guest_agent
($conf);
2044 return $agent->{$key};
2050 return {} if !$value;
2051 my $res = eval { parse_property_string
($vga_fmt, $value) };
2061 my $res = eval { parse_property_string
($rng_fmt, $value) };
2066 sub parse_meta_info
{
2071 my $res = eval { parse_property_string
($meta_info_fmt, $value) };
2076 sub new_meta_info_string
{
2077 my () = @_; # for now do not allow to override any value
2079 return PVE
::JSONSchema
::print_property_string
(
2081 'creation-qemu' => kvm_user_version
(),
2082 ctime
=> "". int(time()),
2088 sub qemu_created_version_fixups
{
2089 my ($conf, $forcemachine, $kvmver) = @_;
2091 my $meta = parse_meta_info
($conf->{meta
}) // {};
2092 my $forced_vers = PVE
::QemuServer
::Machine
::extract_version
($forcemachine);
2094 # check if we need to apply some handling for VMs that always use the latest machine version but
2095 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2096 # would be required (we do not want to pin machine version for non-windows OS type)
2097 my $machine_conf = PVE
::QemuServer
::Machine
::parse_machine
($conf->{machine
});
2099 (!defined($machine_conf->{type
}) || $machine_conf->{type
} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2100 && (!defined($meta->{'creation-qemu'}) || !min_version
($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2101 && (!$forced_vers || min_version
($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2102 && min_version
($kvmver, 6, 1) # only need to apply the change since 6.1
2104 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2105 if ($q35 && $conf->{ostype
} && $conf->{ostype
} eq 'l26') {
2106 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2107 # and thus with the predictable interface naming of systemd
2108 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2114 # add JSON properties for create and set function
2115 sub json_config_properties
{
2116 my ($prop, $with_disk_alloc) = @_;
2118 my $skip_json_config_opts = {
2122 runningmachine
=> 1,
2127 foreach my $opt (keys %$confdesc) {
2128 next if $skip_json_config_opts->{$opt};
2130 if ($with_disk_alloc && is_valid_drivename
($opt)) {
2131 $prop->{$opt} = $PVE::QemuServer
::Drive
::drivedesc_hash_with_alloc-
>{$opt};
2133 $prop->{$opt} = $confdesc->{$opt};
2140 # Properties that we can read from an OVF file
2141 sub json_ovf_properties
{
2144 for my $device (PVE
::QemuServer
::Drive
::valid_drive_names
()) {
2145 $prop->{$device} = {
2147 format
=> 'pve-volume-id-or-absolute-path',
2148 description
=> "Disk image that gets imported to $device",
2155 description
=> "The number of CPU cores.",
2160 description
=> "Amount of RAM for the VM in MB.",
2165 description
=> "Name of the VM.",
2172 # return copy of $confdesc_cloudinit to generate documentation
2173 sub cloudinit_config_properties
{
2175 return dclone
($confdesc_cloudinit);
2178 sub cloudinit_pending_properties
{
2180 map { $_ => 1 } keys $confdesc_cloudinit->%*,
2183 $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2188 my ($key, $value) = @_;
2190 die "unknown setting '$key'\n" if !$confdesc->{$key};
2192 my $type = $confdesc->{$key}->{type
};
2194 if (!defined($value)) {
2195 die "got undefined value\n";
2198 if ($value =~ m/[\n\r]/) {
2199 die "property contains a line feed\n";
2202 if ($type eq 'boolean') {
2203 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2204 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2205 die "type check ('boolean') failed - got '$value'\n";
2206 } elsif ($type eq 'integer') {
2207 return int($1) if $value =~ m/^(\d+)$/;
2208 die "type check ('integer') failed - got '$value'\n";
2209 } elsif ($type eq 'number') {
2210 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2211 die "type check ('number') failed - got '$value'\n";
2212 } elsif ($type eq 'string') {
2213 if (my $fmt = $confdesc->{$key}->{format
}) {
2214 PVE
::JSONSchema
::check_format
($fmt, $value);
2217 $value =~ s/^\"(.*)\"$/$1/;
2220 die "internal error"
2225 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2227 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2229 if (!$skiplock && !PVE
::QemuConfig-
>has_lock($conf, 'suspended')) {
2230 PVE
::QemuConfig-
>check_lock($conf);
2233 if ($conf->{template
}) {
2234 # check if any base image is still used by a linked clone
2235 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2236 my ($ds, $drive) = @_;
2237 return if drive_is_cdrom
($drive);
2239 my $volid = $drive->{file
};
2240 return if !$volid || $volid =~ m
|^/|;
2242 die "base volume '$volid' is still in use by linked cloned\n"
2243 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2249 my $remove_owned_drive = sub {
2250 my ($ds, $drive) = @_;
2251 return if drive_is_cdrom
($drive, 1);
2253 my $volid = $drive->{file
};
2254 return if !$volid || $volid =~ m
|^/|;
2255 return if $volids->{$volid};
2257 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2258 return if !$path || !$owner || ($owner != $vmid);
2260 $volids->{$volid} = 1;
2261 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2262 warn "Could not remove disk '$volid', check manually: $@" if $@;
2265 # only remove disks owned by this VM (referenced in the config)
2266 my $include_opts = {
2267 include_unused
=> 1,
2268 extra_keys
=> ['vmstate'],
2270 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2272 for my $snap (values %{$conf->{snapshots
}}) {
2273 next if !defined($snap->{vmstate
});
2274 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2275 next if !defined($drive);
2276 $remove_owned_drive->('vmstate', $drive);
2279 PVE
::QemuConfig-
>foreach_volume_full($conf->{pending
}, $include_opts, $remove_owned_drive);
2281 if ($purge_unreferenced) { # also remove unreferenced disk
2282 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2283 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2284 my ($volid, $sid, $volname, $d) = @_;
2285 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2290 eval { delete_ifaces_ipams_ips
($conf, $vmid)};
2293 if (defined $replacement_conf) {
2294 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2296 PVE
::QemuConfig-
>destroy_config($vmid);
2300 sub parse_vm_config
{
2301 my ($filename, $raw, $strict) = @_;
2303 return if !defined($raw);
2306 digest
=> Digest
::SHA
::sha1_hex
($raw),
2312 my $handle_error = sub {
2322 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2323 || die "got strange filename '$filename'";
2329 my $finish_description = sub {
2330 if (defined($descr)) {
2332 $conf->{description
} = $descr;
2338 my @lines = split(/\n/, $raw);
2339 foreach my $line (@lines) {
2340 next if $line =~ m/^\s*$/;
2342 if ($line =~ m/^\[PENDING\]\s*$/i) {
2343 $section = 'pending';
2344 $finish_description->();
2345 $conf = $res->{$section} = {};
2347 } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2348 $section = 'cloudinit';
2349 $finish_description->();
2350 $conf = $res->{$section} = {};
2353 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2355 $finish_description->();
2356 $conf = $res->{snapshots
}->{$section} = {};
2360 if ($line =~ m/^\#(.*)$/) {
2361 $descr = '' if !defined($descr);
2362 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2366 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2367 $descr = '' if !defined($descr);
2368 $descr .= PVE
::Tools
::decode_text
($2);
2369 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2370 $conf->{snapstate
} = $1;
2371 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2374 $conf->{$key} = $value;
2375 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2377 if ($section eq 'pending') {
2378 $conf->{delete} = $value; # we parse this later
2380 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2382 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2385 if ($section eq 'cloudinit') {
2386 # ignore validation only used for informative purpose
2387 $conf->{$key} = $value;
2390 eval { $value = check_type
($key, $value); };
2392 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2394 $key = 'ide2' if $key eq 'cdrom';
2395 my $fmt = $confdesc->{$key}->{format
};
2396 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2397 my $v = parse_drive
($key, $value);
2398 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2399 $v->{file
} = $volid;
2400 $value = print_drive
($v);
2402 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2407 $conf->{$key} = $value;
2410 $handle_error->("vm $vmid - unable to parse config: $line\n");
2414 $finish_description->();
2415 delete $res->{snapstate
}; # just to be sure
2420 sub write_vm_config
{
2421 my ($filename, $conf) = @_;
2423 delete $conf->{snapstate
}; # just to be sure
2425 if ($conf->{cdrom
}) {
2426 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2427 $conf->{ide2
} = $conf->{cdrom
};
2428 delete $conf->{cdrom
};
2431 # we do not use 'smp' any longer
2432 if ($conf->{sockets
}) {
2433 delete $conf->{smp
};
2434 } elsif ($conf->{smp
}) {
2435 $conf->{sockets
} = $conf->{smp
};
2436 delete $conf->{cores
};
2437 delete $conf->{smp
};
2440 my $used_volids = {};
2442 my $cleanup_config = sub {
2443 my ($cref, $pending, $snapname) = @_;
2445 foreach my $key (keys %$cref) {
2446 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2447 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2448 my $value = $cref->{$key};
2449 if ($key eq 'delete') {
2450 die "propertry 'delete' is only allowed in [PENDING]\n"
2452 # fixme: check syntax?
2455 eval { $value = check_type
($key, $value); };
2456 die "unable to parse value of '$key' - $@" if $@;
2458 $cref->{$key} = $value;
2460 if (!$snapname && is_valid_drivename
($key)) {
2461 my $drive = parse_drive
($key, $value);
2462 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2467 &$cleanup_config($conf);
2469 &$cleanup_config($conf->{pending
}, 1);
2471 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2472 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2473 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2476 # remove 'unusedX' settings if we re-add a volume
2477 foreach my $key (keys %$conf) {
2478 my $value = $conf->{$key};
2479 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2480 delete $conf->{$key};
2484 my $generate_raw_config = sub {
2485 my ($conf, $pending) = @_;
2489 # add description as comment to top of file
2490 if (defined(my $descr = $conf->{description
})) {
2492 foreach my $cl (split(/\n/, $descr)) {
2493 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2496 $raw .= "#\n" if $pending;
2500 foreach my $key (sort keys %$conf) {
2501 next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2502 $raw .= "$key: $conf->{$key}\n";
2507 my $raw = &$generate_raw_config($conf);
2509 if (scalar(keys %{$conf->{pending
}})){
2510 $raw .= "\n[PENDING]\n";
2511 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2514 if (scalar(keys %{$conf->{cloudinit
}}) && PVE
::QemuConfig-
>has_cloudinit($conf)){
2515 $raw .= "\n[special:cloudinit]\n";
2516 $raw .= &$generate_raw_config($conf->{cloudinit
});
2519 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2520 $raw .= "\n[$snapname]\n";
2521 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2531 # we use static defaults from our JSON schema configuration
2532 foreach my $key (keys %$confdesc) {
2533 if (defined(my $default = $confdesc->{$key}->{default})) {
2534 $res->{$key} = $default;
2542 my $vmlist = PVE
::Cluster
::get_vmlist
();
2544 return $res if !$vmlist || !$vmlist->{ids
};
2545 my $ids = $vmlist->{ids
};
2546 my $nodename = nodename
();
2548 foreach my $vmid (keys %$ids) {
2549 my $d = $ids->{$vmid};
2550 next if !$d->{node
} || $d->{node
} ne $nodename;
2551 next if !$d->{type
} || $d->{type
} ne 'qemu';
2552 $res->{$vmid}->{exists} = 1;
2557 # test if VM uses local resources (to prevent migration)
2558 sub check_local_resources
{
2559 my ($conf, $noerr) = @_;
2562 my $mapped_res = [];
2564 my $nodelist = PVE
::Cluster
::get_nodelist
();
2565 my $pci_map = PVE
::Mapping
::PCI
::config
();
2566 my $usb_map = PVE
::Mapping
::USB
::config
();
2568 my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2570 my $add_missing_mapping = sub {
2571 my ($type, $key, $id) = @_;
2572 for my $node (@$nodelist) {
2574 if ($type eq 'pci') {
2575 $entry = PVE
::Mapping
::PCI
::get_node_mapping
($pci_map, $id, $node);
2576 } elsif ($type eq 'usb') {
2577 $entry = PVE
::Mapping
::USB
::get_node_mapping
($usb_map, $id, $node);
2579 if (!scalar($entry->@*)) {
2580 push @{$missing_mappings_by_node->{$node}}, $key;
2585 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2586 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2588 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2590 foreach my $k (keys %$conf) {
2591 if ($k =~ m/^usb/) {
2592 my $entry = parse_property_string
('pve-qm-usb', $conf->{$k});
2593 next if $entry->{host
} && $entry->{host
} =~ m/^spice$/i;
2594 if ($entry->{mapping
}) {
2595 $add_missing_mapping->('usb', $k, $entry->{mapping
});
2596 push @$mapped_res, $k;
2599 if ($k =~ m/^hostpci/) {
2600 my $entry = parse_property_string
('pve-qm-hostpci', $conf->{$k});
2601 if ($entry->{mapping
}) {
2602 $add_missing_mapping->('pci', $k, $entry->{mapping
});
2603 push @$mapped_res, $k;
2606 # sockets are safe: they will recreated be on the target side post-migrate
2607 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2608 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2611 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2613 return wantarray ?
(\
@loc_res, $mapped_res, $missing_mappings_by_node) : \
@loc_res;
2616 # check if used storages are available on all nodes (use by migrate)
2617 sub check_storage_availability
{
2618 my ($storecfg, $conf, $node) = @_;
2620 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2621 my ($ds, $drive) = @_;
2623 my $volid = $drive->{file
};
2626 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2629 # check if storage is available on both nodes
2630 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2631 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2633 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2635 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2636 if !$scfg->{content
}->{$vtype};
2640 # list nodes where all VM images are available (used by has_feature API)
2642 my ($conf, $storecfg) = @_;
2644 my $nodelist = PVE
::Cluster
::get_nodelist
();
2645 my $nodehash = { map { $_ => 1 } @$nodelist };
2646 my $nodename = nodename
();
2648 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2649 my ($ds, $drive) = @_;
2651 my $volid = $drive->{file
};
2654 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2656 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2657 if ($scfg->{disable
}) {
2659 } elsif (my $avail = $scfg->{nodes
}) {
2660 foreach my $node (keys %$nodehash) {
2661 delete $nodehash->{$node} if !$avail->{$node};
2663 } elsif (!$scfg->{shared
}) {
2664 foreach my $node (keys %$nodehash) {
2665 delete $nodehash->{$node} if $node ne $nodename
2674 sub check_local_storage_availability
{
2675 my ($conf, $storecfg) = @_;
2677 my $nodelist = PVE
::Cluster
::get_nodelist
();
2678 my $nodehash = { map { $_ => {} } @$nodelist };
2680 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2681 my ($ds, $drive) = @_;
2683 my $volid = $drive->{file
};
2686 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2688 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2690 if ($scfg->{disable
}) {
2691 foreach my $node (keys %$nodehash) {
2692 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2694 } elsif (my $avail = $scfg->{nodes
}) {
2695 foreach my $node (keys %$nodehash) {
2696 if (!$avail->{$node}) {
2697 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2704 foreach my $node (values %$nodehash) {
2705 if (my $unavail = $node->{unavailable_storages
}) {
2706 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2713 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2715 my ($vmid, $nocheck, $node) = @_;
2717 # $nocheck is set when called during a migration, in which case the config
2718 # file might still or already reside on the *other* node
2719 # - because rename has already happened, and current node is source
2720 # - because rename hasn't happened yet, and current node is target
2721 # - because rename has happened, current node is target, but hasn't yet
2723 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2724 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2729 my $vzlist = config_list
();
2731 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2733 while (defined(my $de = $fd->read)) {
2734 next if $de !~ m/^(\d+)\.pid$/;
2736 next if !defined($vzlist->{$vmid});
2737 if (my $pid = check_running
($vmid)) {
2738 $vzlist->{$vmid}->{pid
} = $pid;
2745 our $vmstatus_return_properties = {
2746 vmid
=> get_standard_option
('pve-vmid'),
2748 description
=> "QEMU process status.",
2750 enum
=> ['stopped', 'running'],
2753 description
=> "Maximum memory in bytes.",
2756 renderer
=> 'bytes',
2759 description
=> "Root disk size in bytes.",
2762 renderer
=> 'bytes',
2765 description
=> "VM name.",
2770 description
=> "VM run state from the 'query-status' QMP monitor command.",
2775 description
=> "PID of running qemu process.",
2780 description
=> "Uptime.",
2783 renderer
=> 'duration',
2786 description
=> "Maximum usable CPUs.",
2791 description
=> "The current config lock, if any.",
2796 description
=> "The current configured tags, if any",
2800 'running-machine' => {
2801 description
=> "The currently running machine type (if running).",
2806 description
=> "The currently running QEMU version (if running).",
2812 my $last_proc_pid_stat;
2814 # get VM status information
2815 # This must be fast and should not block ($full == false)
2816 # We only query KVM using QMP if $full == true (this can be slow)
2818 my ($opt_vmid, $full) = @_;
2822 my $storecfg = PVE
::Storage
::config
();
2824 my $list = vzlist
();
2825 my $defaults = load_defaults
();
2827 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2829 my $cpucount = $cpuinfo->{cpus
} || 1;
2831 foreach my $vmid (keys %$list) {
2832 next if $opt_vmid && ($vmid ne $opt_vmid);
2834 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2836 my $d = { vmid
=> int($vmid) };
2837 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2839 # fixme: better status?
2840 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2842 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2843 if (defined($size)) {
2844 $d->{disk
} = 0; # no info available
2845 $d->{maxdisk
} = $size;
2851 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2852 * ($conf->{cores
} || $defaults->{cores
});
2853 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2854 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2856 $d->{name
} = $conf->{name
} || "VM $vmid";
2857 $d->{maxmem
} = get_current_memory
($conf->{memory
})*(1024*1024);
2859 if ($conf->{balloon
}) {
2860 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2861 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2862 : $defaults->{shares
};
2873 $d->{diskwrite
} = 0;
2875 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2877 $d->{serial
} = 1 if conf_has_serial
($conf);
2878 $d->{lock} = $conf->{lock} if $conf->{lock};
2879 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2884 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2885 foreach my $dev (keys %$netdev) {
2886 next if $dev !~ m/^tap([1-9]\d*)i/;
2888 my $d = $res->{$vmid};
2891 $d->{netout
} += $netdev->{$dev}->{receive
};
2892 $d->{netin
} += $netdev->{$dev}->{transmit
};
2895 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2896 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2901 my $ctime = gettimeofday
;
2903 foreach my $vmid (keys %$list) {
2905 my $d = $res->{$vmid};
2906 my $pid = $d->{pid
};
2909 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2910 next if !$pstat; # not running
2912 my $used = $pstat->{utime} + $pstat->{stime
};
2914 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2916 if ($pstat->{vsize
}) {
2917 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2920 my $old = $last_proc_pid_stat->{$pid};
2922 $last_proc_pid_stat->{$pid} = {
2930 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2932 if ($dtime > 1000) {
2933 my $dutime = $used - $old->{used
};
2935 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2936 $last_proc_pid_stat->{$pid} = {
2942 $d->{cpu
} = $old->{cpu
};
2946 return $res if !$full;
2948 my $qmpclient = PVE
::QMPClient-
>new();
2950 my $ballooncb = sub {
2951 my ($vmid, $resp) = @_;
2953 my $info = $resp->{'return'};
2954 return if !$info->{max_mem
};
2956 my $d = $res->{$vmid};
2958 # use memory assigned to VM
2959 $d->{maxmem
} = $info->{max_mem
};
2960 $d->{balloon
} = $info->{actual
};
2962 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2963 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2964 $d->{freemem
} = $info->{free_mem
};
2967 $d->{ballooninfo
} = $info;
2970 my $blockstatscb = sub {
2971 my ($vmid, $resp) = @_;
2972 my $data = $resp->{'return'} || [];
2973 my $totalrdbytes = 0;
2974 my $totalwrbytes = 0;
2976 for my $blockstat (@$data) {
2977 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2978 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2980 $blockstat->{device
} =~ s/drive-//;
2981 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2983 $res->{$vmid}->{diskread
} = $totalrdbytes;
2984 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2987 my $machinecb = sub {
2988 my ($vmid, $resp) = @_;
2989 my $data = $resp->{'return'} || [];
2991 $res->{$vmid}->{'running-machine'} =
2992 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
2995 my $versioncb = sub {
2996 my ($vmid, $resp) = @_;
2997 my $data = $resp->{'return'} // {};
2998 my $version = 'unknown';
3000 if (my $v = $data->{qemu
}) {
3001 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
3004 $res->{$vmid}->{'running-qemu'} = $version;
3007 my $statuscb = sub {
3008 my ($vmid, $resp) = @_;
3010 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3011 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3012 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3013 # this fails if ballon driver is not loaded, so this must be
3014 # the last commnand (following command are aborted if this fails).
3015 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3017 my $status = 'unknown';
3018 if (!defined($status = $resp->{'return'}->{status
})) {
3019 warn "unable to get VM status\n";
3023 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3026 foreach my $vmid (keys %$list) {
3027 next if $opt_vmid && ($vmid ne $opt_vmid);
3028 next if !$res->{$vmid}->{pid
}; # not running
3029 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3032 $qmpclient->queue_execute(undef, 2);
3034 foreach my $vmid (keys %$list) {
3035 next if $opt_vmid && ($vmid ne $opt_vmid);
3036 next if !$res->{$vmid}->{pid
}; #not running
3038 # we can't use the $qmpclient since it might have already aborted on
3039 # 'query-balloon', but this might also fail for older versions...
3040 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
3041 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3044 foreach my $vmid (keys %$list) {
3045 next if $opt_vmid && ($vmid ne $opt_vmid);
3046 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3052 sub conf_has_serial
{
3055 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3056 if ($conf->{"serial$i"}) {
3064 sub conf_has_audio
{
3065 my ($conf, $id) = @_;
3068 my $audio = $conf->{"audio$id"};
3069 return if !defined($audio);
3071 my $audioproperties = parse_property_string
($audio_fmt, $audio);
3072 my $audiodriver = $audioproperties->{driver
} // 'spice';
3075 dev
=> $audioproperties->{device
},
3076 dev_id
=> "audiodev$id",
3077 backend
=> $audiodriver,
3078 backend_id
=> "$audiodriver-backend${id}",
3083 my ($audio, $audiopciaddr, $machine_version) = @_;
3087 my $id = $audio->{dev_id
};
3089 if (min_version
($machine_version, 4, 2)) {
3090 $audiodev = ",audiodev=$audio->{backend_id}";
3093 if ($audio->{dev
} eq 'AC97') {
3094 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3095 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3096 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3097 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3098 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3100 die "unkown audio device '$audio->{dev}', implement me!";
3103 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3111 socket => "/var/run/qemu-server/$vmid.swtpm",
3112 pid
=> "/var/run/qemu-server/$vmid.swtpm.pid",
3116 sub add_tpm_device
{
3117 my ($vmid, $devices, $conf) = @_;
3119 return if !$conf->{tpmstate0
};
3121 my $paths = get_tpm_paths
($vmid);
3123 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3124 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3125 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3129 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3131 return if !$tpmdrive;
3134 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
3135 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
3137 $state = PVE
::Storage
::map_volume
($storecfg, $tpm->{file
});
3139 $state = $tpm->{file
};
3142 my $paths = get_tpm_paths
($vmid);
3144 # during migration, we will get state from remote
3147 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3154 "--create-platform-cert",
3157 "/etc/swtpm_setup.conf", # do not use XDG configs
3159 "0", # force creation as root, error if not possible
3160 "--not-overwrite", # ignore existing state, do not modify
3163 push @$setup_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3164 # TPM 2.0 supports ECC crypto, use if possible
3165 push @$setup_cmd, "--ecc" if $tpm->{version
} eq 'v2.0';
3167 run_command
($setup_cmd, outfunc
=> sub {
3168 print "swtpm_setup: $1\n";
3172 # Used to distinguish different invocations in the log.
3173 my $log_prefix = "[id=" . int(time()) . "] ";
3175 my $emulator_cmd = [
3179 "backend-uri=file://$state,mode=0600",
3181 "type=unixio,path=$paths->{socket},mode=0600",
3183 "file=$paths->{pid}",
3184 "--terminate", # terminate on QEMU disconnect
3187 "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3189 push @$emulator_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3190 run_command
($emulator_cmd, outfunc
=> sub { print $1; });
3192 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3193 while (! -e
$paths->{pid
}) {
3194 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3198 # return untainted PID of swtpm daemon so it can be killed on error
3199 file_read_firstline
($paths->{pid
}) =~ m/(\d+)/;
3203 sub vga_conf_has_spice
{
3206 my $vgaconf = parse_vga
($vga);
3207 my $vgatype = $vgaconf->{type
};
3208 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3215 return $conf->{arch
} // get_host_arch
();
3218 my $default_machines = {
3223 sub get_installed_machine_version
{
3224 my ($kvmversion) = @_;
3225 $kvmversion = kvm_user_version
() if !defined($kvmversion);
3226 $kvmversion =~ m/^(\d+\.\d+)/;
3230 sub windows_get_pinned_machine_version
{
3231 my ($machine, $base_version, $kvmversion) = @_;
3233 my $pin_version = $base_version;
3234 if (!defined($base_version) ||
3235 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3237 $pin_version = get_installed_machine_version
($kvmversion);
3239 if (!$machine || $machine eq 'pc') {
3240 $machine = "pc-i440fx-$pin_version";
3241 } elsif ($machine eq 'q35') {
3242 $machine = "pc-q35-$pin_version";
3243 } elsif ($machine eq 'virt') {
3244 $machine = "virt-$pin_version";
3246 warn "unknown machine type '$machine', not touching that!\n";
3252 sub get_vm_machine
{
3253 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3255 my $machine_conf = PVE
::QemuServer
::Machine
::parse_machine
($conf->{machine
});
3256 my $machine = $forcemachine || $machine_conf->{type
};
3258 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3259 $kvmversion //= kvm_user_version
();
3260 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3261 # layout which confuses windows quite a bit and may result in various regressions..
3262 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3263 if (windows_version
($conf->{ostype
})) {
3264 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3267 $machine ||= $default_machines->{$arch};
3268 if ($add_pve_version) {
3269 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3270 $machine .= "+pve$pvever";
3274 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3275 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3276 $machine = $1 if $is_pxe;
3278 # for version-pinned machines that do not include a pve-version (e.g.
3279 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3280 $machine .= '+pve0';
3282 $machine .= '.pxe' if $is_pxe;
3288 sub get_ovmf_files
($$$) {
3289 my ($arch, $efidisk, $smm) = @_;
3291 my $types = $OVMF->{$arch}
3292 or die "no OVMF images known for architecture '$arch'\n";
3294 my $type = 'default';
3295 if ($arch eq 'x86_64') {
3296 if (defined($efidisk->{efitype
}) && $efidisk->{efitype
} eq '4m') {
3297 $type = $smm ?
"4m" : "4m-no-smm";
3298 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3300 # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3304 my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3305 die "EFI base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3306 die "EFI vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
3308 return ($ovmf_code, $ovmf_vars);
3312 aarch64
=> '/usr/bin/qemu-system-aarch64',
3313 x86_64
=> '/usr/bin/qemu-system-x86_64',
3315 sub get_command_for_arch
($) {
3317 return '/usr/bin/kvm' if is_native_arch
($arch);
3319 my $cmd = $Arch2Qemu->{$arch}
3320 or die "don't know how to emulate architecture '$arch'\n";
3324 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3325 # to use in a QEMU command line (-cpu element), first array_intersect the result
3326 # of query_supported_ with query_understood_. This is necessary because:
3328 # a) query_understood_ returns flags the host cannot use and
3329 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3330 # flags, but CPU settings - with most of them being flags. Those settings
3331 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3333 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3334 # expensive. If you need the value returned from this, you can get it much
3335 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3336 # $accel being 'kvm' or 'tcg'.
3338 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3339 # changes, automatically populating pmxcfs.
3341 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3342 # since kvm and tcg machines support different flags
3344 sub query_supported_cpu_flags
{
3347 $arch //= get_host_arch
();
3348 my $default_machine = $default_machines->{$arch};
3352 # FIXME: Once this is merged, the code below should work for ARM as well:
3353 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3354 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3357 my $kvm_supported = defined(kvm_version
());
3358 my $qemu_cmd = get_command_for_arch
($arch);
3360 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3362 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3363 my $query_supported_run_qemu = sub {
3369 '-machine', $default_machine,
3371 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3372 '-mon', 'chardev=qmp,mode=control',
3373 '-pidfile', $pidfile,
3378 push @$cmd, '-accel', 'tcg';
3381 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3382 die "QEMU flag querying VM exited with code " . $rc if $rc;
3385 my $cmd_result = mon_cmd
(
3387 'query-cpu-model-expansion',
3389 model
=> { name
=> 'host' }
3392 my $props = $cmd_result->{model
}->{props
};
3393 foreach my $prop (keys %$props) {
3394 next if $props->{$prop} ne '1';
3395 # QEMU returns some flags multiple times, with '_', '.' or '-'
3396 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3397 # We only keep those with underscores, to match /proc/cpuinfo
3398 $prop =~ s/\.|-/_/g;
3399 $flags->{$prop} = 1;
3404 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3405 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3409 return [ sort keys %$flags ];
3412 # We need to query QEMU twice, since KVM and TCG have different supported flags
3413 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3414 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3415 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3417 if ($kvm_supported) {
3418 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3419 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3426 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3427 my $understood_cpu_flag_dir = "/usr/share/kvm";
3428 sub query_understood_cpu_flags
{
3429 my $arch = get_host_arch
();
3430 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3432 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3435 my $raw = file_get_contents
($filepath);
3436 $raw =~ s/^\s+|\s+$//g;
3437 my @flags = split(/\s+/, $raw);
3442 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3443 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3444 my sub should_disable_smm
{
3445 my ($conf, $vga, $machine) = @_;
3447 return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3449 return (!defined($conf->{bios
}) || $conf->{bios
} eq 'seabios') &&
3450 $vga->{type
} && $vga->{type
} =~ m/^(serial\d+|none)$/;
3453 my sub print_ovmf_drive_commandlines
{
3454 my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3456 my $d = $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
3458 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch, $d, $q35);
3460 my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3462 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3463 my ($path, $format) = $d->@{'file', 'format'};
3465 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3466 if (!defined($format)) {
3467 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3468 $format = qemu_img_format
($scfg, $volname);
3470 } elsif (!defined($format)) {
3471 die "efidisk format must be specified\n";
3473 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3474 if ($path =~ m/^rbd:/) {
3475 $var_drive_str .= ',cache=writeback';
3476 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3478 $var_drive_str .= ",format=$format,file=$path";
3480 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3481 $var_drive_str .= ',readonly=on' if drive_is_read_only
($conf, $d);
3483 log_warn
("no efidisk configured! Using temporary efivars disk.");
3484 my $path = "/tmp/$vmid-ovmf.fd";
3485 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3486 $var_drive_str .= ",format=raw,file=$path";
3487 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $version_guard->(4, 1, 2);
3490 return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3493 sub config_to_command
{
3494 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3495 $live_restore_backing) = @_;
3497 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3500 my $ostype = $conf->{ostype
};
3501 my $winversion = windows_version
($ostype);
3502 my $kvm = $conf->{kvm
};
3503 my $nodename = nodename
();
3505 my $machine_conf = PVE
::QemuServer
::Machine
::parse_machine
($conf->{machine
});
3507 my $arch = get_vm_arch
($conf);
3508 my $kvm_binary = get_command_for_arch
($arch);
3509 my $kvmver = kvm_user_version
($kvm_binary);
3511 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3512 $kvmver //= "undefined";
3513 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3516 my $add_pve_version = min_version
($kvmver, 4, 1);
3518 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3519 my $machine_version = extract_version
($machine_type, $kvmver);
3520 $kvm //= 1 if is_native_arch
($arch);
3522 $machine_version =~ m/(\d+)\.(\d+)/;
3523 my ($machine_major, $machine_minor) = ($1, $2);
3525 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3526 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3527 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3528 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3529 ." please upgrade node '$nodename'\n"
3530 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3531 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3532 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3533 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3534 ." node '$nodename'\n";
3537 # if a specific +pve version is required for a feature, use $version_guard
3538 # instead of min_version to allow machines to be run with the minimum
3540 my $required_pve_version = 0;
3541 my $version_guard = sub {
3542 my ($major, $minor, $pve) = @_;
3543 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3544 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3545 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3546 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3550 if ($kvm && !defined kvm_version
()) {
3551 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3552 ." or enable in BIOS.\n";
3555 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3556 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3557 my $use_old_bios_files = undef;
3558 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3561 if ($conf->{affinity
}) {
3562 push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity
};
3565 push @$cmd, $kvm_binary;
3567 push @$cmd, '-id', $vmid;
3569 my $vmname = $conf->{name
} || "vm$vmid";
3571 push @$cmd, '-name', "$vmname,debug-threads=on";
3573 push @$cmd, '-no-shutdown';
3577 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3578 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3579 push @$cmd, '-mon', "chardev=qmp,mode=control";
3581 if (min_version
($machine_version, 2, 12)) {
3582 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3583 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3586 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3588 push @$cmd, '-daemonize';
3590 if ($conf->{smbios1
}) {
3591 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3592 if ($smbios_conf->{base64
}) {
3593 # Do not pass base64 flag to qemu
3594 delete $smbios_conf->{base64
};
3595 my $smbios_string = "";
3596 foreach my $key (keys %$smbios_conf) {
3598 if ($key eq "uuid") {
3599 $value = $smbios_conf->{uuid
}
3601 $value = decode_base64
($smbios_conf->{$key});
3603 # qemu accepts any binary data, only commas need escaping by double comma
3605 $smbios_string .= "," . $key . "=" . $value if $value;
3607 push @$cmd, '-smbios', "type=1" . $smbios_string;
3609 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3613 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3614 die "OVMF (UEFI) BIOS is not supported on 32-bit CPU types\n"
3615 if !$forcecpu && get_cpu_bitness
($conf->{cpu
}, $arch) == 32;
3617 my ($code_drive_str, $var_drive_str) =
3618 print_ovmf_drive_commandlines
($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3619 push $cmd->@*, '-drive', $code_drive_str;
3620 push $cmd->@*, '-drive', $var_drive_str;
3623 if ($q35) { # tell QEMU to load q35 config early
3624 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3625 if (min_version
($machine_version, 4, 0)) {
3626 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3628 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3632 if (defined(my $fixups = qemu_created_version_fixups
($conf, $forcemachine, $kvmver))) {
3633 push @$cmd, $fixups->@*;
3636 if ($conf->{vmgenid
}) {
3637 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3640 # add usb controllers
3641 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3642 $conf, $bridges, $arch, $machine_type, $machine_version);
3643 push @$devices, @usbcontrollers if @usbcontrollers;
3644 my $vga = parse_vga
($conf->{vga
});
3646 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3647 $vga->{type
} = 'qxl' if $qxlnum;
3649 if (!$vga->{type
}) {
3650 if ($arch eq 'aarch64') {
3651 $vga->{type
} = 'virtio';
3652 } elsif (min_version
($machine_version, 2, 9)) {
3653 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3655 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3659 # enable absolute mouse coordinates (needed by vnc)
3660 my $tablet = $conf->{tablet
};
3661 if (!defined($tablet)) {
3662 $tablet = $defaults->{tablet
};
3663 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3664 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3668 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3669 my $kbd = print_keyboarddevice_full
($conf, $arch);
3670 push @$devices, '-device', $kbd if defined($kbd);
3673 my $bootorder = device_bootorder
($conf);
3675 # host pci device passthrough
3676 my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3677 $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3680 my $usb_dev_features = {};
3681 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3683 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3684 $conf, $usb_dev_features, $bootorder, $machine_version);
3685 push @$devices, @usbdevices if @usbdevices;
3688 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3689 my $path = $conf->{"serial$i"} or next;
3690 if ($path eq 'socket') {
3691 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3692 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3693 # On aarch64, serial0 is the UART device. QEMU only allows
3694 # connecting UART devices via the '-serial' command line, as
3695 # the device has a fixed slot on the hardware...
3696 if ($arch eq 'aarch64' && $i == 0) {
3697 push @$devices, '-serial', "chardev:serial$i";
3699 push @$devices, '-device', "isa-serial,chardev=serial$i";
3702 die "no such serial device\n" if ! -c
$path;
3703 push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3704 push @$devices, '-device', "isa-serial,chardev=serial$i";
3709 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3710 if (my $path = $conf->{"parallel$i"}) {
3711 die "no such parallel device\n" if ! -c
$path;
3712 my $devtype = $path =~ m!^/dev/usb/lp! ?
'serial' : 'parallel';
3713 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3714 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3718 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3719 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3720 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3721 push @$devices, @$audio_devs;
3724 # Add a TPM only if the VM is not a template,
3725 # to support backing up template VMs even if the TPM disk is write-protected.
3726 add_tpm_device
($vmid, $devices, $conf) if (!PVE
::QemuConfig-
>is_template($conf));
3729 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3730 $sockets = $conf->{sockets
} if $conf->{sockets
};
3732 my $cores = $conf->{cores
} || 1;
3734 my $maxcpus = $sockets * $cores;
3736 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3738 my $allowed_vcpus = $cpuinfo->{cpus
};
3740 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3742 if ($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3743 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3744 for (my $i = 2; $i <= $vcpus; $i++) {
3745 my $cpustr = print_cpu_device
($conf, $arch, $i);
3746 push @$cmd, '-device', $cpustr;
3751 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3753 push @$cmd, '-nodefaults';
3755 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3757 push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3759 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3761 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3762 push @$devices, '-device', print_vga_device
(
3763 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3765 push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type
} eq 'virtio-gl'; # VIRGL
3767 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3768 push @$cmd, '-vnc', "unix:$socket,password=on";
3770 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3771 push @$cmd, '-nographic';
3775 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3776 my $useLocaltime = $conf->{localtime};
3778 if ($winversion >= 5) { # windows
3779 $useLocaltime = 1 if !defined($conf->{localtime});
3781 # use time drift fix when acpi is enabled
3782 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3783 $tdf = 1 if !defined($conf->{tdf
});
3787 if ($winversion >= 6) {
3788 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3789 push @$machineFlags, 'hpet=off';
3792 push @$rtcFlags, 'driftfix=slew' if $tdf;
3794 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3795 push @$rtcFlags, "base=$conf->{startdate}";
3796 } elsif ($useLocaltime) {
3797 push @$rtcFlags, 'base=localtime';
3801 push @$cmd, '-cpu', $forcecpu;
3803 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3806 PVE
::QemuServer
::Memory
::config
(
3807 $conf, $vmid, $sockets, $cores, $hotplug_features->{memory
}, $cmd);
3809 push @$cmd, '-S' if $conf->{freeze
};
3811 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3813 my $guest_agent = parse_guest_agent
($conf);
3815 if ($guest_agent->{enabled
}) {
3816 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3817 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3819 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3820 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3821 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3822 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3823 } elsif ($guest_agent->{type
} eq 'isa') {
3824 push @$devices, '-device', "isa-serial,chardev=qga0";
3828 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3829 if ($rng && $version_guard->(4, 1, 2)) {
3830 check_rng_source
($rng->{source
});
3832 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3833 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3834 my $limiter_str = "";
3836 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3839 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3840 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3841 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3846 assert_clipboard_config
($vga);
3847 my $is_spice = $qxlnum || $vga->{type
} =~ /^virtio/;
3849 if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3852 for (my $i = 1; $i < $qxlnum; $i++){
3853 push @$devices, '-device', print_vga_device
(
3854 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3857 # assume other OS works like Linux
3858 my ($ram, $vram) = ("134217728", "67108864");
3859 if ($vga->{memory
}) {
3860 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3861 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3863 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3864 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3868 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3870 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3871 if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3872 push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3874 push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3876 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3879 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3880 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3881 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3883 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3884 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3886 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3887 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3888 if ($spice_enhancement->{foldersharing
}) {
3889 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3890 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3893 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3894 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3895 if $spice_enhancement->{videostreaming
};
3896 push @$devices, '-spice', "$spice_opts";
3900 # enable balloon by default, unless explicitly disabled
3901 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3902 my $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3903 my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3904 $ballooncmd .= ",free-page-reporting=on" if min_version
($machine_version, 6, 2);
3905 push @$devices, '-device', $ballooncmd;
3908 if ($conf->{watchdog
}) {
3909 my $wdopts = parse_watchdog
($conf->{watchdog
});
3910 my $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3911 my $watchdog = $wdopts->{model
} || 'i6300esb';
3912 push @$devices, '-device', "$watchdog$pciaddr";
3913 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3917 my $scsicontroller = {};
3918 my $ahcicontroller = {};
3919 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3921 # Add iscsi initiator name if available
3922 if (my $initiator = get_initiator_name
()) {
3923 push @$devices, '-iscsi', "initiator-name=$initiator";
3926 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3927 my ($ds, $drive) = @_;
3929 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3930 check_volume_storage_type
($storecfg, $drive->{file
});
3931 push @$vollist, $drive->{file
};
3934 # ignore efidisk here, already added in bios/fw handling code above
3935 return if $drive->{interface
} eq 'efidisk';
3937 return if $drive->{interface
} eq 'tpmstate';
3939 $use_virtio = 1 if $ds =~ m/^virtio/;
3941 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3943 if ($drive->{interface
} eq 'virtio'){
3944 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3947 if ($drive->{interface
} eq 'scsi') {
3949 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3951 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3952 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3954 my $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3955 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3958 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3959 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3960 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3961 } elsif ($drive->{iothread
}) {
3963 "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3968 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3969 $queues = ",num_queues=$drive->{queues}";
3972 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3973 if !$scsicontroller->{$controller};
3974 $scsicontroller->{$controller}=1;
3977 if ($drive->{interface
} eq 'sata') {
3978 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3979 my $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3980 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3981 if !$ahcicontroller->{$controller};
3982 $ahcicontroller->{$controller}=1;
3985 my $live_restore = $live_restore_backing->{$ds};
3986 my $live_blockdev_name = undef;
3987 if ($live_restore) {
3988 $live_blockdev_name = $live_restore->{name
};
3989 push @$devices, '-blockdev', $live_restore->{blockdev
};
3992 my $drive_cmd = print_drive_commandline_full
(
3993 $storecfg, $vmid, $drive, $live_blockdev_name, min_version
($kvmver, 6, 0));
3995 # extra protection for templates, but SATA and IDE don't support it..
3996 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
3998 push @$devices, '-drive',$drive_cmd;
3999 push @$devices, '-device', print_drivedevice_full
(
4000 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4003 for (my $i = 0; $i < $MAX_NETS; $i++) {
4004 my $netname = "net$i";
4006 next if !$conf->{$netname};
4007 my $d = parse_net
($conf->{$netname});
4009 # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4011 $use_virtio = 1 if $d->{model
} eq 'virtio';
4013 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
4015 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
4016 push @$devices, '-netdev', $netdevfull;
4018 my $netdevicefull = print_netdevice_full
(
4019 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4021 push @$devices, '-device', $netdevicefull;
4024 if ($conf->{ivshmem
}) {
4025 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4029 $bus = print_pcie_addr
("ivshmem");
4031 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4034 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4035 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4037 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4038 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4039 .",size=$ivshmem->{size}M";
4042 # pci.4 is nested in pci.1
4043 $bridges->{1} = 1 if $bridges->{4};
4045 if (!$q35) { # add pci bridges
4046 if (min_version
($machine_version, 2, 3)) {
4050 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4053 for my $k (sort {$b cmp $a} keys %$bridges) {
4054 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4057 if ($k == 2 && $legacy_igd) {
4060 my $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
4061 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4063 if ($q35) { # add after -readconfig pve-q35.cfg
4064 splice @$devices, 2, 0, '-device', $devstr;
4066 unshift @$devices, '-device', $devstr if $k > 0;
4071 push @$machineFlags, 'accel=tcg';
4074 push @$machineFlags, 'smm=off' if should_disable_smm
($conf, $vga, $machine_type);
4076 my $machine_type_min = $machine_type;
4077 if ($add_pve_version) {
4078 $machine_type_min =~ s/\+pve\d+$//;
4079 $machine_type_min .= "+pve$required_pve_version";
4081 push @$machineFlags, "type=${machine_type_min}";
4083 PVE
::QemuServer
::Machine
::assert_valid_machine_property
($conf, $machine_conf);
4085 if (my $viommu = $machine_conf->{viommu
}) {
4086 if ($viommu eq 'intel') {
4087 unshift @$devices, '-device', 'intel-iommu,intremap=on,caching-mode=on';
4088 push @$machineFlags, 'kernel-irqchip=split';
4089 } elsif ($viommu eq 'virtio') {
4090 push @$devices, '-device', 'virtio-iommu-pci';
4094 push @$cmd, @$devices;
4095 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4096 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4097 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4099 if (my $vmstate = $conf->{vmstate
}) {
4100 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4101 push @$vollist, $vmstate;
4102 push @$cmd, '-loadstate', $statepath;
4103 print "activating and using '$vmstate' as vmstate\n";
4106 if (PVE
::QemuConfig-
>is_template($conf)) {
4107 # needed to workaround base volumes being read-only
4108 push @$cmd, '-snapshot';
4112 if ($conf->{args
}) {
4113 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4117 return wantarray ?
($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4120 sub check_rng_source
{
4123 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4124 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4127 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4128 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
4129 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4130 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4131 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4132 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4140 my $res = mon_cmd
($vmid, 'query-spice');
4142 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4145 sub vm_devices_list
{
4148 my $res = mon_cmd
($vmid, 'query-pci');
4149 my $devices_to_check = [];
4151 foreach my $pcibus (@$res) {
4152 push @$devices_to_check, @{$pcibus->{devices
}},
4155 while (@$devices_to_check) {
4157 for my $d (@$devices_to_check) {
4158 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4159 next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices
};
4161 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4162 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4164 $devices_to_check = $to_check;
4167 my $resblock = mon_cmd
($vmid, 'query-block');
4168 foreach my $block (@$resblock) {
4169 if($block->{device
} =~ m/^drive-(\S+)/){
4174 my $resmice = mon_cmd
($vmid, 'query-mice');
4175 foreach my $mice (@$resmice) {
4176 if ($mice->{name
} eq 'QEMU HID Tablet') {
4177 $devices->{tablet
} = 1;
4182 # for usb devices there is no query-usb
4183 # but we can iterate over the entries in
4184 # qom-list path=/machine/peripheral
4185 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4186 foreach my $per (@$resperipheral) {
4187 if ($per->{name
} =~ m/^usb(?:redirdev)?\d+$/) {
4188 $devices->{$per->{name
}} = 1;
4196 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4198 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4200 my $devices_list = vm_devices_list
($vmid);
4201 return 1 if defined($devices_list->{$deviceid});
4203 # add PCI bridge if we need it for the device
4204 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4206 if ($deviceid eq 'tablet') {
4207 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4208 } elsif ($deviceid eq 'keyboard') {
4209 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4210 } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4212 qemu_spice_usbredir_chardev_add
($vmid, "usbredirchardev$id");
4213 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_spice_usbdevice
($id, "xhci", $id + 1));
4214 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4215 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device, {}, $1 + 1));
4216 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4217 qemu_iothread_add
($vmid, $deviceid, $device);
4219 qemu_driveadd
($storecfg, $vmid, $device);
4220 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4222 qemu_deviceadd
($vmid, $devicefull);
4223 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4225 eval { qemu_drivedel
($vmid, $deviceid); };
4229 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4230 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4231 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4232 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4234 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4236 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4237 qemu_iothread_add
($vmid, $deviceid, $device);
4238 $devicefull .= ",iothread=iothread-$deviceid";
4241 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4242 $devicefull .= ",num_queues=$device->{queues}";
4245 qemu_deviceadd
($vmid, $devicefull);
4246 qemu_deviceaddverify
($vmid, $deviceid);
4247 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4248 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4249 qemu_driveadd
($storecfg, $vmid, $device);
4251 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4252 eval { qemu_deviceadd
($vmid, $devicefull); };
4254 eval { qemu_drivedel
($vmid, $deviceid); };
4258 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4259 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4261 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4262 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type);
4263 my $use_old_bios_files = undef;
4264 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4266 my $netdevicefull = print_netdevice_full
(
4267 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4268 qemu_deviceadd
($vmid, $netdevicefull);
4270 qemu_deviceaddverify
($vmid, $deviceid);
4271 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4274 eval { qemu_netdevdel
($vmid, $deviceid); };
4278 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4280 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4281 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4283 qemu_deviceadd
($vmid, $devicefull);
4284 qemu_deviceaddverify
($vmid, $deviceid);
4286 die "can't hotplug device '$deviceid'\n";
4292 # fixme: this should raise exceptions on error!
4293 sub vm_deviceunplug
{
4294 my ($vmid, $conf, $deviceid) = @_;
4296 my $devices_list = vm_devices_list
($vmid);
4297 return 1 if !defined($devices_list->{$deviceid});
4299 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4300 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4302 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4303 qemu_devicedel
($vmid, $deviceid);
4304 } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4305 qemu_devicedel
($vmid, $deviceid);
4306 qemu_devicedelverify
($vmid, $deviceid);
4307 } elsif ($deviceid =~ m/^usb\d+$/) {
4308 qemu_devicedel
($vmid, $deviceid);
4309 qemu_devicedelverify
($vmid, $deviceid);
4310 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4311 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4313 qemu_devicedel
($vmid, $deviceid);
4314 qemu_devicedelverify
($vmid, $deviceid);
4315 qemu_drivedel
($vmid, $deviceid);
4316 qemu_iothread_del
($vmid, $deviceid, $device);
4317 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4318 qemu_devicedel
($vmid, $deviceid);
4319 qemu_devicedelverify
($vmid, $deviceid);
4320 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4321 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4323 qemu_devicedel
($vmid, $deviceid);
4324 qemu_devicedelverify
($vmid, $deviceid);
4325 qemu_drivedel
($vmid, $deviceid);
4326 qemu_deletescsihw
($conf, $vmid, $deviceid);
4328 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4329 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4330 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4331 qemu_devicedel
($vmid, $deviceid);
4332 qemu_devicedelverify
($vmid, $deviceid);
4333 qemu_netdevdel
($vmid, $deviceid);
4335 die "can't unplug device '$deviceid'\n";
4341 sub qemu_spice_usbredir_chardev_add
{
4342 my ($vmid, $id) = @_;
4344 mon_cmd
($vmid, "chardev-add" , (
4355 sub qemu_iothread_add
{
4356 my ($vmid, $deviceid, $device) = @_;
4358 if ($device->{iothread
}) {
4359 my $iothreads = vm_iothreads_list
($vmid);
4360 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4364 sub qemu_iothread_del
{
4365 my ($vmid, $deviceid, $device) = @_;
4367 if ($device->{iothread
}) {
4368 my $iothreads = vm_iothreads_list
($vmid);
4369 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4374 my ($storecfg, $vmid, $device) = @_;
4376 my $kvmver = get_running_qemu_version
($vmid);
4377 my $io_uring = min_version
($kvmver, 6, 0);
4378 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4379 $drive =~ s/\\/\\\\/g;
4380 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4382 # If the command succeeds qemu prints: "OK
"
4383 return 1 if $ret =~ m/OK/s;
4385 die "adding drive failed
: $ret\n";
4389 my ($vmid, $deviceid) = @_;
4391 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4394 return 1 if $ret eq "";
4396 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4397 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4399 die "deleting drive
$deviceid failed
: $ret\n";
4402 sub qemu_deviceaddverify {
4403 my ($vmid, $deviceid) = @_;
4405 for (my $i = 0; $i <= 5; $i++) {
4406 my $devices_list = vm_devices_list($vmid);
4407 return 1 if defined($devices_list->{$deviceid});
4411 die "error on hotplug device
'$deviceid'\n";
4415 sub qemu_devicedelverify {
4416 my ($vmid, $deviceid) = @_;
4418 # need to verify that the device is correctly removed as device_del
4419 # is async and empty return is not reliable
4421 for (my $i = 0; $i <= 5; $i++) {
4422 my $devices_list = vm_devices_list($vmid);
4423 return 1 if !defined($devices_list->{$deviceid});
4427 die "error on hot-unplugging device
'$deviceid'\n";
4430 sub qemu_findorcreatescsihw {
4431 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4433 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4435 my $scsihwid="$controller_prefix$controller";
4436 my $devices_list = vm_devices_list($vmid);
4438 if (!defined($devices_list->{$scsihwid})) {
4439 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4445 sub qemu_deletescsihw {
4446 my ($conf, $vmid, $opt) = @_;
4448 my $device = parse_drive($opt, $conf->{$opt});
4450 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4451 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4455 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4457 my $devices_list = vm_devices_list($vmid);
4458 foreach my $opt (keys %{$devices_list}) {
4459 if (is_valid_drivename($opt)) {
4460 my $drive = parse_drive($opt, $conf->{$opt});
4461 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4467 my $scsihwid="scsihw
$controller";
4469 vm_deviceunplug($vmid, $conf, $scsihwid);
4474 sub qemu_add_pci_bridge {
4475 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4481 print_pci_addr($device, $bridges, $arch, $machine_type);
4483 while (my ($k, $v) = each %$bridges) {
4486 return 1 if !defined($bridgeid) || $bridgeid < 1;
4488 my $bridge = "pci
.$bridgeid";
4489 my $devices_list = vm_devices_list($vmid);
4491 if (!defined($devices_list->{$bridge})) {
4492 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4498 sub qemu_set_link_status {
4499 my ($vmid, $device, $up) = @_;
4501 mon_cmd($vmid, "set_link
", name => $device,
4502 up => $up ? JSON::true : JSON::false);
4505 sub qemu_netdevadd {
4506 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4508 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4509 my %options = split(/[=,]/, $netdev);
4511 if (defined(my $vhost = $options{vhost})) {
4512 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4515 if (defined(my $queues = $options{queues})) {
4516 $options{queues} = $queues + 0;
4519 mon_cmd($vmid, "netdev_add
", %options);
4523 sub qemu_netdevdel {
4524 my ($vmid, $deviceid) = @_;
4526 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4529 sub qemu_usb_hotplug {
4530 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4534 # remove the old one first
4535 vm_deviceunplug($vmid, $conf, $deviceid);
4537 # check if xhci controller is necessary and available
4538 my $devicelist = vm_devices_list($vmid);
4540 if (!$devicelist->{xhci}) {
4541 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4542 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4546 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4549 sub qemu_cpu_hotplug {
4550 my ($vmid, $conf, $vcpus) = @_;
4552 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4555 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4556 $sockets = $conf->{sockets} if $conf->{sockets};
4557 my $cores = $conf->{cores} || 1;
4558 my $maxcpus = $sockets * $cores;
4560 $vcpus = $maxcpus if !$vcpus;
4562 die "you can
't add more vcpus than maxcpus\n"
4563 if $vcpus > $maxcpus;
4565 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4567 if ($vcpus < $currentvcpus) {
4569 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4571 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4572 qemu_devicedel($vmid, "cpu$i");
4574 my $currentrunningvcpus = undef;
4576 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4577 last if scalar(@{$currentrunningvcpus}) == $i-1;
4578 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4582 #update conf after each succesfull cpu unplug
4583 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4584 PVE::QemuConfig->write_config($vmid, $conf);
4587 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4593 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4594 die "vcpus in running vm does not match its configuration\n"
4595 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4597 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4598 my $arch = get_vm_arch($conf);
4600 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4601 my $cpustr = print_cpu_device($conf, $arch, $i);
4602 qemu_deviceadd($vmid, $cpustr);
4605 my $currentrunningvcpus = undef;
4607 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4608 last if scalar(@{$currentrunningvcpus}) == $i;
4609 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4613 #update conf after each succesfull cpu hotplug
4614 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4615 PVE::QemuConfig->write_config($vmid, $conf);
4619 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4620 mon_cmd($vmid, "cpu-add", id => int($i));
4625 sub qemu_block_set_io_throttle {
4626 my ($vmid, $deviceid,
4627 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4628 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4629 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4630 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4632 return if !check_running($vmid) ;
4634 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4636 bps_rd => int($bps_rd),
4637 bps_wr => int($bps_wr),
4639 iops_rd => int($iops_rd),
4640 iops_wr => int($iops_wr),
4641 bps_max => int($bps_max),
4642 bps_rd_max => int($bps_rd_max),
4643 bps_wr_max => int($bps_wr_max),
4644 iops_max => int($iops_max),
4645 iops_rd_max => int($iops_rd_max),
4646 iops_wr_max => int($iops_wr_max),
4647 bps_max_length => int($bps_max_length),
4648 bps_rd_max_length => int($bps_rd_max_length),
4649 bps_wr_max_length => int($bps_wr_max_length),
4650 iops_max_length => int($iops_max_length),
4651 iops_rd_max_length => int($iops_rd_max_length),
4652 iops_wr_max_length => int($iops_wr_max_length),
4657 sub qemu_block_resize {
4658 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4660 my $running = check_running($vmid);
4662 PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4664 return if !$running;
4666 my $padding = (1024 - $size % 1024) % 1024;
4667 $size = $size + $padding;
4672 device => $deviceid,
4678 sub qemu_volume_snapshot {
4679 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4681 my $running = check_running($vmid);
4683 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4684 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4686 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4690 sub qemu_volume_snapshot_delete {
4691 my ($vmid, $storecfg, $volid, $snap) = @_;
4693 my $running = check_running($vmid);
4694 my $attached_deviceid;
4697 my $conf = PVE::QemuConfig->load_config($vmid);
4698 PVE::QemuConfig->foreach_volume($conf, sub {
4699 my ($ds, $drive) = @_;
4700 $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
4704 if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
4707 'blockdev-snapshot-delete-internal-sync
',
4708 device => $attached_deviceid,
4712 PVE::Storage::volume_snapshot_delete(
4713 $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
4717 sub set_migration_caps {
4718 my ($vmid, $savevm) = @_;
4720 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4722 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4723 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4728 "auto-converge" => 1,
4730 "x-rdma-pin-all" => 0,
4733 "dirty-bitmaps" => $dirty_bitmaps,
4736 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4738 for my $supported_capability (@$supported_capabilities) {
4740 capability => $supported_capability->{capability},
4741 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4745 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4749 my ($conf, $func, @param) = @_;
4753 my $test_volid = sub {
4754 my ($key, $drive, $snapname, $pending) = @_;
4756 my $volid = $drive->{file};
4759 $volhash->{$volid}->{cdrom} //= 1;
4760 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4762 my $replicate = $drive->{replicate} // 1;
4763 $volhash->{$volid}->{replicate} //= 0;
4764 $volhash->{$volid}->{replicate} = 1 if $replicate;
4766 $volhash->{$volid}->{shared} //= 0;
4767 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4769 $volhash->{$volid}->{is_unused} //= 0;
4770 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4772 $volhash->{$volid}->{is_attached} //= 0;
4773 $volhash->{$volid}->{is_attached} = 1
4774 if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4776 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4777 if defined($snapname);
4779 $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4781 my $size = $drive->{size};
4782 $volhash->{$volid}->{size} //= $size if $size;
4784 $volhash->{$volid}->{is_vmstate} //= 0;
4785 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4787 $volhash->{$volid}->{is_tpmstate} //= 0;
4788 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0
';
4790 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4793 my $include_opts = {
4794 extra_keys => ['vmstate
'],
4795 include_unused => 1,
4798 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4800 PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4801 if defined($conf->{pending}) && $conf->{pending}->%*;
4803 foreach my $snapname (keys %{$conf->{snapshots}}) {
4804 my $snap = $conf->{snapshots}->{$snapname};
4805 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4808 foreach my $volid (keys %$volhash) {
4809 &$func($volid, $volhash->{$volid}, @param);
4813 my $fast_plug_option = {
4817 'migrate_downtime
' => 1,
4818 'migrate_speed
' => 1,
4825 'vmstatestorage
' => 1,
4828 for my $opt (keys %$confdesc_cloudinit) {
4829 $fast_plug_option->{$opt} = 1;
4832 # hotplug changes in [PENDING]
4833 # $selection hash can be used to only apply specified options, for
4834 # example: { cores => 1 } (only apply changed 'cores
')
4835 # $errors ref is used to return error messages
4836 sub vmconfig_hotplug_pending {
4837 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4839 my $defaults = load_defaults();
4840 my $arch = get_vm_arch($conf);
4841 my $machine_type = get_vm_machine($conf, undef, $arch);
4843 # commit values which do not have any impact on running VM first
4844 # Note: those option cannot raise errors, we we do not care about
4845 # $selection and always apply them.
4847 my $add_error = sub {
4848 my ($opt, $msg) = @_;
4849 $errors->{$opt} = "hotplug problem - $msg";
4852 my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4854 my $cloudinit_record_changed = sub {
4855 my ($conf, $opt, $old, $new) = @_;
4856 return if !$cloudinit_pending_properties->{$opt};
4858 my $ci = ($conf->{cloudinit} //= {});
4860 my $recorded = $ci->{$opt};
4861 my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4863 if (defined($new)) {
4864 if (defined($old)) {
4865 # an existing value is being modified
4866 if (defined($recorded)) {
4867 # the value was already not in sync
4868 if ($new eq $recorded) {
4869 # a value is being reverted to the cloud-init state:
4871 delete $added{$opt};
4873 # the value was changed multiple times, do nothing
4875 } elsif ($added{$opt}) {
4876 # the value had been marked as added and is being changed, do nothing
4878 # the value is new, record it:
4882 # a new value is being added
4883 if (defined($recorded)) {
4884 # it was already not in sync
4885 if ($new eq $recorded) {
4886 # a value is being reverted to the cloud-init state:
4888 delete $added{$opt};
4890 # the value had temporarily been removed, do nothing
4892 } elsif ($added{$opt}) {
4893 # the value had been marked as added already, do nothing
4895 # the value is new, add it
4899 } elsif (!defined($old)) {
4900 # a non-existent value is being removed? ignore...
4902 # a value is being deleted
4903 if (defined($recorded)) {
4904 # a value was already recorded, just keep it
4905 } elsif ($added{$opt}) {
4906 # the value was marked as added, remove it
4907 delete $added{$opt};
4909 # a previously unrecorded value is being removed, record the old value:
4914 my $added = join(',', sort keys %added);
4915 $ci->{added} = $added if length($added);
4919 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4920 if ($fast_plug_option->{$opt}) {
4921 my $new = delete $conf->{pending}->{$opt};
4922 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4923 $conf->{$opt} = $new;
4929 PVE::QemuConfig->write_config($vmid, $conf);
4932 my $ostype = $conf->{ostype};
4933 my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4934 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4935 my $usb_hotplug = $hotplug_features->{usb}
4936 && min_version($version, 7, 1)
4937 && defined($ostype) && ($ostype eq 'l26
' || windows_version($ostype) > 7);
4939 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4940 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4942 foreach my $opt (sort keys %$pending_delete_hash) {
4943 next if $selection && !$selection->{$opt};
4944 my $force = $pending_delete_hash->{$opt}->{force};
4946 if ($opt eq 'hotplug
') {
4947 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4948 } elsif ($opt eq 'tablet
') {
4949 die "skip\n" if !$hotplug_features->{usb};
4950 if ($defaults->{tablet}) {
4951 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4952 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4953 if $arch eq 'aarch64
';
4955 vm_deviceunplug($vmid, $conf, 'tablet
');
4956 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4958 } elsif ($opt =~ m/^usb(\d+)$/) {
4960 die "skip\n" if !$usb_hotplug;
4961 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4962 vm_deviceunplug
($vmid, $conf, $opt);
4963 } elsif ($opt eq 'vcpus') {
4964 die "skip\n" if !$hotplug_features->{cpu
};
4965 qemu_cpu_hotplug
($vmid, $conf, undef);
4966 } elsif ($opt eq 'balloon') {
4967 # enable balloon device is not hotpluggable
4968 die "skip\n" if defined($conf->{balloon
}) && $conf->{balloon
} == 0;
4969 # here we reset the ballooning value to memory
4970 my $balloon = get_current_memory
($conf->{memory
});
4971 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
4972 } elsif ($fast_plug_option->{$opt}) {
4974 } elsif ($opt =~ m/^net(\d+)$/) {
4975 die "skip\n" if !$hotplug_features->{network
};
4976 vm_deviceunplug
($vmid, $conf, $opt);
4978 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
4979 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
});
4981 } elsif (is_valid_drivename
($opt)) {
4982 die "skip\n" if !$hotplug_features->{disk
} || $opt =~ m/(ide|sata)(\d+)/;
4983 vm_deviceunplug
($vmid, $conf, $opt);
4984 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
4985 } elsif ($opt =~ m/^memory$/) {
4986 die "skip\n" if !$hotplug_features->{memory
};
4987 PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf);
4988 } elsif ($opt eq 'cpuunits') {
4989 $cgroup->change_cpu_shares(undef);
4990 } elsif ($opt eq 'cpulimit') {
4991 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4997 &$add_error($opt, $err) if $err ne "skip\n";
4999 my $old = delete $conf->{$opt};
5000 $cloudinit_record_changed->($conf, $opt, $old, undef);
5001 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5006 foreach my $opt (keys %{$conf->{pending
}}) {
5007 next if $selection && !$selection->{$opt};
5008 my $value = $conf->{pending
}->{$opt};
5010 if ($opt eq 'hotplug') {
5011 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug
} =~ /memory/);
5012 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug
} =~ /cpu/);
5013 } elsif ($opt eq 'tablet') {
5014 die "skip\n" if !$hotplug_features->{usb
};
5016 vm_deviceplug
($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5017 vm_deviceplug
($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5018 if $arch eq 'aarch64';
5019 } elsif ($value == 0) {
5020 vm_deviceunplug
($vmid, $conf, 'tablet');
5021 vm_deviceunplug
($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5023 } elsif ($opt =~ m/^usb(\d+)$/) {
5025 die "skip\n" if !$usb_hotplug;
5026 my $d = eval { parse_property_string
('pve-qm-usb', $value) };
5028 if ($d->{host
} =~ m/^spice$/i) {
5029 $id = "usbredirdev$index";
5031 qemu_usb_hotplug
($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5032 } elsif ($opt eq 'vcpus') {
5033 die "skip\n" if !$hotplug_features->{cpu
};
5034 qemu_cpu_hotplug
($vmid, $conf, $value);
5035 } elsif ($opt eq 'balloon') {
5036 # enable/disable balloning device is not hotpluggable
5037 my $old_balloon_enabled = !!(!defined($conf->{balloon
}) || $conf->{balloon
});
5038 my $new_balloon_enabled = !!(!defined($conf->{pending
}->{balloon
}) || $conf->{pending
}->{balloon
});
5039 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5041 # allow manual ballooning if shares is set to zero
5042 if ((defined($conf->{shares
}) && ($conf->{shares
} == 0))) {
5043 my $memory = get_current_memory
($conf->{memory
});
5044 my $balloon = $conf->{pending
}->{balloon
} || $memory;
5045 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5047 } elsif ($opt =~ m/^net(\d+)$/) {
5048 # some changes can be done without hotplug
5049 vmconfig_update_net
($storecfg, $conf, $hotplug_features->{network
},
5050 $vmid, $opt, $value, $arch, $machine_type);
5051 } elsif (is_valid_drivename
($opt)) {
5052 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5053 # some changes can be done without hotplug
5054 my $drive = parse_drive
($opt, $value);
5055 if (drive_is_cloudinit
($drive)) {
5056 $cloudinit_opt = [$opt, $drive];
5057 # apply all the other changes first, then generate the cloudinit disk
5060 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5061 $vmid, $opt, $value, $arch, $machine_type);
5062 } elsif ($opt =~ m/^memory$/) { #dimms
5063 die "skip\n" if !$hotplug_features->{memory
};
5064 $value = PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $value);
5065 } elsif ($opt eq 'cpuunits') {
5066 my $new_cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{pending
}->{$opt}); #clamp
5067 $cgroup->change_cpu_shares($new_cpuunits);
5068 } elsif ($opt eq 'cpulimit') {
5069 my $cpulimit = $conf->{pending
}->{$opt} == 0 ?
-1 : int($conf->{pending
}->{$opt} * 100000);
5070 $cgroup->change_cpu_quota($cpulimit, 100000);
5071 } elsif ($opt eq 'agent') {
5072 vmconfig_update_agent
($conf, $opt, $value);
5074 die "skip\n"; # skip non-hot-pluggable options
5078 &$add_error($opt, $err) if $err ne "skip\n";
5080 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5081 $conf->{$opt} = $value;
5082 delete $conf->{pending
}->{$opt};
5086 if (defined($cloudinit_opt)) {
5087 my ($opt, $drive) = @$cloudinit_opt;
5088 my $value = $conf->{pending
}->{$opt};
5090 my $temp = {%$conf, $opt => $value};
5091 PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($temp, $vmid);
5092 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5093 $vmid, $opt, $value, $arch, $machine_type);
5096 &$add_error($opt, $err) if $err ne "skip\n";
5098 $conf->{$opt} = $value;
5099 delete $conf->{pending
}->{$opt};
5103 # unplug xhci controller if no usb device is left
5106 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
5107 next if !defined($conf->{"usb$i"});
5112 vm_deviceunplug
($vmid, $conf, 'xhci');
5116 PVE
::QemuConfig-
>write_config($vmid, $conf);
5118 if ($hotplug_features->{cloudinit
} && PVE
::QemuServer
::Cloudinit
::has_changes
($conf)) {
5119 PVE
::QemuServer
::vmconfig_update_cloudinit_drive
($storecfg, $conf, $vmid);
5123 sub try_deallocate_drive
{
5124 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5126 if (($force || $key =~ /^unused/) && !drive_is_cdrom
($drive, 1)) {
5127 my $volid = $drive->{file
};
5128 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
5129 my $sid = PVE
::Storage
::parse_volume_id
($volid);
5130 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5132 # check if the disk is really unused
5133 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5134 if PVE
::QemuServer
::Drive
::is_volume_in_use
($storecfg, $conf, $key, $volid);
5135 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5138 # If vm is not owner of this disk remove from config
5146 sub vmconfig_delete_or_detach_drive
{
5147 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5149 my $drive = parse_drive
($opt, $conf->{$opt});
5151 my $rpcenv = PVE
::RPCEnvironment
::get
();
5152 my $authuser = $rpcenv->get_user();
5155 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5156 try_deallocate_drive
($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5158 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $drive);
5164 sub vmconfig_apply_pending
{
5165 my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5167 return if !scalar(keys %{$conf->{pending
}});
5169 my $add_apply_error = sub {
5170 my ($opt, $msg) = @_;
5171 my $err_msg = "unable to apply pending change $opt : $msg";
5172 $errors->{$opt} = $err_msg;
5178 my $pending_delete_hash = PVE
::QemuConfig-
>parse_pending_delete($conf->{pending
}->{delete});
5179 foreach my $opt (sort keys %$pending_delete_hash) {
5180 my $force = $pending_delete_hash->{$opt}->{force
};
5182 if ($opt =~ m/^unused/) {
5183 die "internal error";
5184 } elsif (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5185 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5186 } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5188 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
5189 eval { PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
}) };
5195 $add_apply_error->($opt, $err);
5197 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5198 delete $conf->{$opt};
5202 PVE
::QemuConfig-
>cleanup_pending($conf);
5204 my $generate_cloudinit = $skip_cloud_init ?
0 : undef;
5206 foreach my $opt (keys %{$conf->{pending
}}) { # add/change
5207 next if $opt eq 'delete'; # just to be sure
5209 if (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5210 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, parse_drive
($opt, $conf->{$opt}))
5211 } elsif (defined($conf->{pending
}->{$opt}) && $opt =~ m/^net\d+$/) {
5212 return if !$have_sdn; # return from eval if SDN is not available
5214 my $new_net = PVE
::QemuServer
::parse_net
($conf->{pending
}->{$opt});
5215 if ($conf->{$opt}) {
5216 my $old_net = PVE
::QemuServer
::parse_net
($conf->{$opt});
5218 if (defined($old_net->{bridge
}) && defined($old_net->{macaddr
}) && (
5219 safe_string_ne
($old_net->{bridge
}, $new_net->{bridge
}) ||
5220 safe_string_ne
($old_net->{macaddr
}, $new_net->{macaddr
})
5222 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($old_net->{bridge
}, $old_net->{macaddr
}, $conf->{name
});
5225 #fixme: reuse ip if mac change && same bridge
5226 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($new_net->{bridge
}, $conf->{name
}, $new_net->{macaddr
}, $vmid, undef, 1);
5230 $add_apply_error->($opt, $err);
5233 if (is_valid_drivename
($opt)) {
5234 my $drive = parse_drive
($opt, $conf->{pending
}->{$opt});
5235 $generate_cloudinit //= 1 if drive_is_cloudinit
($drive);
5238 $conf->{$opt} = delete $conf->{pending
}->{$opt};
5242 # write all changes at once to avoid unnecessary i/o
5243 PVE
::QemuConfig-
>write_config($vmid, $conf);
5244 if ($generate_cloudinit) {
5245 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5246 # After successful generation and if there were changes to be applied, update the
5247 # config to drop the {cloudinit} entry.
5248 PVE
::QemuConfig-
>write_config($vmid, $conf);
5253 sub vmconfig_update_net
{
5254 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5256 my $newnet = parse_net
($value);
5258 if ($conf->{$opt}) {
5259 my $oldnet = parse_net
($conf->{$opt});
5261 if (safe_string_ne
($oldnet->{model
}, $newnet->{model
}) ||
5262 safe_string_ne
($oldnet->{macaddr
}, $newnet->{macaddr
}) ||
5263 safe_num_ne
($oldnet->{queues
}, $newnet->{queues
}) ||
5264 safe_num_ne
($oldnet->{mtu
}, $newnet->{mtu
}) ||
5265 !($newnet->{bridge
} && $oldnet->{bridge
})
5266 ) { # bridge/nat mode change
5268 # for non online change, we try to hot-unplug
5269 die "skip\n" if !$hotplug;
5270 vm_deviceunplug
($vmid, $conf, $opt);
5273 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($oldnet->{bridge
}, $oldnet->{macaddr
}, $conf->{name
});
5278 die "internal error" if $opt !~ m/net(\d+)/;
5279 my $iface = "tap${vmid}i$1";
5281 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5282 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
}) ||
5283 safe_string_ne
($oldnet->{trunks
}, $newnet->{trunks
}) ||
5284 safe_num_ne
($oldnet->{firewall
}, $newnet->{firewall
})
5286 PVE
::Network
::tap_unplug
($iface);
5288 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5289 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5290 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
})
5292 qemu_set_link_status
($vmid, $opt, 0);
5295 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
})) {
5297 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($oldnet->{bridge
}, $oldnet->{macaddr
}, $conf->{name
});
5298 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($newnet->{bridge
}, $conf->{name
}, $newnet->{macaddr
}, $vmid, undef, 1);
5303 PVE
::Network
::SDN
::Zones
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5305 PVE
::Network
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5308 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5309 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5310 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
})
5312 qemu_set_link_status
($vmid, $opt, 1);
5315 } elsif (safe_num_ne
($oldnet->{rate
}, $newnet->{rate
})) {
5316 # Rate can be applied on its own but any change above needs to
5317 # include the rate in tap_plug since OVS resets everything.
5318 PVE
::Network
::tap_rate_limit
($iface, $newnet->{rate
});
5321 if (safe_string_ne
($oldnet->{link_down
}, $newnet->{link_down
})) {
5322 qemu_set_link_status
($vmid, $opt, !$newnet->{link_down
});
5331 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($newnet->{bridge
}, $conf->{name
}, $newnet->{macaddr
}, $vmid, undef, 1);
5332 PVE
::Network
::SDN
::Vnets
::add_dhcp_mapping
($newnet->{bridge
}, $newnet->{macaddr
}, $vmid, $conf->{name
});
5334 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5340 sub vmconfig_update_agent
{
5341 my ($conf, $opt, $value) = @_;
5343 die "skip\n" if !$conf->{$opt};
5345 my $hotplug_options = { fstrim_cloned_disks
=> 1 };
5347 my $old_agent = parse_guest_agent
($conf);
5348 my $agent = parse_guest_agent
({$opt => $value});
5350 for my $option (keys %$agent) { # added/changed options
5351 next if defined($hotplug_options->{$option});
5352 die "skip\n" if safe_string_ne
($agent->{$option}, $old_agent->{$option});
5355 for my $option (keys %$old_agent) { # removed options
5356 next if defined($hotplug_options->{$option});
5357 die "skip\n" if safe_string_ne
($old_agent->{$option}, $agent->{$option});
5360 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5363 sub vmconfig_update_disk
{
5364 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5366 my $drive = parse_drive
($opt, $value);
5368 if ($conf->{$opt} && (my $old_drive = parse_drive
($opt, $conf->{$opt}))) {
5369 my $media = $drive->{media
} || 'disk';
5370 my $oldmedia = $old_drive->{media
} || 'disk';
5371 die "unable to change media type\n" if $media ne $oldmedia;
5373 if (!drive_is_cdrom
($old_drive)) {
5375 if ($drive->{file
} ne $old_drive->{file
}) {
5377 die "skip\n" if !$hotplug;
5379 # unplug and register as unused
5380 vm_deviceunplug
($vmid, $conf, $opt);
5381 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive)
5384 # update existing disk
5386 # skip non hotpluggable value
5387 if (safe_string_ne
($drive->{aio
}, $old_drive->{aio
}) ||
5388 safe_string_ne
($drive->{discard
}, $old_drive->{discard
}) ||
5389 safe_string_ne
($drive->{iothread
}, $old_drive->{iothread
}) ||
5390 safe_string_ne
($drive->{queues
}, $old_drive->{queues
}) ||
5391 safe_string_ne
($drive->{product
}, $old_drive->{product
}) ||
5392 safe_string_ne
($drive->{cache
}, $old_drive->{cache
}) ||
5393 safe_string_ne
($drive->{ssd
}, $old_drive->{ssd
}) ||
5394 safe_string_ne
($drive->{vendor
}, $old_drive->{vendor
}) ||
5395 safe_string_ne
($drive->{ro
}, $old_drive->{ro
})) {
5400 if (safe_num_ne
($drive->{mbps
}, $old_drive->{mbps
}) ||
5401 safe_num_ne
($drive->{mbps_rd
}, $old_drive->{mbps_rd
}) ||
5402 safe_num_ne
($drive->{mbps_wr
}, $old_drive->{mbps_wr
}) ||
5403 safe_num_ne
($drive->{iops
}, $old_drive->{iops
}) ||
5404 safe_num_ne
($drive->{iops_rd
}, $old_drive->{iops_rd
}) ||
5405 safe_num_ne
($drive->{iops_wr
}, $old_drive->{iops_wr
}) ||
5406 safe_num_ne
($drive->{mbps_max
}, $old_drive->{mbps_max
}) ||
5407 safe_num_ne
($drive->{mbps_rd_max
}, $old_drive->{mbps_rd_max
}) ||
5408 safe_num_ne
($drive->{mbps_wr_max
}, $old_drive->{mbps_wr_max
}) ||
5409 safe_num_ne
($drive->{iops_max
}, $old_drive->{iops_max
}) ||
5410 safe_num_ne
($drive->{iops_rd_max
}, $old_drive->{iops_rd_max
}) ||
5411 safe_num_ne
($drive->{iops_wr_max
}, $old_drive->{iops_wr_max
}) ||
5412 safe_num_ne
($drive->{bps_max_length
}, $old_drive->{bps_max_length
}) ||
5413 safe_num_ne
($drive->{bps_rd_max_length
}, $old_drive->{bps_rd_max_length
}) ||
5414 safe_num_ne
($drive->{bps_wr_max_length
}, $old_drive->{bps_wr_max_length
}) ||
5415 safe_num_ne
($drive->{iops_max_length
}, $old_drive->{iops_max_length
}) ||
5416 safe_num_ne
($drive->{iops_rd_max_length
}, $old_drive->{iops_rd_max_length
}) ||
5417 safe_num_ne
($drive->{iops_wr_max_length
}, $old_drive->{iops_wr_max_length
})) {
5419 qemu_block_set_io_throttle
(
5421 ($drive->{mbps
} || 0)*1024*1024,
5422 ($drive->{mbps_rd
} || 0)*1024*1024,
5423 ($drive->{mbps_wr
} || 0)*1024*1024,
5424 $drive->{iops
} || 0,
5425 $drive->{iops_rd
} || 0,
5426 $drive->{iops_wr
} || 0,
5427 ($drive->{mbps_max
} || 0)*1024*1024,
5428 ($drive->{mbps_rd_max
} || 0)*1024*1024,
5429 ($drive->{mbps_wr_max
} || 0)*1024*1024,
5430 $drive->{iops_max
} || 0,
5431 $drive->{iops_rd_max
} || 0,
5432 $drive->{iops_wr_max
} || 0,
5433 $drive->{bps_max_length
} || 1,
5434 $drive->{bps_rd_max_length
} || 1,
5435 $drive->{bps_wr_max_length
} || 1,
5436 $drive->{iops_max_length
} || 1,
5437 $drive->{iops_rd_max_length
} || 1,
5438 $drive->{iops_wr_max_length
} || 1,
5448 if ($drive->{file
} eq 'none') {
5449 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5450 if (drive_is_cloudinit
($old_drive)) {
5451 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive);
5454 my $path = get_iso_path
($storecfg, $vmid, $drive->{file
});
5456 # force eject if locked
5457 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5460 mon_cmd
($vmid, "blockdev-change-medium",
5461 id
=> "$opt", filename
=> "$path");
5469 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5471 PVE
::Storage
::activate_volumes
($storecfg, [$drive->{file
}]) if $drive->{file
} !~ m
|^/dev/.+|;
5472 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5475 sub vmconfig_update_cloudinit_drive
{
5476 my ($storecfg, $conf, $vmid) = @_;
5478 my $cloudinit_ds = undef;
5479 my $cloudinit_drive = undef;
5481 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5482 my ($ds, $drive) = @_;
5483 if (PVE
::QemuServer
::drive_is_cloudinit
($drive)) {
5484 $cloudinit_ds = $ds;
5485 $cloudinit_drive = $drive;
5489 return if !$cloudinit_drive;
5491 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5492 PVE
::QemuConfig-
>write_config($vmid, $conf);
5495 my $running = PVE
::QemuServer
::check_running
($vmid);
5498 my $path = PVE
::Storage
::path
($storecfg, $cloudinit_drive->{file
});
5500 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$cloudinit_ds");
5501 mon_cmd
($vmid, "blockdev-change-medium", id
=> "$cloudinit_ds", filename
=> "$path");
5506 # called in locked context by incoming migration
5507 sub vm_migrate_get_nbd_disks
{
5508 my ($storecfg, $conf, $replicated_volumes) = @_;
5510 my $local_volumes = {};
5511 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5512 my ($ds, $drive) = @_;
5514 return if drive_is_cdrom
($drive);
5515 return if $ds eq 'tpmstate0';
5517 my $volid = $drive->{file
};
5521 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid);
5523 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5524 return if $scfg->{shared
};
5526 my $format = qemu_img_format
($scfg, $volname);
5528 # replicated disks re-use existing state via bitmap
5529 my $use_existing = $replicated_volumes->{$volid} ?
1 : 0;
5530 $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5532 return $local_volumes;
5535 # called in locked context by incoming migration
5536 sub vm_migrate_alloc_nbd_disks
{
5537 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5540 foreach my $opt (sort keys %$source_volumes) {
5541 my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5543 if ($use_existing) {
5544 $nbd->{$opt}->{drivestr
} = print_drive
($drive);
5545 $nbd->{$opt}->{volid
} = $volid;
5546 $nbd->{$opt}->{replicated
} = 1;
5550 $storeid = PVE
::JSONSchema
::map_id
($storagemap, $storeid);
5552 # order of precedence, filtered by whether storage supports it:
5553 # 1. explicit requested format
5554 # 2. default format of storage
5555 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5556 $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5558 my $size = $drive->{size
} / 1024;
5559 my $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $format, undef, $size);
5560 my $newdrive = $drive;
5561 $newdrive->{format
} = $format;
5562 $newdrive->{file
} = $newvolid;
5563 my $drivestr = print_drive
($newdrive);
5564 $nbd->{$opt}->{drivestr
} = $drivestr;
5565 $nbd->{$opt}->{volid
} = $newvolid;
5571 # see vm_start_nolock for parameters, additionally:
5573 # storagemap = parsed storage map for allocating NBD disks
5575 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5577 return PVE
::QemuConfig-
>lock_config($vmid, sub {
5578 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migrate_opts->{migratedfrom
});
5580 die "you can't start a vm if it's a template\n"
5581 if !$params->{skiptemplate
} && PVE
::QemuConfig-
>is_template($conf);
5583 my $has_suspended_lock = PVE
::QemuConfig-
>has_lock($conf, 'suspended');
5584 my $has_backup_lock = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5586 my $running = check_running
($vmid, undef, $migrate_opts->{migratedfrom
});
5588 if ($has_backup_lock && $running) {
5589 # a backup is currently running, attempt to start the guest in the
5590 # existing QEMU instance
5591 return vm_resume
($vmid);
5594 PVE
::QemuConfig-
>check_lock($conf)
5595 if !($params->{skiplock
} || $has_suspended_lock);
5597 $params->{resume
} = $has_suspended_lock || defined($conf->{vmstate
});
5599 die "VM $vmid already running\n" if $running;
5601 if (my $storagemap = $migrate_opts->{storagemap
}) {
5602 my $replicated = $migrate_opts->{replicated_volumes
};
5603 my $disks = vm_migrate_get_nbd_disks
($storecfg, $conf, $replicated);
5604 $migrate_opts->{nbd
} = vm_migrate_alloc_nbd_disks
($storecfg, $vmid, $disks, $storagemap);
5606 foreach my $opt (keys %{$migrate_opts->{nbd
}}) {
5607 $conf->{$opt} = $migrate_opts->{nbd
}->{$opt}->{drivestr
};
5611 return vm_start_nolock
($storecfg, $vmid, $conf, $params, $migrate_opts);
5617 # statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5618 # skiplock => 0/1, skip checking for config lock
5619 # skiptemplate => 0/1, skip checking whether VM is template
5620 # forcemachine => to force QEMU machine (rollback/migration)
5621 # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5622 # timeout => in seconds
5623 # paused => start VM in paused state (backup)
5624 # resume => resume from hibernation
5625 # live-restore-backing => {
5627 # name => blockdev-name,
5628 # blockdev => "arg to the -blockdev command instantiating device named 'name'",
5633 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5634 # migratedfrom => source node
5635 # spice_ticket => used for spice migration, passed via tunnel/stdin
5636 # network => CIDR of migration network
5637 # type => secure/insecure - tunnel over encrypted connection or plain-text
5638 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5639 # replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5640 # offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5641 # contained in config
5642 sub vm_start_nolock
{
5643 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5645 my $statefile = $params->{statefile
};
5646 my $resume = $params->{resume
};
5648 my $migratedfrom = $migrate_opts->{migratedfrom
};
5649 my $migration_type = $migrate_opts->{type
};
5653 # clean up leftover reboot request files
5654 eval { clear_reboot_request
($vmid); };
5657 if (!$statefile && scalar(keys %{$conf->{pending
}})) {
5658 vmconfig_apply_pending
($vmid, $conf, $storecfg);
5659 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5662 # don't regenerate the ISO if the VM is started as part of a live migration
5663 # this way we can reuse the old ISO with the correct config
5664 if (!$migratedfrom) {
5665 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5666 # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5667 # $conf->{cloudinit}, so we could just not do this?
5668 # But we do it above, so for now let's be consistent.
5669 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5673 # override offline migrated volumes, conf is out of date still
5674 if (my $offline_volumes = $migrate_opts->{offline_volumes
}) {
5675 for my $key (sort keys $offline_volumes->%*) {
5676 my $parsed = parse_drive
($key, $conf->{$key});
5677 $parsed->{file
} = $offline_volumes->{$key};
5678 $conf->{$key} = print_drive
($parsed);
5682 my $defaults = load_defaults
();
5684 # set environment variable useful inside network script
5685 # for remote migration the config is available on the target node!
5686 if (!$migrate_opts->{remote_node
}) {
5687 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom;
5690 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5692 my $forcemachine = $params->{forcemachine
};
5693 my $forcecpu = $params->{forcecpu
};
5695 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5696 $forcemachine = $conf->{runningmachine
};
5697 $forcecpu = $conf->{runningcpu
};
5698 print "Resuming suspended VM\n";
5701 my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command
($storecfg, $vmid,
5702 $conf, $defaults, $forcemachine, $forcecpu, $params->{'live-restore-backing'});
5705 my $get_migration_ip = sub {
5706 my ($nodename) = @_;
5708 return $migration_ip if defined($migration_ip);
5710 my $cidr = $migrate_opts->{network
};
5712 if (!defined($cidr)) {
5713 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5714 $cidr = $dc_conf->{migration
}->{network
};
5717 if (defined($cidr)) {
5718 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5720 die "could not get IP: no address configured on local " .
5721 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5723 die "could not get IP: multiple addresses configured on local " .
5724 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5726 $migration_ip = @$ips[0];
5729 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5730 if !defined($migration_ip);
5732 return $migration_ip;
5736 if ($statefile eq 'tcp') {
5737 my $migrate = $res->{migrate
} = { proto
=> 'tcp' };
5738 $migrate->{addr
} = "localhost";
5739 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5740 my $nodename = nodename
();
5742 if (!defined($migration_type)) {
5743 if (defined($datacenterconf->{migration
}->{type
})) {
5744 $migration_type = $datacenterconf->{migration
}->{type
};
5746 $migration_type = 'secure';
5750 if ($migration_type eq 'insecure') {
5751 $migrate->{addr
} = $get_migration_ip->($nodename);
5752 $migrate->{addr
} = "[$migrate->{addr}]" if Net
::IP
::ip_is_ipv6
($migrate->{addr
});
5755 # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5757 push @$cmd, '-incoming', 'defer';
5760 } elsif ($statefile eq 'unix') {
5761 # should be default for secure migrations as a ssh TCP forward
5762 # tunnel is not deterministic reliable ready and fails regurarly
5763 # to set up in time, so use UNIX socket forwards
5764 my $migrate = $res->{migrate
} = { proto
=> 'unix' };
5765 $migrate->{addr
} = "/run/qemu-server/$vmid.migrate";
5766 unlink $migrate->{addr
};
5768 $migrate->{uri
} = "unix:$migrate->{addr}";
5769 push @$cmd, '-incoming', $migrate->{uri
};
5772 } elsif (-e
$statefile) {
5773 push @$cmd, '-loadstate', $statefile;
5775 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5776 push @$vollist, $statefile;
5777 push @$cmd, '-loadstate', $statepath;
5779 } elsif ($params->{paused
}) {
5783 my $memory = get_current_memory
($conf->{memory
});
5784 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $memory, $resume);
5786 my $pci_reserve_list = [];
5787 for my $device (values $pci_devices->%*) {
5788 next if $device->{mdev
}; # we don't reserve for mdev devices
5789 push $pci_reserve_list->@*, map { $_->{id
} } $device->{ids
}->@*;
5792 # reserve all PCI IDs before actually doing anything with them
5793 PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, $start_timeout);
5797 for my $id (sort keys %$pci_devices) {
5798 my $d = $pci_devices->{$id};
5799 my ($index) = ($id =~ m/^hostpci(\d+)$/);
5802 for my $dev ($d->{ids
}->@*) {
5803 my $info = eval { PVE
::QemuServer
::PCI
::prepare_pci_device
($vmid, $dev->{id
}, $index, $d->{mdev
}) };
5806 $chosen_mdev = $info;
5807 last if $chosen_mdev; # if successful, we're done
5813 next if !$d->{mdev
};
5814 die "could not create mediated device\n" if !defined($chosen_mdev);
5816 # nvidia grid needs the uuid of the mdev as qemu parameter
5817 if (!defined($uuid) && $chosen_mdev->{vendor
} =~ m/^(0x)?10de$/) {
5818 if (defined($conf->{smbios1
})) {
5819 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
5820 $uuid = $smbios_conf->{uuid
} if defined($smbios_conf->{uuid
});
5822 $uuid = PVE
::QemuServer
::PCI
::generate_mdev_uuid
($vmid, $index) if !defined($uuid);
5825 push @$cmd, '-uuid', $uuid if defined($uuid);
5828 eval { cleanup_pci_devices
($vmid, $conf) };
5833 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5836 my %silence_std_outs = (outfunc
=> sub {}, errfunc
=> sub {});
5837 eval { run_command
(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5838 eval { run_command
(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5839 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5840 # timeout should be more than enough here...
5841 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 20);
5843 my $cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{cpuunits
});
5846 timeout
=> $statefile ?
undef : $start_timeout,
5851 # when migrating, prefix QEMU output so other side can pick up any
5852 # errors that might occur and show the user
5853 if ($migratedfrom) {
5854 $run_params{quiet
} = 1;
5855 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5858 my %systemd_properties = (
5859 Slice
=> 'qemu.slice',
5860 KillMode
=> 'process',
5862 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5865 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5866 $systemd_properties{CPUWeight
} = $cpuunits;
5868 $systemd_properties{CPUShares
} = $cpuunits;
5871 if (my $cpulimit = $conf->{cpulimit
}) {
5872 $systemd_properties{CPUQuota
} = int($cpulimit * 100);
5874 $systemd_properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5876 my $run_qemu = sub {
5877 PVE
::Tools
::run_fork
sub {
5878 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5881 if ((my $tpm = $conf->{tpmstate0
}) && !PVE
::QemuConfig-
>is_template($conf)) {
5882 # start the TPM emulator so QEMU can connect on start
5883 $tpmpid = start_swtpm
($storecfg, $vmid, $tpm, $migratedfrom);
5886 my $exitcode = run_command
($cmd, %run_params);
5889 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5890 kill 'TERM', $tpmpid;
5892 die "QEMU exited with code $exitcode\n";
5897 if ($conf->{hugepages
}) {
5900 my $hotplug_features =
5901 parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
5902 my $hugepages_topology =
5903 PVE
::QemuServer
::Memory
::hugepages_topology
($conf, $hotplug_features->{memory
});
5905 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5907 PVE
::QemuServer
::Memory
::hugepages_mount
();
5908 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5910 eval { $run_qemu->() };
5912 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5913 if !$conf->{keephugepages
};
5917 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5918 if !$conf->{keephugepages
};
5920 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5923 eval { $run_qemu->() };
5927 # deactivate volumes if start fails
5928 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5930 eval { cleanup_pci_devices
($vmid, $conf) };
5933 die "start failed: $err";
5936 # re-reserve all PCI IDs now that we can know the actual VM PID
5937 my $pid = PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
5938 eval { PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, undef, $pid) };
5941 if (defined(my $migrate = $res->{migrate
})) {
5942 if ($migrate->{proto
} eq 'tcp') {
5943 my $nodename = nodename
();
5944 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5945 $migrate->{port
} = PVE
::Tools
::next_migrate_port
($pfamily);
5946 $migrate->{uri
} = "tcp:$migrate->{addr}:$migrate->{port}";
5947 mon_cmd
($vmid, "migrate-incoming", uri
=> $migrate->{uri
});
5949 print "migration listens on $migrate->{uri}\n";
5950 } elsif ($statefile) {
5951 eval { mon_cmd
($vmid, "cont"); };
5955 #start nbd server for storage migration
5956 if (my $nbd = $migrate_opts->{nbd
}) {
5957 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5959 my $migrate_storage_uri;
5960 # nbd_protocol_version > 0 for unix socket support
5961 if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5962 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5963 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
5964 $migrate_storage_uri = "nbd:unix:$socket_path";
5965 $res->{migrate
}->{unix_sockets
} = [$socket_path];
5967 my $nodename = nodename
();
5968 my $localip = $get_migration_ip->($nodename);
5969 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5970 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5972 mon_cmd
($vmid, "nbd-server-start", addr
=> {
5975 host
=> "${localip}",
5976 port
=> "${storage_migrate_port}",
5979 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5980 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5983 my $block_info = mon_cmd
($vmid, "query-block");
5984 $block_info = { map { $_->{device
} => $_ } $block_info->@* };
5986 foreach my $opt (sort keys %$nbd) {
5987 my $drivestr = $nbd->{$opt}->{drivestr
};
5988 my $volid = $nbd->{$opt}->{volid
};
5990 my $block_node = $block_info->{"drive-$opt"}->{inserted
}->{'node-name'};
5996 'node-name' => $block_node,
5997 writable
=> JSON
::true
,
5999 name
=> "drive-$opt", # NBD export name
6002 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6003 print "storage migration listens on $nbd_uri volume:$drivestr\n";
6004 print "re-using replicated volume: $opt - $volid\n"
6005 if $nbd->{$opt}->{replicated
};
6007 $res->{drives
}->{$opt} = $nbd->{$opt};
6008 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
6012 if ($migratedfrom) {
6014 set_migration_caps
($vmid);
6019 print "spice listens on port $spice_port\n";
6020 $res->{spice_port
} = $spice_port;
6021 if ($migrate_opts->{spice_ticket
}) {
6022 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
6023 $migrate_opts->{spice_ticket
});
6024 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
6029 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
6030 if !$statefile && $conf->{balloon
};
6032 foreach my $opt (keys %$conf) {
6033 next if $opt !~ m/^net\d+$/;
6034 my $nicconf = parse_net
($conf->{$opt});
6035 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
6037 add_nets_bridge_fdb
($conf, $vmid);
6040 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
6045 path
=> "machine/peripheral/balloon0",
6046 property
=> "guest-stats-polling-interval",
6050 log_warn
("could not set polling interval for ballooning - $@") if $@;
6054 print "Resumed VM, removing state\n";
6055 if (my $vmstate = $conf->{vmstate
}) {
6056 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6057 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6059 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6060 PVE
::QemuConfig-
>write_config($vmid, $conf);
6063 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
6065 my ($current_machine, $is_deprecated) =
6066 PVE
::QemuServer
::Machine
::get_current_qemu_machine
($vmid);
6067 if ($is_deprecated) {
6069 "current machine version '$current_machine' is deprecated - see the documentation and ".
6070 "change to a newer one",
6077 sub vm_commandline
{
6078 my ($storecfg, $vmid, $snapname) = @_;
6080 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6082 my ($forcemachine, $forcecpu);
6084 my $snapshot = $conf->{snapshots
}->{$snapname};
6085 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6087 # check for machine or CPU overrides in snapshot
6088 $forcemachine = $snapshot->{runningmachine
};
6089 $forcecpu = $snapshot->{runningcpu
};
6091 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
6096 my $defaults = load_defaults
();
6098 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6100 return PVE
::Tools
::cmd2string
($cmd);
6104 my ($vmid, $skiplock) = @_;
6106 PVE
::QemuConfig-
>lock_config($vmid, sub {
6108 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6110 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6112 mon_cmd
($vmid, "system_reset");
6116 sub get_vm_volumes
{
6120 foreach_volid
($conf, sub {
6121 my ($volid, $attr) = @_;
6123 return if $volid =~ m
|^/|;
6125 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6128 push @$vollist, $volid;
6134 sub cleanup_pci_devices
{
6135 my ($vmid, $conf) = @_;
6137 foreach my $key (keys %$conf) {
6138 next if $key !~ m/^hostpci(\d+)$/;
6139 my $hostpciindex = $1;
6140 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
6141 my $d = parse_hostpci
($conf->{$key});
6143 # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6144 # don't want to break ABI just for this two liner
6145 my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6147 # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6148 # out when we do it first. so wait for up to 10 seconds and then try it manually
6149 if ($d->{ids
}->[0]->[0]->{vendor
} =~ m/^(0x)?10de$/ && -e
$dev_sysfs_dir) {
6151 while (-e
$dev_sysfs_dir && $count < 10) {
6155 print "waited $count seconds for mediated device driver finishing clean up\n";
6158 if (-e
$dev_sysfs_dir) {
6159 print "actively clean up mediated device with UUID $uuid\n";
6160 PVE
::SysFSTools
::file_write
("$dev_sysfs_dir/remove", "1");
6164 PVE
::QemuServer
::PCI
::remove_pci_reservation
($vmid);
6167 sub vm_stop_cleanup
{
6168 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6173 my $vollist = get_vm_volumes
($conf);
6174 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6176 if (my $tpmdrive = $conf->{tpmstate0
}) {
6177 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
6178 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
6180 PVE
::Storage
::unmap_volume
($storecfg, $tpm->{file
});
6185 foreach my $ext (qw(mon qmp pid vnc qga)) {
6186 unlink "/var/run/qemu-server/${vmid}.$ext";
6189 if ($conf->{ivshmem
}) {
6190 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
6191 # just delete it for now, VMs which have this already open do not
6192 # are affected, but new VMs will get a separated one. If this
6193 # becomes an issue we either add some sort of ref-counting or just
6194 # add a "don't delete on stop" flag to the ivshmem format.
6195 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
6198 cleanup_pci_devices
($vmid, $conf);
6200 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
6202 warn $@ if $@; # avoid errors - just warn
6205 # call only in locked context
6207 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6209 my $pid = check_running
($vmid, $nocheck);
6214 $conf = PVE
::QemuConfig-
>load_config($vmid);
6215 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6216 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
6217 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
6218 $timeout = $opts->{down
} if $opts->{down
};
6220 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
6225 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
6226 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
6228 mon_cmd
($vmid, "system_powerdown");
6231 mon_cmd
($vmid, "quit");
6237 $timeout = 60 if !defined($timeout);
6240 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6245 if ($count >= $timeout) {
6247 warn "VM still running - terminating now with SIGTERM\n";
6250 die "VM quit/powerdown failed - got timeout\n";
6253 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6257 if (!check_running
($vmid, $nocheck)) {
6258 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6262 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6265 die "VM quit/powerdown failed\n";
6273 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6278 if ($count >= $timeout) {
6279 warn "VM still running - terminating now with SIGKILL\n";
6284 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6287 # Note: use $nocheck to skip tests if VM configuration file exists.
6288 # We need that when migration VMs to other nodes (files already moved)
6289 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6291 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6293 $force = 1 if !defined($force) && !$shutdown;
6296 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
6297 kill 15, $pid if $pid;
6298 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
6299 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
6303 PVE
::QemuConfig-
>lock_config($vmid, sub {
6304 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6309 my ($vmid, $timeout) = @_;
6311 PVE
::QemuConfig-
>lock_config($vmid, sub {
6314 # only reboot if running, as qmeventd starts it again on a stop event
6315 return if !check_running
($vmid);
6317 create_reboot_request
($vmid);
6319 my $storecfg = PVE
::Storage
::config
();
6320 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
6324 # avoid that the next normal shutdown will be confused for a reboot
6325 clear_reboot_request
($vmid);
6331 # note: if using the statestorage parameter, the caller has to check privileges
6333 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6340 PVE
::QemuConfig-
>lock_config($vmid, sub {
6342 $conf = PVE
::QemuConfig-
>load_config($vmid);
6344 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
6345 PVE
::QemuConfig-
>check_lock($conf)
6346 if !($skiplock || $is_backing_up);
6348 die "cannot suspend to disk during backup\n"
6349 if $is_backing_up && $includestate;
6351 if ($includestate) {
6352 $conf->{lock} = 'suspending';
6353 my $date = strftime
("%Y-%m-%d", localtime(time()));
6354 $storecfg = PVE
::Storage
::config
();
6355 if (!$statestorage) {
6356 $statestorage = find_vmstate_storage
($conf, $storecfg);
6357 # check permissions for the storage
6358 my $rpcenv = PVE
::RPCEnvironment
::get
();
6359 if ($rpcenv->{type
} ne 'cli') {
6360 my $authuser = $rpcenv->get_user();
6361 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6366 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
6367 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6368 $path = PVE
::Storage
::path
($storecfg, $vmstate);
6369 PVE
::QemuConfig-
>write_config($vmid, $conf);
6371 mon_cmd
($vmid, "stop");
6375 if ($includestate) {
6377 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
6380 set_migration_caps
($vmid, 1);
6381 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
6383 my $state = mon_cmd
($vmid, "query-savevm");
6384 if (!$state->{status
}) {
6385 die "savevm not active\n";
6386 } elsif ($state->{status
} eq 'active') {
6389 } elsif ($state->{status
} eq 'completed') {
6390 print "State saved, quitting\n";
6392 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
6393 die "query-savevm failed with error '$state->{error}'\n"
6395 die "query-savevm returned status '$state->{status}'\n";
6401 PVE
::QemuConfig-
>lock_config($vmid, sub {
6402 $conf = PVE
::QemuConfig-
>load_config($vmid);
6404 # cleanup, but leave suspending lock, to indicate something went wrong
6406 mon_cmd
($vmid, "savevm-end");
6407 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6408 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6409 delete $conf->@{qw(vmstate runningmachine runningcpu)};
6410 PVE
::QemuConfig-
>write_config($vmid, $conf);
6416 die "lock changed unexpectedly\n"
6417 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
6419 mon_cmd
($vmid, "quit");
6420 $conf->{lock} = 'suspended';
6421 PVE
::QemuConfig-
>write_config($vmid, $conf);
6426 # $nocheck is set when called as part of a migration - in this context the
6427 # location of the config file (source or target node) is not deterministic,
6428 # since migration cannot wait for pmxcfs to process the rename
6430 my ($vmid, $skiplock, $nocheck) = @_;
6432 PVE
::QemuConfig-
>lock_config($vmid, sub {
6433 my $res = mon_cmd
($vmid, 'query-status');
6434 my $resume_cmd = 'cont';
6438 $conf = eval { PVE
::QemuConfig-
>load_config($vmid) }; # try on target node
6440 my $vmlist = PVE
::Cluster
::get_vmlist
();
6441 if (exists($vmlist->{ids
}->{$vmid})) {
6442 my $node = $vmlist->{ids
}->{$vmid}->{node
};
6443 $conf = eval { PVE
::QemuConfig-
>load_config($vmid, $node) }; # try on source node
6446 PVE
::Cluster
::cfs_update
(); # vmlist was wrong, invalidate cache
6447 $conf = PVE
::QemuConfig-
>load_config($vmid); # last try on target node again
6451 $conf = PVE
::QemuConfig-
>load_config($vmid);
6454 if ($res->{status
}) {
6455 return if $res->{status
} eq 'running'; # job done, go home
6456 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
6457 $reset = 1 if $res->{status
} eq 'shutdown';
6461 PVE
::QemuConfig-
>check_lock($conf)
6462 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
6466 # required if a VM shuts down during a backup and we get a resume
6467 # request before the backup finishes for example
6468 mon_cmd
($vmid, "system_reset");
6471 add_nets_bridge_fdb
($conf, $vmid) if $resume_cmd eq 'cont';
6473 mon_cmd
($vmid, $resume_cmd);
6478 my ($vmid, $skiplock, $key) = @_;
6480 PVE
::QemuConfig-
>lock_config($vmid, sub {
6482 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6484 # there is no qmp command, so we use the human monitor command
6485 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
6486 die $res if $res ne '';
6490 sub check_bridge_access
{
6491 my ($rpcenv, $authuser, $conf) = @_;
6493 return 1 if $authuser eq 'root@pam';
6495 for my $opt (sort keys $conf->%*) {
6496 next if $opt !~ m/^net\d+$/;
6497 my $net = parse_net
($conf->{$opt});
6498 my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6499 PVE
::GuestHelpers
::check_vnet_access
($rpcenv, $authuser, $bridge, $tag, $trunks);
6504 sub check_mapping_access
{
6505 my ($rpcenv, $user, $conf) = @_;
6507 for my $opt (keys $conf->%*) {
6508 if ($opt =~ m/^usb\d+$/) {
6509 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-usb', $conf->{$opt});
6510 if (my $host = $device->{host
}) {
6511 die "only root can set '$opt' config for real devices\n"
6512 if $host !~ m/^spice$/i && $user ne 'root@pam';
6513 } elsif ($device->{mapping
}) {
6514 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6516 die "either 'host' or 'mapping' must be set.\n";
6518 } elsif ($opt =~ m/^hostpci\d+$/) {
6519 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-hostpci', $conf->{$opt});
6520 if ($device->{host
}) {
6521 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6522 } elsif ($device->{mapping
}) {
6523 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6525 die "either 'host' or 'mapping' must be set.\n";
6531 sub check_restore_permissions
{
6532 my ($rpcenv, $user, $conf) = @_;
6534 check_bridge_access
($rpcenv, $user, $conf);
6535 check_mapping_access
($rpcenv, $user, $conf);
6537 # vzdump restore implementaion
6539 sub tar_archive_read_firstfile
{
6540 my $archive = shift;
6542 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
6544 # try to detect archive type first
6545 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6546 die "unable to open file '$archive'\n";
6547 my $firstfile = <$fh>;
6551 die "ERROR: archive contaions no data\n" if !$firstfile;
6557 sub tar_restore_cleanup
{
6558 my ($storecfg, $statfile) = @_;
6560 print STDERR
"starting cleanup\n";
6562 if (my $fd = IO
::File-
>new($statfile, "r")) {
6563 while (defined(my $line = <$fd>)) {
6564 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6567 if ($volid =~ m
|^/|) {
6568 unlink $volid || die 'unlink failed\n';
6570 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6572 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6574 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6576 print STDERR
"unable to parse line in statfile - $line";
6583 sub restore_file_archive
{
6584 my ($archive, $vmid, $user, $opts) = @_;
6586 return restore_vma_archive
($archive, $vmid, $user, $opts)
6589 my $info = PVE
::Storage
::archive_info
($archive);
6590 my $format = $opts->{format
} // $info->{format
};
6591 my $comp = $info->{compression
};
6593 # try to detect archive format
6594 if ($format eq 'tar') {
6595 return restore_tar_archive
($archive, $vmid, $user, $opts);
6597 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
6601 # hepler to remove disks that will not be used after restore
6602 my $restore_cleanup_oldconf = sub {
6603 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6605 my $kept_disks = {};
6607 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
6608 my ($ds, $drive) = @_;
6610 return if drive_is_cdrom
($drive, 1);
6612 my $volid = $drive->{file
};
6613 return if !$volid || $volid =~ m
|^/|;
6615 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
6616 return if !$path || !$owner || ($owner != $vmid);
6618 # Note: only delete disk we want to restore
6619 # other volumes will become unused
6620 if ($virtdev_hash->{$ds}) {
6621 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
6626 $kept_disks->{$volid} = 1;
6630 # after the restore we have no snapshots anymore
6631 for my $snapname (keys $oldconf->{snapshots
}->%*) {
6632 my $snap = $oldconf->{snapshots
}->{$snapname};
6633 if ($snap->{vmstate
}) {
6634 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
6640 for my $volid (keys $kept_disks->%*) {
6641 eval { PVE
::Storage
::volume_snapshot_delete
($storecfg, $volid, $snapname); };
6647 # Helper to parse vzdump backup device hints
6649 # $rpcenv: Environment, used to ckeck storage permissions
6650 # $user: User ID, to check storage permissions
6651 # $storecfg: Storage configuration
6652 # $fh: the file handle for reading the configuration
6653 # $devinfo: should contain device sizes for all backu-up'ed devices
6654 # $options: backup options (pool, default storage)
6656 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6657 my $parse_backup_hints = sub {
6658 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6660 my $check_storage = sub { # assert if an image can be allocate
6661 my ($storeid, $scfg) = @_;
6662 die "Content type 'images' is not available on storage '$storeid'\n"
6663 if !$scfg->{content
}->{images
};
6664 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6665 if $user ne 'root@pam';
6668 my $virtdev_hash = {};
6669 while (defined(my $line = <$fh>)) {
6670 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6671 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6672 die "archive does not contain data for drive '$virtdev'\n"
6673 if !$devinfo->{$devname};
6675 if (defined($options->{storage
})) {
6676 $storeid = $options->{storage
} || 'local';
6677 } elsif (!$storeid) {
6680 $format = 'raw' if !$format;
6681 $devinfo->{$devname}->{devname
} = $devname;
6682 $devinfo->{$devname}->{virtdev
} = $virtdev;
6683 $devinfo->{$devname}->{format
} = $format;
6684 $devinfo->{$devname}->{storeid
} = $storeid;
6686 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6687 $check_storage->($storeid, $scfg); # permission and content type check
6689 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6690 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6692 my $drive = parse_drive
($virtdev, $2);
6694 if (drive_is_cloudinit
($drive)) {
6695 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6696 $storeid = $options->{storage
} if defined ($options->{storage
});
6697 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6698 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6700 $check_storage->($storeid, $scfg); # permission and content type check
6702 $virtdev_hash->{$virtdev} = {
6704 storeid
=> $storeid,
6705 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6712 return $virtdev_hash;
6715 # Helper to allocate and activate all volumes required for a restore
6717 # $storecfg: Storage configuration
6718 # $virtdev_hash: as returned by parse_backup_hints()
6720 # Returns: { $virtdev => $volid }
6721 my $restore_allocate_devices = sub {
6722 my ($storecfg, $virtdev_hash, $vmid) = @_;
6725 foreach my $virtdev (sort keys %$virtdev_hash) {
6726 my $d = $virtdev_hash->{$virtdev};
6727 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6728 my $storeid = $d->{storeid
};
6729 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6731 # test if requested format is supported
6732 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6733 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6734 $d->{format
} = $defFormat if !$supported;
6737 if ($d->{is_cloudinit
}) {
6738 $name = "vm-$vmid-cloudinit";
6739 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6740 if ($scfg->{path
}) {
6741 $name .= ".$d->{format}";
6745 my $volid = PVE
::Storage
::vdisk_alloc
(
6746 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6748 print STDERR
"new volume ID is '$volid'\n";
6749 $d->{volid
} = $volid;
6751 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6753 $map->{$virtdev} = $volid;
6759 sub restore_update_config_line
{
6760 my ($cookie, $map, $line, $unique) = @_;
6762 return '' if $line =~ m/^\#qmdump\#/;
6763 return '' if $line =~ m/^\#vzdump\#/;
6764 return '' if $line =~ m/^lock:/;
6765 return '' if $line =~ m/^unused\d+:/;
6766 return '' if $line =~ m/^parent:/;
6770 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6771 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6772 # try to convert old 1.X settings
6773 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6774 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6775 my ($model, $macaddr) = split(/\=/, $devconfig);
6776 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6779 bridge
=> "vmbr$ind",
6780 macaddr
=> $macaddr,
6782 my $netstr = print_net
($net);
6784 $res .= "net$cookie->{netcount}: $netstr\n";
6785 $cookie->{netcount
}++;
6787 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6788 my ($id, $netstr) = ($1, $2);
6789 my $net = parse_net
($netstr);
6790 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6791 $netstr = print_net
($net);
6792 $res .= "$id: $netstr\n";
6793 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6796 my $di = parse_drive
($virtdev, $value);
6797 if (defined($di->{backup
}) && !$di->{backup
}) {
6799 } elsif ($map->{$virtdev}) {
6800 delete $di->{format
}; # format can change on restore
6801 $di->{file
} = $map->{$virtdev};
6802 $value = print_drive
($di);
6803 $res .= "$virtdev: $value\n";
6807 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6809 if ($vmgenid ne '0') {
6810 # always generate a new vmgenid if there was a valid one setup
6811 $vmgenid = generate_uuid
();
6813 $res .= "vmgenid: $vmgenid\n";
6814 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6815 my ($uuid, $uuid_str);
6816 UUID
::generate
($uuid);
6817 UUID
::unparse
($uuid, $uuid_str);
6818 my $smbios1 = parse_smbios1
($2);
6819 $smbios1->{uuid
} = $uuid_str;
6820 $res .= $1.print_smbios1
($smbios1)."\n";
6828 my $restore_deactivate_volumes = sub {
6829 my ($storecfg, $virtdev_hash) = @_;
6832 for my $dev (values $virtdev_hash->%*) {
6833 push $vollist->@*, $dev->{volid
} if $dev->{volid
};
6836 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
6837 print STDERR
$@ if $@;
6840 my $restore_destroy_volumes = sub {
6841 my ($storecfg, $virtdev_hash) = @_;
6843 for my $dev (values $virtdev_hash->%*) {
6844 my $volid = $dev->{volid
} or next;
6846 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6847 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6849 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6853 sub restore_merge_config
{
6854 my ($filename, $backup_conf_raw, $override_conf) = @_;
6856 my $backup_conf = parse_vm_config
($filename, $backup_conf_raw);
6857 for my $key (keys $override_conf->%*) {
6858 $backup_conf->{$key} = $override_conf->{$key};
6861 return $backup_conf;
6865 my ($cfg, $vmid) = @_;
6867 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6869 my $volid_hash = {};
6870 foreach my $storeid (keys %$info) {
6871 foreach my $item (@{$info->{$storeid}}) {
6872 next if !($item->{volid
} && $item->{size
});
6873 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6874 $volid_hash->{$item->{volid
}} = $item;
6881 sub update_disk_config
{
6882 my ($vmid, $conf, $volid_hash) = @_;
6885 my $prefix = "VM $vmid";
6887 # used and unused disks
6888 my $referenced = {};
6890 # Note: it is allowed to define multiple storages with same path (alias), so
6891 # we need to check both 'volid' and real 'path' (two different volid can point
6892 # to the same path).
6894 my $referencedpath = {};
6897 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6898 my ($opt, $drive) = @_;
6900 my $volid = $drive->{file
};
6902 my $volume = $volid_hash->{$volid};
6904 # mark volid as "in-use" for next step
6905 $referenced->{$volid} = 1;
6906 if ($volume && (my $path = $volume->{path
})) {
6907 $referencedpath->{$path} = 1;
6910 return if drive_is_cdrom
($drive);
6913 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6914 if (defined($updated)) {
6916 $conf->{$opt} = print_drive
($updated);
6917 print "$prefix ($opt): $msg\n";
6921 # remove 'unusedX' entry if volume is used
6922 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6923 my ($opt, $drive) = @_;
6925 my $volid = $drive->{file
};
6929 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6930 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6931 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6933 delete $conf->{$opt};
6936 $referenced->{$volid} = 1;
6937 $referencedpath->{$path} = 1 if $path;
6940 foreach my $volid (sort keys %$volid_hash) {
6941 next if $volid =~ m/vm-$vmid-state-/;
6942 next if $referenced->{$volid};
6943 my $path = $volid_hash->{$volid}->{path
};
6944 next if !$path; # just to be sure
6945 next if $referencedpath->{$path};
6947 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6948 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6949 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6956 my ($vmid, $nolock, $dryrun) = @_;
6958 my $cfg = PVE
::Storage
::config
();
6960 print "rescan volumes...\n";
6961 my $volid_hash = scan_volids
($cfg, $vmid);
6963 my $updatefn = sub {
6966 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6968 PVE
::QemuConfig-
>check_lock($conf);
6971 foreach my $volid (keys %$volid_hash) {
6972 my $info = $volid_hash->{$volid};
6973 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6976 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6978 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6981 if (defined($vmid)) {
6985 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6988 my $vmlist = config_list
();
6989 foreach my $vmid (keys %$vmlist) {
6993 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6999 sub restore_proxmox_backup_archive
{
7000 my ($archive, $vmid, $user, $options) = @_;
7002 my $storecfg = PVE
::Storage
::config
();
7004 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
7005 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7007 my $fingerprint = $scfg->{fingerprint
};
7008 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
7010 my $repo = PVE
::PBSClient
::get_repository
($scfg);
7011 my $namespace = $scfg->{namespace
};
7013 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7014 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
7015 local $ENV{PBS_PASSWORD
} = $password;
7016 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
7018 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7019 PVE
::Storage
::parse_volname
($storecfg, $archive);
7021 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7023 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7025 my $tmpdir = "/var/tmp/vzdumptmp$$";
7029 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7030 # disable interrupts (always do cleanups)
7034 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7036 # Note: $oldconf is undef if VM does not exists
7037 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7038 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7039 my $new_conf_raw = '';
7041 my $rpcenv = PVE
::RPCEnvironment
::get
();
7042 my $devinfo = {}; # info about drives included in backup
7043 my $virtdev_hash = {}; # info about allocated drives
7051 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7053 my $cfgfn = "$tmpdir/qemu-server.conf";
7054 my $firewall_config_fn = "$tmpdir/fw.conf";
7055 my $index_fn = "$tmpdir/index.json";
7057 my $cmd = "restore";
7059 my $param = [$pbs_backup_name, "index.json", $index_fn];
7060 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7061 my $index = PVE
::Tools
::file_get_contents
($index_fn);
7062 $index = decode_json
($index);
7064 foreach my $info (@{$index->{files
}}) {
7065 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
7067 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
7068 $devinfo->{$devname}->{size
} = $1;
7070 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7075 my $is_qemu_server_backup = scalar(
7076 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
7078 if (!$is_qemu_server_backup) {
7079 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7081 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
7083 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7084 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7086 if ($has_firewall_config) {
7087 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7088 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7090 my $pve_firewall_dir = '/etc/pve/firewall';
7091 mkdir $pve_firewall_dir; # make sure the dir exists
7092 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7095 my $fh = IO
::File-
>new($cfgfn, "r") ||
7096 die "unable to read qemu-server.conf - $!\n";
7098 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7100 # fixme: rate limit?
7102 # create empty/temp config
7103 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
7105 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7108 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7110 foreach my $virtdev (sort keys %$virtdev_hash) {
7111 my $d = $virtdev_hash->{$virtdev};
7112 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7114 # this fails if storage is unavailable
7115 my $volid = $d->{volid
};
7116 my $path = PVE
::Storage
::path
($storecfg, $volid);
7118 # for live-restore we only want to preload the efidisk and TPM state
7119 next if $options->{live
} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7122 if (defined(my $ns = $scfg->{namespace
})) {
7123 @ns_arg = ('--ns', $ns);
7126 my $pbs_restore_cmd = [
7127 '/usr/bin/pbs-restore',
7128 '--repository', $repo,
7131 "$d->{devname}.img.fidx",
7136 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
7137 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
7139 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
7140 push @$pbs_restore_cmd, '--skip-zero';
7143 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
7144 print "restore proxmox backup image: $dbg_cmdstring\n";
7145 run_command
($pbs_restore_cmd);
7148 $fh->seek(0, 0) || die "seek failed - $!\n";
7150 my $cookie = { netcount
=> 0 };
7151 while (defined(my $line = <$fh>)) {
7152 $new_conf_raw .= restore_update_config_line
(
7164 if ($err || !$options->{live
}) {
7165 $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7171 $restore_destroy_volumes->($storecfg, $virtdev_hash);
7175 if ($options->{live
}) {
7176 # keep lock during live-restore
7177 $new_conf_raw .= "\nlock: create";
7180 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $options->{override_conf
});
7181 check_restore_permissions
($rpcenv, $user, $new_conf);
7182 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7184 eval { rescan
($vmid, 1); };
7187 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
7189 if ($options->{live
}) {
7195 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
7197 my $conf = PVE
::QemuConfig-
>load_config($vmid);
7198 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
7200 # these special drives are already restored before start
7201 delete $devinfo->{'drive-efidisk0'};
7202 delete $devinfo->{'drive-tpmstate0-backup'};
7206 keyfile
=> $keyfile,
7207 snapshot
=> $pbs_backup_name,
7208 namespace
=> $namespace,
7210 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7212 PVE
::QemuConfig-
>remove_lock($vmid, "create");
7216 sub pbs_live_restore
{
7217 my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7219 print "starting VM for live-restore\n";
7220 print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7222 my $live_restore_backing = {};
7223 for my $ds (keys %$restored_disks) {
7224 $ds =~ m/^drive-(.*)$/;
7228 repository
=> $opts->{repo
},
7229 snapshot
=> $opts->{snapshot
},
7230 archive
=> "$ds.img.fidx",
7232 $pbs_conf->{keyfile
} = $opts->{keyfile
} if -e
$opts->{keyfile
};
7233 $pbs_conf->{namespace
} = $opts->{namespace
} if defined($opts->{namespace
});
7235 my $drive = parse_drive
($confname, $conf->{$confname});
7236 print "restoring '$ds' to '$drive->{file}'\n";
7238 my $pbs_name = "drive-${confname}-pbs";
7239 $live_restore_backing->{$confname} = {
7241 blockdev
=> print_pbs_blockdev
($pbs_conf, $pbs_name),
7245 my $drives_streamed = 0;
7247 # make sure HA doesn't interrupt our restore by stopping the VM
7248 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
7249 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7252 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7253 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7254 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'live-restore-backing' => $live_restore_backing}, {});
7256 my $qmeventd_fd = register_qmeventd_handle
($vmid);
7258 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7259 # this will effectively collapse the backing image chain consisting of
7260 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7261 # removes itself once all backing images vanish with 'auto-remove=on')
7263 for my $ds (sort keys %$restored_disks) {
7264 my $job_id = "restore-$ds";
7265 mon_cmd
($vmid, 'block-stream',
7266 'job-id' => $job_id,
7269 $jobs->{$job_id} = {};
7272 mon_cmd
($vmid, 'cont');
7273 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
7275 print "restore-drive jobs finished successfully, removing all tracking block devices"
7276 ." to disconnect from Proxmox Backup Server\n";
7278 for my $ds (sort keys %$restored_disks) {
7279 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7282 close($qmeventd_fd);
7288 warn "An error occurred during live-restore: $err\n";
7289 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
7290 die "live-restore failed\n";
7294 # Inspired by pbs live-restore, this restores with the disks being available as files.
7295 # Theoretically this can also be used to quick-start a full-clone vm if the
7296 # disks are all available as files.
7298 # The mapping should provide a path by config entry, such as
7299 # `{ scsi0 => { format => <qcow2|raw|...>, path => "/path/to/file", sata1 => ... } }`
7301 # This is used when doing a `create` call with the `--live-import` parameter,
7302 # where the disks get an `import-from=` property. The non-live part is
7303 # therefore already handled in the `$create_disks()` call happening in the
7305 sub live_import_from_files
{
7306 my ($mapping, $vmid, $conf, $restore_options) = @_;
7308 my $live_restore_backing = {};
7309 for my $dev (keys %$mapping) {
7310 die "disk not support for live-restoring: '$dev'\n"
7311 if !is_valid_drivename
($dev) || $dev =~ /^(?:efidisk|tpmstate)/;
7313 die "mapping contains disk '$dev' which does not exist in the config\n"
7314 if !exists($conf->{$dev});
7316 my $info = $mapping->{$dev};
7317 my ($format, $path) = $info->@{qw(format path)};
7318 die "missing path for '$dev' mapping\n" if !$path;
7319 die "missing format for '$dev' mapping\n" if !$format;
7320 die "invalid format '$format' for '$dev' mapping\n"
7321 if !grep { $format eq $_ } qw(raw qcow2 vmdk);
7323 $live_restore_backing->{$dev} = {
7324 name
=> "drive-$dev-restore",
7325 blockdev
=> "driver=$format,node-name=drive-$dev-restore"
7327 . ",file.driver=file,file.filename=$path"
7331 my $storecfg = PVE
::Storage
::config
();
7334 # make sure HA doesn't interrupt our restore by stopping the VM
7335 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
7336 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7339 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'live-restore-backing' => $live_restore_backing}, {});
7341 # prevent shutdowns from qmeventd when the VM powers off from the inside
7342 my $qmeventd_fd = register_qmeventd_handle
($vmid);
7344 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7345 # this will effectively collapse the backing image chain consisting of
7346 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7347 # removes itself once all backing images vanish with 'auto-remove=on')
7349 for my $ds (sort keys %$live_restore_backing) {
7350 my $job_id = "restore-$ds";
7351 mon_cmd
($vmid, 'block-stream',
7352 'job-id' => $job_id,
7353 device
=> "drive-$ds",
7355 $jobs->{$job_id} = {};
7358 mon_cmd
($vmid, 'cont');
7359 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
7361 print "restore-drive jobs finished successfully, removing all tracking block devices\n";
7363 for my $ds (sort keys %$live_restore_backing) {
7364 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "drive-$ds-restore");
7367 close($qmeventd_fd);
7373 warn "An error occurred during live-restore: $err\n";
7374 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
7375 die "live-restore failed\n";
7378 PVE
::QemuConfig-
>remove_lock($vmid, "import");
7381 sub restore_vma_archive
{
7382 my ($archive, $vmid, $user, $opts, $comp) = @_;
7384 my $readfrom = $archive;
7386 my $cfg = PVE
::Storage
::config
();
7388 my $bwlimit = $opts->{bwlimit
};
7390 my $dbg_cmdstring = '';
7391 my $add_pipe = sub {
7393 push @$commands, $cmd;
7394 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7395 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
7400 if ($archive eq '-') {
7403 # If we use a backup from a PVE defined storage we also consider that
7404 # storage's rate limit:
7405 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
7406 if (defined($volid)) {
7407 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
7408 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
7410 print STDERR
"applying read rate limit: $readlimit\n";
7411 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7412 $add_pipe->($cstream);
7418 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
7419 my $cmd = $info->{decompressor
};
7420 push @$cmd, $readfrom;
7424 my $tmpdir = "/var/tmp/vzdumptmp$$";
7427 # disable interrupts (always do cleanups)
7431 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
7433 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7434 POSIX
::mkfifo
($mapfifo, 0600);
7436 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7438 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7440 my $devinfo = {}; # info about drives included in backup
7441 my $virtdev_hash = {}; # info about allocated drives
7443 my $rpcenv = PVE
::RPCEnvironment
::get
();
7445 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7447 # Note: $oldconf is undef if VM does not exist
7448 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7449 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7450 my $new_conf_raw = '';
7454 my $print_devmap = sub {
7455 my $cfgfn = "$tmpdir/qemu-server.conf";
7457 # we can read the config - that is already extracted
7458 my $fh = IO
::File-
>new($cfgfn, "r") ||
7459 die "unable to read qemu-server.conf - $!\n";
7461 my $fwcfgfn = "$tmpdir/qemu-server.fw";
7463 my $pve_firewall_dir = '/etc/pve/firewall';
7464 mkdir $pve_firewall_dir; # make sure the dir exists
7465 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7468 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7470 foreach my $info (values %{$virtdev_hash}) {
7471 my $storeid = $info->{storeid
};
7472 next if defined($storage_limits{$storeid});
7474 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
7475 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7476 $storage_limits{$storeid} = $limit * 1024;
7479 foreach my $devname (keys %$devinfo) {
7480 die "found no device mapping information for device '$devname'\n"
7481 if !$devinfo->{$devname}->{virtdev
};
7484 # create empty/temp config
7486 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
7487 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7491 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7493 # print restore information to $fifofh
7494 foreach my $virtdev (sort keys %$virtdev_hash) {
7495 my $d = $virtdev_hash->{$virtdev};
7496 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7498 my $storeid = $d->{storeid
};
7499 my $volid = $d->{volid
};
7502 if (my $limit = $storage_limits{$storeid}) {
7503 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7506 my $write_zeros = 1;
7507 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
7511 my $path = PVE
::Storage
::path
($cfg, $volid);
7513 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7515 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7518 $fh->seek(0, 0) || die "seek failed - $!\n";
7520 my $cookie = { netcount
=> 0 };
7521 while (defined(my $line = <$fh>)) {
7522 $new_conf_raw .= restore_update_config_line
(
7541 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7542 local $SIG{ALRM
} = sub { die "got timeout\n"; };
7544 $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7551 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7552 my ($dev_id, $size, $devname) = ($1, $2, $3);
7553 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
7554 } elsif ($line =~ m/^CTIME: /) {
7555 # we correctly received the vma config, so we can disable
7556 # the timeout now for disk allocation
7557 alarm($oldtimeout || 0);
7558 $oldtimeout = undef;
7560 print $fifofh "done\n";
7566 print "restore vma archive: $dbg_cmdstring\n";
7567 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
7571 alarm($oldtimeout) if $oldtimeout;
7573 $restore_deactivate_volumes->($cfg, $virtdev_hash);
7575 close($fifofh) if $fifofh;
7580 $restore_destroy_volumes->($cfg, $virtdev_hash);
7584 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $opts->{override_conf
});
7585 check_restore_permissions
($rpcenv, $user, $new_conf);
7586 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7588 eval { rescan
($vmid, 1); };
7591 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
7594 sub restore_tar_archive
{
7595 my ($archive, $vmid, $user, $opts) = @_;
7597 if (scalar(keys $opts->{override_conf
}->%*) > 0) {
7598 my $keystring = join(' ', keys $opts->{override_conf
}->%*);
7599 die "cannot pass along options ($keystring) when restoring from tar archive\n";
7602 if ($archive ne '-') {
7603 my $firstfile = tar_archive_read_firstfile
($archive);
7604 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7605 if $firstfile ne 'qemu-server.conf';
7608 my $storecfg = PVE
::Storage
::config
();
7610 # avoid zombie disks when restoring over an existing VM -> cleanup first
7611 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7612 # skiplock=1 because qmrestore has set the 'create' lock itself already
7613 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
7614 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
7616 my $tocmd = "/usr/lib/qemu-server/qmextract";
7618 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
7619 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
7620 $tocmd .= ' --prealloc' if $opts->{prealloc
};
7621 $tocmd .= ' --info' if $opts->{info
};
7623 # tar option "xf" does not autodetect compression when read from STDIN,
7624 # so we pipe to zcat
7625 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
7626 PVE
::Tools
::shellquote
("--to-command=$tocmd");
7628 my $tmpdir = "/var/tmp/vzdumptmp$$";
7631 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
7632 local $ENV{VZDUMP_VMID
} = $vmid;
7633 local $ENV{VZDUMP_USER
} = $user;
7635 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7636 my $new_conf_raw = '';
7638 # disable interrupts (always do cleanups)
7642 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7650 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7652 if ($archive eq '-') {
7653 print "extracting archive from STDIN\n";
7654 run_command
($cmd, input
=> "<&STDIN");
7656 print "extracting archive '$archive'\n";
7660 return if $opts->{info
};
7664 my $statfile = "$tmpdir/qmrestore.stat";
7665 if (my $fd = IO
::File-
>new($statfile, "r")) {
7666 while (defined (my $line = <$fd>)) {
7667 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7668 $map->{$1} = $2 if $1;
7670 print STDERR
"unable to parse line in statfile - $line\n";
7676 my $confsrc = "$tmpdir/qemu-server.conf";
7678 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7680 my $cookie = { netcount
=> 0 };
7681 while (defined (my $line = <$srcfd>)) {
7682 $new_conf_raw .= restore_update_config_line
(
7693 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
7699 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7701 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7703 eval { rescan
($vmid, 1); };
7707 sub foreach_storage_used_by_vm
{
7708 my ($conf, $func) = @_;
7712 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7713 my ($ds, $drive) = @_;
7714 return if drive_is_cdrom
($drive);
7716 my $volid = $drive->{file
};
7718 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
7719 $sidhash->{$sid} = $sid if $sid;
7722 foreach my $sid (sort keys %$sidhash) {
7727 my $qemu_snap_storage = {
7730 sub do_snapshots_with_qemu
{
7731 my ($storecfg, $volid, $deviceid) = @_;
7733 return if $deviceid =~ m/tpmstate0/;
7735 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
7736 my $scfg = $storecfg->{ids
}->{$storage_name};
7737 die "could not find storage '$storage_name'\n" if !defined($scfg);
7739 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
7743 if ($volid =~ m/\.(qcow2|qed)$/){
7750 sub qga_check_running
{
7751 my ($vmid, $nowarn) = @_;
7753 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
7755 warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7761 sub template_create
{
7762 my ($vmid, $conf, $disk) = @_;
7764 my $storecfg = PVE
::Storage
::config
();
7766 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7767 my ($ds, $drive) = @_;
7769 return if drive_is_cdrom
($drive);
7770 return if $disk && $ds ne $disk;
7772 my $volid = $drive->{file
};
7773 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
7775 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
7776 $drive->{file
} = $voliddst;
7777 $conf->{$ds} = print_drive
($drive);
7778 PVE
::QemuConfig-
>write_config($vmid, $conf);
7782 sub convert_iscsi_path
{
7785 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
7790 my $initiator_name = get_initiator_name
();
7792 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7793 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7796 die "cannot convert iscsi path '$path', unkown format\n";
7799 sub qemu_img_convert
{
7800 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7802 my $storecfg = PVE
::Storage
::config
();
7803 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
7804 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
7806 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7810 my $src_is_iscsi = 0;
7814 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
7815 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7816 $src_format = qemu_img_format
($src_scfg, $src_volname);
7817 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
7818 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
7819 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
7820 } elsif (-f
$src_volid || -b
$src_volid) {
7821 $src_path = $src_volid;
7822 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7827 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7829 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7830 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7831 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7832 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7835 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7836 push @$cmd, '-l', "snapshot.name=$snapname"
7837 if $snapname && $src_format && $src_format eq "qcow2";
7838 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7839 push @$cmd, '-T', $cachemode if defined($cachemode);
7840 push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7842 if ($src_is_iscsi) {
7843 push @$cmd, '--image-opts';
7844 $src_path = convert_iscsi_path
($src_path);
7845 } elsif ($src_format) {
7846 push @$cmd, '-f', $src_format;
7849 if ($dst_is_iscsi) {
7850 push @$cmd, '--target-image-opts';
7851 $dst_path = convert_iscsi_path
($dst_path);
7853 push @$cmd, '-O', $dst_format;
7856 push @$cmd, $src_path;
7858 if (!$dst_is_iscsi && $is_zero_initialized) {
7859 push @$cmd, "zeroinit:$dst_path";
7861 push @$cmd, $dst_path;
7866 if($line =~ m/\((\S+)\/100\
%\)/){
7868 my $transferred = int($size * $percent / 100);
7869 my $total_h = render_bytes
($size, 1);
7870 my $transferred_h = render_bytes
($transferred, 1);
7872 print "transferred $transferred_h of $total_h ($percent%)\n";
7877 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7879 die "copy failed: $err" if $err;
7882 sub qemu_img_format
{
7883 my ($scfg, $volname) = @_;
7885 # FIXME: this entire function is kind of weird given that `parse_volname`
7886 # also already gives us a format?
7887 my $is_path_storage = $scfg->{path
} || $scfg->{type
} eq 'esxi';
7889 if ($is_path_storage && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7896 sub qemu_drive_mirror
{
7897 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7899 $jobs = {} if !$jobs;
7903 $jobs->{"drive-$drive"} = {};
7905 if ($dst_volid =~ /^nbd:/) {
7906 $qemu_target = $dst_volid;
7909 my $storecfg = PVE
::Storage
::config
();
7910 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7912 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7914 $format = qemu_img_format
($dst_scfg, $dst_volname);
7916 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7918 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7921 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7922 $opts->{format
} = $format if $format;
7924 if (defined($src_bitmap)) {
7925 $opts->{sync
} = 'incremental';
7926 $opts->{bitmap
} = $src_bitmap;
7927 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7930 if (defined($bwlimit)) {
7931 $opts->{speed
} = $bwlimit * 1024;
7932 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7934 print "drive mirror is starting for drive-$drive\n";
7937 # if a job already runs for this device we get an error, catch it for cleanup
7938 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7940 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7942 die "mirroring error: $err\n";
7945 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7948 # $completion can be either
7949 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7950 # 'cancel': wait until all jobs are ready, block-job-cancel them
7951 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7952 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7953 sub qemu_drive_mirror_monitor
{
7954 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7956 $completion //= 'complete';
7960 my $err_complete = 0;
7962 my $starttime = time ();
7964 die "block job ('$op') timed out\n" if $err_complete > 300;
7966 my $stats = mon_cmd
($vmid, "query-block-jobs");
7969 my $running_jobs = {};
7970 for my $stat (@$stats) {
7971 next if $stat->{type
} ne $op;
7972 $running_jobs->{$stat->{device
}} = $stat;
7975 my $readycounter = 0;
7977 for my $job_id (sort keys %$jobs) {
7978 my $job = $running_jobs->{$job_id};
7980 my $vanished = !defined($job);
7981 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7982 if($complete || ($vanished && $completion eq 'auto')) {
7983 print "$job_id: $op-job finished\n";
7984 delete $jobs->{$job_id};
7988 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7990 my $busy = $job->{busy
};
7991 my $ready = $job->{ready
};
7992 if (my $total = $job->{len
}) {
7993 my $transferred = $job->{offset
} || 0;
7994 my $remaining = $total - $transferred;
7995 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7997 my $duration = $ctime - $starttime;
7998 my $total_h = render_bytes
($total, 1);
7999 my $transferred_h = render_bytes
($transferred, 1);
8001 my $status = sprintf(
8002 "transferred $transferred_h of $total_h ($percent%%) in %s",
8003 render_duration
($duration),
8008 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
8010 $status .= ", ready";
8013 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
8014 $jobs->{$job_id}->{ready
} = $ready;
8017 $readycounter++ if $job->{ready
};
8020 last if scalar(keys %$jobs) == 0;
8022 if ($readycounter == scalar(keys %$jobs)) {
8023 print "all '$op' jobs are ready\n";
8025 # do the complete later (or has already been done)
8026 last if $completion eq 'skip' || $completion eq 'auto';
8028 if ($vmiddst && $vmiddst != $vmid) {
8029 my $agent_running = $qga && qga_check_running
($vmid);
8030 if ($agent_running) {
8031 print "freeze filesystem\n";
8032 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
8035 print "suspend vm\n";
8036 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
8040 # if we clone a disk for a new target vm, we don't switch the disk
8041 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
8043 if ($agent_running) {
8044 print "unfreeze filesystem\n";
8045 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
8048 print "resume vm\n";
8049 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
8056 for my $job_id (sort keys %$jobs) {
8057 # try to switch the disk if source and destination are on the same guest
8058 print "$job_id: Completing block job_id...\n";
8061 if ($completion eq 'complete') {
8062 $op = 'block-job-complete';
8063 } elsif ($completion eq 'cancel') {
8064 $op = 'block-job-cancel';
8066 die "invalid completion value: $completion\n";
8068 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
8069 if ($@ =~ m/cannot be completed/) {
8070 print "$job_id: block job cannot be completed, trying again.\n";
8073 print "$job_id: Completed successfully.\n";
8074 $jobs->{$job_id}->{complete
} = 1;
8085 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
8086 die "block job ($op) error: $err";
8090 sub qemu_blockjobs_cancel
{
8091 my ($vmid, $jobs) = @_;
8093 foreach my $job (keys %$jobs) {
8094 print "$job: Cancelling block job\n";
8095 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
8096 $jobs->{$job}->{cancel
} = 1;
8100 my $stats = mon_cmd
($vmid, "query-block-jobs");
8102 my $running_jobs = {};
8103 foreach my $stat (@$stats) {
8104 $running_jobs->{$stat->{device
}} = $stat;
8107 foreach my $job (keys %$jobs) {
8109 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
8110 print "$job: Done.\n";
8111 delete $jobs->{$job};
8115 last if scalar(keys %$jobs) == 0;
8121 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8122 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8123 my sub clone_disk_check_io_uring
{
8124 my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8126 return if !$use_drive_mirror;
8128 # Don't complain when not changing storage.
8129 # Assume if it works for the source, it'll work for the target too.
8130 return if $src_storeid eq $dst_storeid;
8132 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
8133 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
8135 my $cache_direct = drive_uses_cache_direct
($src_drive);
8137 my $src_uses_io_uring;
8138 if ($src_drive->{aio
}) {
8139 $src_uses_io_uring = $src_drive->{aio
} eq 'io_uring';
8141 $src_uses_io_uring = storage_allows_io_uring_default
($src_scfg, $cache_direct);
8144 die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8145 if $src_uses_io_uring && !storage_allows_io_uring_default
($dst_scfg, $cache_direct);
8149 my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8151 my ($vmid, $running) = $source->@{qw(vmid running)};
8152 my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8154 my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8155 my ($storage, $format) = $dest->@{qw(storage format)};
8157 my $unused = defined($src_drivename) && $src_drivename =~ /^unused/;
8158 my $use_drive_mirror = $full && $running && $src_drivename && !$snapname && !$unused;
8160 if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8161 die "cloning from/to EFI disk requires EFI disk\n"
8162 if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8163 die "cloning from/to TPM state requires TPM state\n"
8164 if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8166 # This would lead to two device nodes in QEMU pointing to the same backing image!
8167 die "cannot change drive name when cloning disk from/to the same VM\n"
8168 if $use_drive_mirror && $vmid == $newvmid;
8171 die "cannot move TPM state while VM is running\n"
8172 if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8176 print "create " . ($full ?
'full' : 'linked') . " clone of drive ";
8177 print "$src_drivename " if $src_drivename;
8178 print "($drive->{file})\n";
8181 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
8182 push @$newvollist, $newvolid;
8184 my ($src_storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
8185 my $storeid = $storage || $src_storeid;
8187 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
8191 if (drive_is_cloudinit
($drive)) {
8192 $name = "vm-$newvmid-cloudinit";
8193 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8194 if ($scfg->{path
}) {
8195 $name .= ".$dst_format";
8198 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
8199 } elsif ($dst_drivename eq 'efidisk0') {
8200 $size = $efisize or die "internal error - need to specify EFI disk size\n";
8201 } elsif ($dst_drivename eq 'tpmstate0') {
8202 $dst_format = 'raw';
8203 $size = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8205 clone_disk_check_io_uring
($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8207 $size = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
8209 $newvolid = PVE
::Storage
::vdisk_alloc
(
8210 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8212 push @$newvollist, $newvolid;
8214 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
8216 if (drive_is_cloudinit
($drive)) {
8217 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8218 # if this is the case, we have to complete any block-jobs still there from
8219 # previous drive-mirrors
8220 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8221 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
8226 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
8227 if ($use_drive_mirror) {
8228 qemu_drive_mirror
($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8229 $completion, $qga, $bwlimit);
8231 if ($dst_drivename eq 'efidisk0') {
8232 # the relevant data on the efidisk may be smaller than the source
8233 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8234 # that is given by the OVMF_VARS.fd
8235 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
8236 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
8238 my $src_format = (PVE
::Storage
::parse_volname
($storecfg, $drive->{file
}))[6];
8240 # better for Ceph if block size is not too small, see bug #3324
8243 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8245 if ($src_format eq 'qcow2' && $snapname) {
8246 die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8247 if !min_version
(kvm_user_version
(), 6, 2);
8248 push $cmd->@*, '-l', $snapname;
8250 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8253 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8259 my $size = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
8261 my $disk = dclone
($drive);
8262 delete $disk->{format
};
8263 $disk->{file
} = $newvolid;
8264 $disk->{size
} = $size if defined($size) && !$unused;
8269 sub get_running_qemu_version
{
8271 my $res = mon_cmd
($vmid, "query-version");
8272 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8275 sub qemu_use_old_bios_files
{
8276 my ($machine_type) = @_;
8278 return if !$machine_type;
8280 my $use_old_bios_files = undef;
8282 if ($machine_type =~ m/^(\S+)\.pxe$/) {
8284 $use_old_bios_files = 1;
8286 my $version = extract_version
($machine_type, kvm_user_version
());
8287 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8288 # load new efi bios files on migration. So this hack is required to allow
8289 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8290 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8291 $use_old_bios_files = !min_version
($version, 2, 4);
8294 return ($use_old_bios_files, $machine_type);
8297 sub get_efivars_size
{
8298 my ($conf, $efidisk) = @_;
8300 my $arch = get_vm_arch
($conf);
8301 $efidisk //= $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
8302 my $smm = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
8303 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8304 return -s
$ovmf_vars;
8307 sub update_efidisk_size
{
8310 return if !defined($conf->{efidisk0
});
8312 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
8313 $disk->{size
} = get_efivars_size
($conf);
8314 $conf->{efidisk0
} = print_drive
($disk);
8319 sub update_tpmstate_size
{
8322 my $disk = PVE
::QemuServer
::parse_drive
('tpmstate0', $conf->{tpmstate0
});
8323 $disk->{size
} = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8324 $conf->{tpmstate0
} = print_drive
($disk);
8327 sub create_efidisk
($$$$$$$) {
8328 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8330 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8332 my $vars_size_b = -s
$ovmf_vars;
8333 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
8334 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8335 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
8337 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
8338 my $size = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
8340 return ($volid, $size/1024);
8343 sub vm_iothreads_list
{
8346 my $res = mon_cmd
($vmid, 'query-iothreads');
8349 foreach my $iothread (@$res) {
8350 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
8357 my ($conf, $drive) = @_;
8361 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
8363 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
8369 my $controller = int($drive->{index} / $maxdev);
8370 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
8374 return ($maxdev, $controller, $controller_prefix);
8377 sub resolve_dst_disk_format
{
8378 my ($storecfg, $storeid, $src_volname, $format) = @_;
8379 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
8382 # if no target format is specified, use the source disk format as hint
8384 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8385 $format = qemu_img_format
($scfg, $src_volname);
8391 # test if requested format is supported - else use default
8392 my $supported = grep { $_ eq $format } @$validFormats;
8393 $format = $defFormat if !$supported;
8397 # NOTE: if this logic changes, please update docs & possibly gui logic
8398 sub find_vmstate_storage
{
8399 my ($conf, $storecfg) = @_;
8401 # first, return storage from conf if set
8402 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
8404 my ($target, $shared, $local);
8406 foreach_storage_used_by_vm
($conf, sub {
8408 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
8409 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
8410 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
8413 # second, use shared storage where VM has at least one disk
8414 # third, use local storage where VM has at least one disk
8415 # fall back to local storage
8416 $target = $shared // $local // 'local';
8422 my ($uuid, $uuid_str);
8423 UUID
::generate
($uuid);
8424 UUID
::unparse
($uuid, $uuid_str);
8428 sub generate_smbios1_uuid
{
8429 return "uuid=".generate_uuid
();
8435 mon_cmd
($vmid, 'nbd-server-stop', timeout
=> 25);
8438 sub create_reboot_request
{
8440 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8441 or die "failed to create reboot trigger file: $!\n";
8445 sub clear_reboot_request
{
8447 my $path = "/run/qemu-server/$vmid.reboot";
8450 $res = unlink($path);
8451 die "could not remove reboot request for $vmid: $!"
8452 if !$res && $! != POSIX
::ENOENT
;
8457 sub bootorder_from_legacy
{
8458 my ($conf, $bootcfg) = @_;
8460 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
8461 my $bootindex_hash = {};
8463 foreach my $o (split(//, $boot)) {
8464 $bootindex_hash->{$o} = $i*100;
8470 PVE
::QemuConfig-
>foreach_volume($conf, sub {
8471 my ($ds, $drive) = @_;
8473 if (drive_is_cdrom
($drive, 1)) {
8474 if ($bootindex_hash->{d
}) {
8475 $bootorder->{$ds} = $bootindex_hash->{d
};
8476 $bootindex_hash->{d
} += 1;
8478 } elsif ($bootindex_hash->{c
}) {
8479 $bootorder->{$ds} = $bootindex_hash->{c
}
8480 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
8481 $bootindex_hash->{c
} += 1;
8485 if ($bootindex_hash->{n
}) {
8486 for (my $i = 0; $i < $MAX_NETS; $i++) {
8487 my $netname = "net$i";
8488 next if !$conf->{$netname};
8489 $bootorder->{$netname} = $bootindex_hash->{n
};
8490 $bootindex_hash->{n
} += 1;
8497 # Generate default device list for 'boot: order=' property. Matches legacy
8498 # default boot order, but with explicit device names. This is important, since
8499 # the fallback for when neither 'order' nor the old format is specified relies
8500 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8501 sub get_default_bootdevices
{
8507 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
8508 push @ret, $first if $first;
8511 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
8512 push @ret, $first if $first;
8515 for (my $i = 0; $i < $MAX_NETS; $i++) {
8516 my $netname = "net$i";
8517 next if !$conf->{$netname};
8518 push @ret, $netname;
8525 sub device_bootorder
{
8528 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
8530 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
8533 if (!defined($boot) || $boot->{legacy
}) {
8534 $bootorder = bootorder_from_legacy
($conf, $boot);
8535 } elsif ($boot->{order
}) {
8536 my $i = 100; # start at 100 to allow user to insert devices before us with -args
8537 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
8538 $bootorder->{$dev} = $i++;
8545 sub register_qmeventd_handle
{
8549 my $peer = "/var/run/qmeventd.sock";
8554 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
8556 if ($! != EINTR
&& $! != EAGAIN
) {
8557 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8560 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8561 . "after $count retries\n";
8566 # send handshake to mark VM as backing up
8567 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
8569 # return handle to be closed later when inhibit is no longer required
8573 # bash completion helper
8575 sub complete_backup_archives
{
8576 my ($cmdname, $pname, $cvalue) = @_;
8578 my $cfg = PVE
::Storage
::config
();
8582 if ($cvalue =~ m/^([^:]+):/) {
8586 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
8589 foreach my $id (keys %$data) {
8590 foreach my $item (@{$data->{$id}}) {
8591 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8592 push @$res, $item->{volid
} if defined($item->{volid
});
8599 my $complete_vmid_full = sub {
8602 my $idlist = vmstatus
();
8606 foreach my $id (keys %$idlist) {
8607 my $d = $idlist->{$id};
8608 if (defined($running)) {
8609 next if $d->{template
};
8610 next if $running && $d->{status
} ne 'running';
8611 next if !$running && $d->{status
} eq 'running';
8620 return &$complete_vmid_full();
8623 sub complete_vmid_stopped
{
8624 return &$complete_vmid_full(0);
8627 sub complete_vmid_running
{
8628 return &$complete_vmid_full(1);
8631 sub complete_storage
{
8633 my $cfg = PVE
::Storage
::config
();
8634 my $ids = $cfg->{ids
};
8637 foreach my $sid (keys %$ids) {
8638 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
8639 next if !$ids->{$sid}->{content
}->{images
};
8646 sub complete_migration_storage
{
8647 my ($cmd, $param, $current_value, $all_args) = @_;
8649 my $targetnode = @$all_args[1];
8651 my $cfg = PVE
::Storage
::config
();
8652 my $ids = $cfg->{ids
};
8655 foreach my $sid (keys %$ids) {
8656 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
8657 next if !$ids->{$sid}->{content
}->{images
};
8665 my ($vmid, $include_suspended) = @_;
8666 my $qmpstatus = eval {
8667 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
8668 mon_cmd
($vmid, "query-status");
8671 return $qmpstatus && (
8672 $qmpstatus->{status
} eq "paused" ||
8673 $qmpstatus->{status
} eq "prelaunch" ||
8674 ($include_suspended && $qmpstatus->{status
} eq "suspended")
8678 sub check_volume_storage_type
{
8679 my ($storecfg, $vol) = @_;
8681 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
8682 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8683 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
8685 die "storage '$storeid' does not support content-type '$vtype'\n"
8686 if !$scfg->{content
}->{$vtype};
8691 sub add_nets_bridge_fdb
{
8692 my ($conf, $vmid) = @_;
8694 for my $opt (keys %$conf) {
8695 next if $opt !~ m/^net(\d+)$/;
8696 my $iface = "tap${vmid}i$1";
8697 # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8698 my $net = parse_net
($conf->{$opt}, 1) or next;
8700 my $mac = $net->{macaddr
};
8702 log_warn
("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8703 if !file_read_firstline
("/sys/class/net/$iface/brport/learning");
8707 my $bridge = $net->{bridge
};
8709 log_warn
("Interface '$iface' not attached to any bridge.");
8713 PVE
::Network
::SDN
::Zones
::add_bridge_fdb
($iface, $mac, $bridge);
8714 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8715 PVE
::Network
::add_bridge_fdb
($iface, $mac);
8720 sub del_nets_bridge_fdb
{
8721 my ($conf, $vmid) = @_;
8723 for my $opt (keys %$conf) {
8724 next if $opt !~ m/^net(\d+)$/;
8725 my $iface = "tap${vmid}i$1";
8727 my $net = parse_net
($conf->{$opt}) or next;
8728 my $mac = $net->{macaddr
} or next;
8730 my $bridge = $net->{bridge
};
8732 PVE
::Network
::SDN
::Zones
::del_bridge_fdb
($iface, $mac, $bridge);
8733 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8734 PVE
::Network
::del_bridge_fdb
($iface, $mac);
8739 sub create_ifaces_ipams_ips
{
8740 my ($conf, $vmid) = @_;
8742 return if !$have_sdn;
8744 foreach my $opt (keys %$conf) {
8745 if ($opt =~ m/^net(\d+)$/) {
8746 my $value = $conf->{$opt};
8747 my $net = PVE
::QemuServer
::parse_net
($value);
8748 eval { PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($net->{bridge
}, $conf->{name
}, $net->{macaddr
}, $vmid, undef, 1) };
8754 sub delete_ifaces_ipams_ips
{
8755 my ($conf, $vmid) = @_;
8757 return if !$have_sdn;
8759 foreach my $opt (keys %$conf) {
8760 if ($opt =~ m/^net(\d+)$/) {
8761 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
8762 eval { PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
}) };