1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
22 use List
::Util
qw(first);
25 use Storable
qw(dclone);
26 use Time
::HiRes
qw(gettimeofday usleep);
30 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
33 use PVE
::DataCenterConfig
;
34 use PVE
::Exception
qw(raise raise_param_exc);
35 use PVE
::Format
qw(render_duration render_bytes);
36 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
38 use PVE
::Mapping
::PCI
;
39 use PVE
::Mapping
::USB
;
41 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
44 use PVE
::RESTEnvironment
qw(log_warn);
45 use PVE
::RPCEnvironment
;
49 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
53 use PVE
::QemuServer
::Helpers
qw(config_aware_timeout min_version windows_version);
54 use PVE
::QemuServer
::Cloudinit
;
55 use PVE
::QemuServer
::CGroup
;
56 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch);
57 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
58 use PVE
::QemuServer
::Machine
;
59 use PVE
::QemuServer
::Memory
qw(get_current_memory);
60 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
61 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
62 use PVE
::QemuServer
::QMPHelpers
qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
63 use PVE
::QemuServer
::USB
;
67 require PVE
::Network
::SDN
::Zones
;
68 require PVE
::Network
::SDN
::Vnets
;
72 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
76 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
77 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
80 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
81 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
84 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
85 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
88 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
89 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
91 # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
92 # anymore. how can we deperacate this sanely without breaking existing instances, or using
93 # older backups and snapshot?
95 "$EDK2_FW_BASE/OVMF_CODE.fd",
96 "$EDK2_FW_BASE/OVMF_VARS.fd",
101 "$EDK2_FW_BASE/AAVMF_CODE.fd",
102 "$EDK2_FW_BASE/AAVMF_VARS.fd",
107 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
109 # Note about locking: we use flock on the config file protect against concurent actions.
110 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
111 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
112 # But you can ignore this kind of lock with the --skiplock flag.
120 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
121 description
=> "Some command save/restore state from this location.",
127 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
128 description
=> "Specifies the QEMU machine type.",
130 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
135 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
138 $nodename_cache //= PVE
::INotify
::nodename
();
139 return $nodename_cache;
146 enum
=> [qw(i6300esb ib700)],
147 description
=> "Watchdog type to emulate.",
148 default => 'i6300esb',
153 enum
=> [qw(reset shutdown poweroff pause debug none)],
154 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
158 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
162 description
=> "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
167 fstrim_cloned_disks
=> {
168 description
=> "Run fstrim after moving a disk or migrating the VM.",
173 'freeze-fs-on-backup' => {
174 description
=> "Freeze/thaw guest filesystems on backup for consistency.",
180 description
=> "Select the agent type",
184 enum
=> [qw(virtio isa)],
190 description
=> "Select the VGA type.",
195 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
198 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
205 description
=> 'Enable a specific clipboard. If not set, depending on the display type the'
206 .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
217 description
=> "The size of the file in MB.",
221 pattern
=> '[a-zA-Z0-9\-]+',
223 format_description
=> 'string',
224 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
231 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
232 description
=> "Configure an audio device."
236 enum
=> ['spice', 'none'],
239 description
=> "Driver backend for the audio device."
243 my $spice_enhancements_fmt = {
248 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
252 enum
=> ['off', 'all', 'filter'],
255 description
=> "Enable video streaming. Uses compression for detected video streams."
262 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
264 description
=> "The file on the host to gather entropy from. In most cases '/dev/urandom'"
265 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
266 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
267 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
268 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
269 ." a hardware RNG from the host.",
273 description
=> "Maximum bytes of entropy allowed to get injected into the guest every"
274 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
275 ." `0` to disable limiting (potentially dangerous!).",
278 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
279 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
280 # reading from /dev/urandom
285 description
=> "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
286 ." the guest to retrieve another 'max_bytes' of entropy.",
292 my $meta_info_fmt = {
295 description
=> "The guest creation timestamp as UNIX epoch time",
301 description
=> "The QEMU (machine) version from the time this VM was created.",
302 pattern
=> '\d+(\.\d+)+',
311 description
=> "Specifies whether a VM will be started during system bootup.",
317 description
=> "Automatic restart after crash (currently ignored).",
322 type
=> 'string', format
=> 'pve-hotplug-features',
323 description
=> "Selectively enable hotplug features. This is a comma separated list of"
324 ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
325 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
326 ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
328 default => 'network,disk,usb',
333 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
339 description
=> "Lock/unlock the VM.",
340 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
345 description
=> "Limit of CPU usage.",
346 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
347 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
355 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
356 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
357 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
358 ." weights of all the other running VMs.",
361 default => 'cgroup v1: 1024, cgroup v2: 100',
366 description
=> "Memory properties.",
367 format
=> $PVE::QemuServer
::Memory
::memory_fmt
372 description
=> "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
378 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the"
379 ." more memory this VM gets. Number is relative to weights of all other running VMs."
380 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
388 description
=> "Keyboard layout for VNC server. This option is generally not required and"
389 ." is often better handled from within the guest OS.",
390 enum
=> PVE
::Tools
::kvmkeymaplist
(),
395 type
=> 'string', format
=> 'dns-name',
396 description
=> "Set a name for the VM. Only used on the configuration web interface.",
401 description
=> "SCSI controller model",
402 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
408 description
=> "Description for the VM. Shown in the web-interface VM's summary."
409 ." This is saved as comment inside the configuration file.",
410 maxLength
=> 1024 * 8,
415 # NOTE: When extending, also consider extending `%guest_types` in `Import/ESXi.pm`.
416 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
417 description
=> "Specify guest operating system.",
418 verbose_description
=> <<EODESC,
419 Specify guest operating system. This is used to enable special
420 optimization/features for specific operating systems:
423 other;; unspecified OS
424 wxp;; Microsoft Windows XP
425 w2k;; Microsoft Windows 2000
426 w2k3;; Microsoft Windows 2003
427 w2k8;; Microsoft Windows 2008
428 wvista;; Microsoft Windows Vista
429 win7;; Microsoft Windows 7
430 win8;; Microsoft Windows 8/2012/2012r2
431 win10;; Microsoft Windows 10/2016/2019
432 win11;; Microsoft Windows 11/2022
433 l24;; Linux 2.4 Kernel
434 l26;; Linux 2.6 - 6.X Kernel
435 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
440 type
=> 'string', format
=> 'pve-qm-boot',
441 description
=> "Specify guest boot order. Use the 'order=' sub-property as usage with no"
442 ." key or 'legacy=' is deprecated.",
446 type
=> 'string', format
=> 'pve-qm-bootdisk',
447 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
448 pattern
=> '(ide|sata|scsi|virtio)\d+',
453 description
=> "The number of CPUs. Please use option -sockets instead.",
460 description
=> "The number of CPU sockets.",
467 description
=> "The number of cores per socket.",
474 description
=> "Enable/disable NUMA.",
480 description
=> "Enable/disable hugepages memory.",
481 enum
=> [qw(any 2 1024)],
487 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
488 ." after VM shutdown and can be used for subsequent starts.",
493 description
=> "Number of hotplugged vcpus.",
500 description
=> "Enable/disable ACPI.",
505 description
=> "Enable/disable communication with the QEMU Guest Agent and its properties.",
507 format
=> $agent_fmt,
512 description
=> "Enable/disable KVM hardware virtualization.",
518 description
=> "Enable/disable time drift fix.",
524 description
=> "Set the real time clock (RTC) to local time. This is enabled by default if"
525 ." the `ostype` indicates a Microsoft Windows OS.",
530 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
534 type
=> 'string', format
=> $vga_fmt,
535 description
=> "Configure the VGA hardware.",
536 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
537 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
538 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
539 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
540 ." display server. For win* OS you can select how many independent displays you want,"
541 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
542 ." using a serial device as terminal.",
546 type
=> 'string', format
=> 'pve-qm-watchdog',
547 description
=> "Create a virtual hardware watchdog device.",
548 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
549 ." action), the watchdog must be periodically polled by an agent inside the guest or"
550 ." else the watchdog will reset the guest (or execute the respective action specified)",
555 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
556 description
=> "Set the initial date of the real time clock. Valid format for date are:"
557 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
558 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
561 startup
=> get_standard_option
('pve-startup-order'),
565 description
=> "Enable/disable Template.",
571 description
=> "Arbitrary arguments passed to kvm.",
572 verbose_description
=> <<EODESCR,
573 Arbitrary arguments passed to kvm, for example:
575 args: -no-reboot -smbios 'type=0,vendor=FOO'
577 NOTE: this option is for experts only.
584 description
=> "Enable/disable the USB tablet device.",
585 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
586 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
587 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
588 ." may consider disabling this to save some context switches. This is turned off by"
589 ." default if you use spice (`qm set <vmid> --vga qxl`).",
594 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
598 migrate_downtime
=> {
601 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
607 type
=> 'string', format
=> 'pve-qm-ide',
608 typetext
=> '<volume>',
609 description
=> "This is an alias for option -ide2",
613 description
=> "Emulated CPU type.",
615 format
=> 'pve-vm-cpu-conf',
617 parent
=> get_standard_option
('pve-snapshot-name', {
619 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
623 description
=> "Timestamp for snapshots.",
629 type
=> 'string', format
=> 'pve-volume-id',
630 description
=> "Reference to a volume which stores the VM state. This is used internally"
633 vmstatestorage
=> get_standard_option
('pve-storage-id', {
634 description
=> "Default storage for VM state volumes/files.",
637 runningmachine
=> get_standard_option
('pve-qemu-machine', {
638 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
642 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
643 ." internally for snapshots.",
646 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
647 format_description
=> 'QEMU -cpu parameter'
649 machine
=> get_standard_option
('pve-qemu-machine'),
651 description
=> "Virtual processor architecture. Defaults to the host.",
654 enum
=> [qw(x86_64 aarch64)],
657 description
=> "Specify SMBIOS type 1 fields.",
658 type
=> 'string', format
=> 'pve-qm-smbios1',
665 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
666 ." remove disk operations.",
672 enum
=> [ qw(seabios ovmf) ],
673 description
=> "Select BIOS implementation.",
674 default => 'seabios',
678 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
679 format_description
=> 'UUID',
680 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
681 ." to disable explicitly.",
682 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
683 ." value identifier to the guest OS. This allows to notify the guest operating system"
684 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
685 ." execution or creation from a template). The guest operating system notices the"
686 ." change, and is then able to react as appropriate by marking its copies of"
687 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
688 ."Note that auto-creation only works when done through API/CLI create or update methods"
689 .", but not when manually editing the config file.",
690 default => "1 (autogenerated)",
695 format
=> 'pve-volume-id',
697 description
=> "Script that will be executed during various steps in the vms lifetime.",
701 format
=> $ivshmem_fmt,
702 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
708 format
=> $audio_fmt,
709 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
712 spice_enhancements
=> {
714 format
=> $spice_enhancements_fmt,
715 description
=> "Configure additional enhancements for SPICE.",
719 type
=> 'string', format
=> 'pve-tag-list',
720 description
=> 'Tags of the VM. This is only meta information.',
726 description
=> "Configure a VirtIO-based Random Number Generator.",
731 format
=> $meta_info_fmt,
732 description
=> "Some (read-only) meta-information about this guest.",
736 type
=> 'string', format
=> 'pve-cpuset',
737 description
=> "List of host cores used to execute guest processes, for example: 0,5,8-11",
746 description
=> 'Specify a custom file containing all meta data passed to the VM via"
747 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
748 format
=> 'pve-volume-id',
749 format_description
=> 'volume',
754 description
=> 'To pass a custom file containing all network data to the VM via cloud-init.',
755 format
=> 'pve-volume-id',
756 format_description
=> 'volume',
761 description
=> 'To pass a custom file containing all user data to the VM via cloud-init.',
762 format
=> 'pve-volume-id',
763 format_description
=> 'volume',
768 description
=> 'To pass a custom file containing all vendor data to the VM via cloud-init.',
769 format
=> 'pve-volume-id',
770 format_description
=> 'volume',
773 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
775 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
776 my $confdesc_cloudinit = {
780 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
781 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
782 .' and `configdrive2` for windows.',
783 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
788 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
789 ." image's configured default user.",
794 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
795 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
796 .' support hashed passwords.',
801 description
=> 'cloud-init: do an automatic package upgrade after the first boot.',
807 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
809 format
=> 'pve-qm-cicustom',
814 description
=> 'cloud-init: Sets DNS search domains for a container. Create will'
815 .' automatically use the setting from the host if neither searchdomain nor nameserver'
820 type
=> 'string', format
=> 'address-list',
821 description
=> 'cloud-init: Sets DNS server IP address for a container. Create will'
822 .' automatically use the setting from the host if neither searchdomain nor nameserver'
828 format
=> 'urlencoded',
829 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
833 # what about other qemu settings ?
835 #machine => 'string',
848 ##soundhw => 'string',
850 while (my ($k, $v) = each %$confdesc) {
851 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
855 my $MAX_SERIAL_PORTS = 4;
856 my $MAX_PARALLEL_PORTS = 3;
858 for (my $i = 0; $i < $PVE::QemuServer
::Memory
::MAX_NUMA
; $i++) {
859 $confdesc->{"numa$i"} = $PVE::QemuServer
::Memory
::numadesc
;
862 my $nic_model_list = [
878 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
880 my $net_fmt_bridge_descr = <<__EOD__;
881 Bridge to attach the network device to. The Proxmox VE standard bridge
884 If you do not specify a bridge, we create a kvm user (NATed) network
885 device, which provides DHCP and DNS services. The following addresses
892 The DHCP server assign addresses to the guest starting from 10.0.2.15.
896 macaddr
=> get_standard_option
('mac-addr', {
897 description
=> "MAC address. That address must be unique withing your network. This is"
898 ." automatically generated if not specified.",
902 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
903 ." very low CPU overhead. If your guest does not support this driver, it is usually"
904 ." best to use 'e1000'.",
905 enum
=> $nic_model_list,
908 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
909 bridge
=> get_standard_option
('pve-bridge-id', {
910 description
=> $net_fmt_bridge_descr,
915 minimum
=> 0, maximum
=> 64,
916 description
=> 'Number of packet queues to be used on the device.',
922 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
927 minimum
=> 1, maximum
=> 4094,
928 description
=> 'VLAN tag to apply to packets on this interface.',
933 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
934 description
=> 'VLAN trunks to pass through this interface.',
935 format_description
=> 'vlanid[;vlanid...]',
940 description
=> 'Whether this interface should be protected by the firewall.',
945 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
950 minimum
=> 1, maximum
=> 65520,
951 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
958 type
=> 'string', format
=> $net_fmt,
959 description
=> "Specify network devices.",
962 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
967 format
=> 'pve-ipv4-config',
968 format_description
=> 'IPv4Format/CIDR',
969 description
=> 'IPv4 address in CIDR format.',
976 format_description
=> 'GatewayIPv4',
977 description
=> 'Default gateway for IPv4 traffic.',
983 format
=> 'pve-ipv6-config',
984 format_description
=> 'IPv6Format/CIDR',
985 description
=> 'IPv6 address in CIDR format.',
992 format_description
=> 'GatewayIPv6',
993 description
=> 'Default gateway for IPv6 traffic.',
998 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
1001 type
=> 'string', format
=> 'pve-qm-ipconfig',
1002 description
=> <<'EODESCR',
1003 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1005 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1007 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1008 gateway should be provided.
1009 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1010 cloud-init 19.4 or newer.
1012 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1016 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
1018 for (my $i = 0; $i < $MAX_NETS; $i++) {
1019 $confdesc->{"net$i"} = $netdesc;
1020 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1023 foreach my $key (keys %$confdesc_cloudinit) {
1024 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1027 PVE
::JSONSchema
::register_format
('pve-cpuset', \
&pve_verify_cpuset
);
1028 sub pve_verify_cpuset
{
1029 my ($set_text, $noerr) = @_;
1031 my ($count, $members) = eval { PVE
::CpuSet
::parse_cpuset
($set_text) };
1035 die "unable to parse cpuset option\n";
1038 return PVE
::CpuSet-
>new($members)->short_string();
1041 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1042 sub verify_volume_id_or_qm_path
{
1043 my ($volid, $noerr) = @_;
1045 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1047 return verify_volume_id_or_absolute_path
($volid, $noerr);
1050 PVE
::JSONSchema
::register_format
('pve-volume-id-or-absolute-path', \
&verify_volume_id_or_absolute_path
);
1051 sub verify_volume_id_or_absolute_path
{
1052 my ($volid, $noerr) = @_;
1054 return $volid if $volid =~ m
|^/|;
1056 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1067 pattern
=> '(/dev/.+|socket)',
1068 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1069 verbose_description
=> <<EODESCR,
1070 Create a serial device inside the VM (n is 0 to 3), and pass through a
1071 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1072 host side (use 'qm terminal' to open a terminal connection).
1074 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1075 use with special care.
1077 CAUTION: Experimental! User reported problems with this option.
1084 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1085 description
=> "Map host parallel devices (n is 0 to 2).",
1086 verbose_description
=> <<EODESCR,
1087 Map host parallel devices (n is 0 to 2).
1089 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1090 machines - use with special care.
1092 CAUTION: Experimental! User reported problems with this option.
1096 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1097 $confdesc->{"parallel$i"} = $paralleldesc;
1100 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1101 $confdesc->{"serial$i"} = $serialdesc;
1104 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1105 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1108 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1109 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1112 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
1113 $confdesc->{"usb$i"} = $PVE::QemuServer
::USB
::usbdesc
;
1121 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1122 . " Deprecated, use 'order=' instead.",
1123 pattern
=> '[acdn]{1,4}',
1124 format_description
=> "[acdn]{1,4}",
1126 # note: this is also the fallback if boot: is not given at all
1132 format
=> 'pve-qm-bootdev-list',
1133 format_description
=> "device[;device...]",
1134 description
=> <<EODESC,
1135 The guest will attempt to boot from devices in the order they appear here.
1137 Disks, optical drives and passed-through storage USB devices will be directly
1138 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1139 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1141 Note that only devices in this list will be marked as bootable and thus loaded
1142 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1143 (e.g. software-raid), you need to specify all of them here.
1145 Overrides the deprecated 'legacy=[acdn]*' value when given.
1149 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1151 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1152 sub verify_bootdev
{
1153 my ($dev, $noerr) = @_;
1155 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1156 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && !$special;
1160 return 0 if $dev !~ m/^$base\d+$/;
1161 return 0 if !$confdesc->{$dev};
1165 return $dev if $check->("net");
1166 return $dev if $check->("usb");
1167 return $dev if $check->("hostpci");
1170 die "invalid boot device '$dev'\n";
1173 sub print_bootorder
{
1175 return "" if !@$devs;
1176 my $data = { order
=> join(';', @$devs) };
1177 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1180 my $kvm_api_version = 0;
1183 return $kvm_api_version if $kvm_api_version;
1185 open my $fh, '<', '/dev/kvm' or return;
1187 # 0xae00 => KVM_GET_API_VERSION
1188 $kvm_api_version = ioctl($fh, 0xae00, 0);
1191 return $kvm_api_version;
1194 my $kvm_user_version = {};
1197 sub kvm_user_version
{
1200 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1201 my $st = stat($binary);
1203 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1204 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1205 $cachedmtime == $st->mtime;
1207 $kvm_user_version->{$binary} = 'unknown';
1208 $kvm_mtime->{$binary} = $st->mtime;
1212 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1213 $kvm_user_version->{$binary} = $2;
1217 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1220 return $kvm_user_version->{$binary};
1223 my sub extract_version
{
1224 my ($machine_type, $version) = @_;
1225 $version = kvm_user_version
() if !defined($version);
1226 return PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1229 sub kernel_has_vhost_net
{
1230 return -c
'/dev/vhost-net';
1235 return defined($confdesc->{$key});
1239 sub get_cdrom_path
{
1241 return $cdrom_path if defined($cdrom_path);
1243 $cdrom_path = first
{ -l
$_ } map { "/dev/cdrom$_" } ('', '1', '2');
1245 if (!defined($cdrom_path)) {
1246 log_warn
("no physical CD-ROM available, ignoring");
1254 my ($storecfg, $vmid, $cdrom) = @_;
1256 if ($cdrom eq 'cdrom') {
1257 return get_cdrom_path
();
1258 } elsif ($cdrom eq 'none') {
1260 } elsif ($cdrom =~ m
|^/|) {
1263 return PVE
::Storage
::path
($storecfg, $cdrom);
1267 # try to convert old style file names to volume IDs
1268 sub filename_to_volume_id
{
1269 my ($vmid, $file, $media) = @_;
1271 if (!($file eq 'none' || $file eq 'cdrom' ||
1272 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1274 return if $file =~ m
|/|;
1276 if ($media && $media eq 'cdrom') {
1277 $file = "local:iso/$file";
1279 $file = "local:$vmid/$file";
1286 sub verify_media_type
{
1287 my ($opt, $vtype, $media) = @_;
1292 if ($media eq 'disk') {
1294 } elsif ($media eq 'cdrom') {
1297 die "internal error";
1300 return if ($vtype eq $etype);
1302 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1305 sub cleanup_drive_path
{
1306 my ($opt, $storecfg, $drive) = @_;
1308 # try to convert filesystem paths to volume IDs
1310 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1311 ($drive->{file
} !~ m
|^/dev/.+|) &&
1312 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1313 ($drive->{file
} !~ m/^\d+$/)) {
1314 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1315 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1317 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1318 verify_media_type
($opt, $vtype, $drive->{media
});
1319 $drive->{file
} = $volid;
1322 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1325 sub parse_hotplug_features
{
1330 return $res if $data eq '0';
1332 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1334 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1335 if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1338 die "invalid hotplug feature '$feature'\n";
1344 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1345 sub pve_verify_hotplug_features
{
1346 my ($value, $noerr) = @_;
1348 return $value if parse_hotplug_features
($value);
1352 die "unable to parse hotplug option\n";
1355 sub assert_clipboard_config
{
1358 my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1362 && $vga->{'clipboard'} eq 'vnc'
1364 && $vga->{type
} !~ $clipboard_regex
1366 die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1370 sub print_tabletdevice_full
{
1371 my ($conf, $arch) = @_;
1373 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1375 # we use uhci for old VMs because tablet driver was buggy in older qemu
1377 if ($q35 || $arch eq 'aarch64') {
1383 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1386 sub print_keyboarddevice_full
{
1387 my ($conf, $arch) = @_;
1389 return if $arch ne 'aarch64';
1391 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1394 my sub get_drive_id
{
1396 return "$drive->{interface}$drive->{index}";
1399 sub print_drivedevice_full
{
1400 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1405 my $drive_id = get_drive_id
($drive);
1406 if ($drive->{interface
} eq 'virtio') {
1407 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1408 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1409 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1410 } elsif ($drive->{interface
} eq 'scsi') {
1412 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1413 my $unit = $drive->{index} % $maxdev;
1415 my $machine_version = extract_version
($machine_type, kvm_user_version
());
1416 my $devicetype = PVE
::QemuServer
::Drive
::get_scsi_devicetype
(
1417 $drive, $storecfg, $machine_version);
1419 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1420 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1422 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1423 .",lun=$drive->{index}";
1425 $device .= ",drive=drive-$drive_id,id=$drive_id";
1427 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1428 $device .= ",rotation_rate=1";
1430 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1432 # only scsi-hd and scsi-cd support passing vendor and product information
1433 if ($devicetype eq 'hd' || $devicetype eq 'cd') {
1434 if (my $vendor = $drive->{vendor
}) {
1435 $device .= ",vendor=$vendor";
1437 if (my $product = $drive->{product
}) {
1438 $device .= ",product=$product";
1442 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1443 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1444 my $controller = int($drive->{index} / $maxdev);
1445 my $unit = $drive->{index} % $maxdev;
1447 # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1448 # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1449 # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1450 # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1451 # were before. Move odd ones up by 2 where they don't clash.
1452 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) && $drive->{interface
} eq 'ide') {
1453 $controller += 2 * ($unit % 2);
1457 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1459 $device = "ide-$devicetype";
1460 if ($drive->{interface
} eq 'ide') {
1461 $device .= ",bus=ide.$controller,unit=$unit";
1463 $device .= ",bus=ahci$controller.$unit";
1465 $device .= ",drive=drive-$drive_id,id=$drive_id";
1467 if ($devicetype eq 'hd') {
1468 if (my $model = $drive->{model
}) {
1469 $model = URI
::Escape
::uri_unescape
($model);
1470 $device .= ",model=$model";
1472 if ($drive->{ssd
}) {
1473 $device .= ",rotation_rate=1";
1476 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1477 } elsif ($drive->{interface
} eq 'usb') {
1479 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1481 die "unsupported interface type";
1484 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1486 if (my $serial = $drive->{serial
}) {
1487 $serial = URI
::Escape
::uri_unescape
($serial);
1488 $device .= ",serial=$serial";
1495 sub get_initiator_name
{
1498 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1499 while (defined(my $line = <$fh>)) {
1500 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1509 my sub storage_allows_io_uring_default
{
1510 my ($scfg, $cache_direct) = @_;
1512 # io_uring with cache mode writeback or writethrough on krbd will hang...
1513 return if $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1515 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1516 # sometimes, just plain disable...
1517 return if $scfg && $scfg->{type
} eq 'lvm';
1519 # io_uring causes problems when used with CIFS since kernel 5.15
1520 # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1521 return if $scfg && $scfg->{type
} eq 'cifs';
1526 my sub drive_uses_cache_direct
{
1527 my ($drive, $scfg) = @_;
1529 my $cache_direct = 0;
1531 if (my $cache = $drive->{cache
}) {
1532 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1533 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1537 return $cache_direct;
1540 sub print_drive_commandline_full
{
1541 my ($storecfg, $vmid, $drive, $live_restore_name, $io_uring) = @_;
1544 my $volid = $drive->{file
};
1545 my $format = $drive->{format
};
1546 my $drive_id = get_drive_id
($drive);
1548 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1549 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1551 if (drive_is_cdrom
($drive)) {
1552 $path = get_iso_path
($storecfg, $vmid, $volid);
1553 die "$drive_id: cannot back cdrom drive with a live restore image\n" if $live_restore_name;
1556 $path = PVE
::Storage
::path
($storecfg, $volid);
1557 $format //= qemu_img_format
($scfg, $volname);
1564 my $is_rbd = $path =~ m/^rbd:/;
1567 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1568 foreach my $o (@qemu_drive_options) {
1569 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1572 # snapshot only accepts on|off
1573 if (defined($drive->{snapshot
})) {
1574 my $v = $drive->{snapshot
} ?
'on' : 'off';
1575 $opts .= ",snapshot=$v";
1578 if (defined($drive->{ro
})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1579 $opts .= ",readonly=" . ($drive->{ro
} ?
'on' : 'off');
1582 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1583 my ($dir, $qmpname) = @$type;
1584 if (my $v = $drive->{"mbps$dir"}) {
1585 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1587 if (my $v = $drive->{"mbps${dir}_max"}) {
1588 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1590 if (my $v = $drive->{"bps${dir}_max_length"}) {
1591 $opts .= ",throttling.bps$qmpname-max-length=$v";
1593 if (my $v = $drive->{"iops${dir}"}) {
1594 $opts .= ",throttling.iops$qmpname=$v";
1596 if (my $v = $drive->{"iops${dir}_max"}) {
1597 $opts .= ",throttling.iops$qmpname-max=$v";
1599 if (my $v = $drive->{"iops${dir}_max_length"}) {
1600 $opts .= ",throttling.iops$qmpname-max-length=$v";
1604 if ($live_restore_name) {
1605 $format = "rbd" if $is_rbd;
1606 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1608 $opts .= ",format=alloc-track,file.driver=$format";
1610 $opts .= ",format=$format";
1613 my $cache_direct = drive_uses_cache_direct
($drive, $scfg);
1615 $opts .= ",cache=none" if !$drive->{cache
} && $cache_direct;
1617 if (!$drive->{aio
}) {
1618 if ($io_uring && storage_allows_io_uring_default
($scfg, $cache_direct)) {
1619 # io_uring supports all cache modes
1620 $opts .= ",aio=io_uring";
1622 # aio native works only with O_DIRECT
1624 $opts .= ",aio=native";
1626 $opts .= ",aio=threads";
1631 if (!drive_is_cdrom
($drive)) {
1633 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1634 $detectzeroes = 'off';
1635 } elsif ($drive->{discard
}) {
1636 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1638 # This used to be our default with discard not being specified:
1639 $detectzeroes = 'on';
1642 # note: 'detect-zeroes' works per blockdev and we want it to persist
1643 # after the alloc-track is removed, so put it on 'file' directly
1644 my $dz_param = $live_restore_name ?
"file.detect-zeroes" : "detect-zeroes";
1645 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1648 if ($live_restore_name) {
1649 $opts .= ",backing=$live_restore_name";
1650 $opts .= ",auto-remove=on";
1653 # my $file_param = $live_restore_name ? "file.file.filename" : "file";
1654 my $file_param = "file";
1655 if ($live_restore_name) {
1656 # non-rbd drivers require the underlying file to be a seperate block
1657 # node, so add a second .file indirection
1658 $file_param .= ".file" if !$is_rbd;
1659 $file_param .= ".filename";
1661 my $pathinfo = $path ?
"$file_param=$path," : '';
1663 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1666 sub print_pbs_blockdev
{
1667 my ($pbs_conf, $pbs_name) = @_;
1668 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1669 $blockdev .= ",repository=$pbs_conf->{repository}";
1670 $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace
};
1671 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1672 $blockdev .= ",archive=$pbs_conf->{archive}";
1673 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1677 sub print_netdevice_full
{
1678 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1680 my $device = $net->{model
};
1681 if ($net->{model
} eq 'virtio') {
1682 $device = 'virtio-net-pci';
1685 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1686 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1687 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1688 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1689 # and out of each queue plus one config interrupt and control vector queue
1690 my $vectors = $net->{queues
} * 2 + 2;
1691 $tmpstr .= ",vectors=$vectors,mq=on";
1692 if (min_version
($machine_version, 7, 1)) {
1693 $tmpstr .= ",packed=on";
1697 if (min_version
($machine_version, 7, 1) && $net->{model
} eq 'virtio'){
1698 $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1701 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1703 if (my $mtu = $net->{mtu
}) {
1704 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1705 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1708 } elsif ($mtu < 576) {
1709 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1710 } elsif ($mtu > $bridge_mtu) {
1711 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1713 $tmpstr .= ",host_mtu=$mtu";
1715 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1719 if ($use_old_bios_files) {
1721 if ($device eq 'virtio-net-pci') {
1722 $romfile = 'pxe-virtio.rom';
1723 } elsif ($device eq 'e1000') {
1724 $romfile = 'pxe-e1000.rom';
1725 } elsif ($device eq 'e1000e') {
1726 $romfile = 'pxe-e1000e.rom';
1727 } elsif ($device eq 'ne2k') {
1728 $romfile = 'pxe-ne2k_pci.rom';
1729 } elsif ($device eq 'pcnet') {
1730 $romfile = 'pxe-pcnet.rom';
1731 } elsif ($device eq 'rtl8139') {
1732 $romfile = 'pxe-rtl8139.rom';
1734 $tmpstr .= ",romfile=$romfile" if $romfile;
1740 sub print_netdev_full
{
1741 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1744 if ($netid =~ m/^net(\d+)$/) {
1748 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1750 my $ifname = "tap${vmid}i$i";
1752 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1753 die "interface name '$ifname' is too long (max 15 character)\n"
1754 if length($ifname) >= 16;
1756 my $vhostparam = '';
1757 if (is_native_arch
($arch)) {
1758 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1761 my $vmname = $conf->{name
} || "vm$vmid";
1764 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1766 if ($net->{bridge
}) {
1767 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1768 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1770 $netdev = "type=user,id=$netid,hostname=$vmname";
1773 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1779 'cirrus' => 'cirrus-vga',
1781 'vmware' => 'vmware-svga',
1782 'virtio' => 'virtio-vga',
1783 'virtio-gl' => 'virtio-vga-gl',
1786 sub print_vga_device
{
1787 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1789 my $type = $vga_map->{$vga->{type
}};
1790 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1791 $type = 'virtio-gpu';
1793 my $vgamem_mb = $vga->{memory
};
1795 my $max_outputs = '';
1797 $type = $id ?
'qxl' : 'qxl-vga';
1799 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1800 # set max outputs so linux can have up to 4 qxl displays with one device
1801 if (min_version
($machine_version, 4, 1)) {
1802 $max_outputs = ",max_outputs=4";
1807 die "no devicetype for $vga->{type}\n" if !$type;
1811 if ($vga->{type
} =~ /^virtio/) {
1812 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1813 $memory = ",max_hostmem=$bytes";
1815 # from https://www.spice-space.org/multiple-monitors.html
1816 $memory = ",vgamem_mb=$vga->{memory}";
1817 my $ram = $vgamem_mb * 4;
1818 my $vram = $vgamem_mb * 2;
1819 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1821 $memory = ",vgamem_mb=$vga->{memory}";
1823 } elsif ($qxlnum && $id) {
1824 $memory = ",ram_size=67108864,vram_size=33554432";
1828 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1829 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1832 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1833 my $vgaid = "vga" . ($id // '');
1835 if ($q35 && $vgaid eq 'vga') {
1836 # the first display uses pcie.0 bus on q35 machines
1837 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1839 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1842 if ($vga->{type
} eq 'virtio-gl') {
1843 my $base = '/usr/lib/x86_64-linux-gnu/lib';
1844 die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1845 if !-e
"${base}EGL.so.1" || !-e
"${base}GL.so.1";
1847 die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1848 if !PVE
::Tools
::dir_glob_regex
('/dev/dri/', "renderD.*");
1851 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1854 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1856 my ($data, $disable_mac_autogen) = @_;
1858 my $res = eval { parse_property_string
($net_fmt, $data) };
1863 if (!defined($res->{macaddr
}) && !$disable_mac_autogen) {
1864 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1865 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1870 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1871 sub parse_ipconfig
{
1874 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1880 if ($res->{gw
} && !$res->{ip
}) {
1881 warn 'gateway specified without specifying an IP address';
1884 if ($res->{gw6
} && !$res->{ip6
}) {
1885 warn 'IPv6 gateway specified without specifying an IPv6 address';
1888 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1889 warn 'gateway specified together with DHCP';
1892 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1894 warn "IPv6 gateway specified together with $res->{ip6} address";
1898 if (!$res->{ip
} && !$res->{ip6
}) {
1899 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1908 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1911 sub add_random_macs
{
1912 my ($settings) = @_;
1914 foreach my $opt (keys %$settings) {
1915 next if $opt !~ m/^net(\d+)$/;
1916 my $net = parse_net
($settings->{$opt});
1918 $settings->{$opt} = print_net
($net);
1922 sub vm_is_volid_owner
{
1923 my ($storecfg, $vmid, $volid) = @_;
1925 if ($volid !~ m
|^/|) {
1927 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1928 if ($owner && ($owner == $vmid)) {
1936 sub vmconfig_register_unused_drive
{
1937 my ($storecfg, $vmid, $conf, $drive) = @_;
1939 if (drive_is_cloudinit
($drive)) {
1940 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1942 delete $conf->{cloudinit
};
1943 } elsif (!drive_is_cdrom
($drive)) {
1944 my $volid = $drive->{file
};
1945 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1946 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1951 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1955 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1956 format_description
=> 'UUID',
1957 description
=> "Set SMBIOS1 UUID.",
1962 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1963 format_description
=> 'Base64 encoded string',
1964 description
=> "Set SMBIOS1 version.",
1969 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1970 format_description
=> 'Base64 encoded string',
1971 description
=> "Set SMBIOS1 serial number.",
1976 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1977 format_description
=> 'Base64 encoded string',
1978 description
=> "Set SMBIOS1 manufacturer.",
1983 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1984 format_description
=> 'Base64 encoded string',
1985 description
=> "Set SMBIOS1 product ID.",
1990 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1991 format_description
=> 'Base64 encoded string',
1992 description
=> "Set SMBIOS1 SKU string.",
1997 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1998 format_description
=> 'Base64 encoded string',
1999 description
=> "Set SMBIOS1 family string.",
2004 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2012 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2019 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2022 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2024 sub parse_watchdog
{
2029 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2034 sub parse_guest_agent
{
2037 return {} if !defined($conf->{agent
});
2039 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2042 # if the agent is disabled ignore the other potentially set properties
2043 return {} if !$res->{enabled
};
2048 my ($conf, $key) = @_;
2049 return undef if !defined($conf->{agent
});
2051 my $agent = parse_guest_agent
($conf);
2052 return $agent->{$key};
2058 return {} if !$value;
2059 my $res = eval { parse_property_string
($vga_fmt, $value) };
2069 my $res = eval { parse_property_string
($rng_fmt, $value) };
2074 sub parse_meta_info
{
2079 my $res = eval { parse_property_string
($meta_info_fmt, $value) };
2084 sub new_meta_info_string
{
2085 my () = @_; # for now do not allow to override any value
2087 return PVE
::JSONSchema
::print_property_string
(
2089 'creation-qemu' => kvm_user_version
(),
2090 ctime
=> "". int(time()),
2096 sub qemu_created_version_fixups
{
2097 my ($conf, $forcemachine, $kvmver) = @_;
2099 my $meta = parse_meta_info
($conf->{meta
}) // {};
2100 my $forced_vers = PVE
::QemuServer
::Machine
::extract_version
($forcemachine);
2102 # check if we need to apply some handling for VMs that always use the latest machine version but
2103 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2104 # would be required (we do not want to pin machine version for non-windows OS type)
2106 (!defined($conf->{machine
}) || $conf->{machine
} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2107 && (!defined($meta->{'creation-qemu'}) || !min_version
($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2108 && (!$forced_vers || min_version
($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2109 && min_version
($kvmver, 6, 1) # only need to apply the change since 6.1
2111 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2112 if ($q35 && $conf->{ostype
} && $conf->{ostype
} eq 'l26') {
2113 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2114 # and thus with the predictable interface naming of systemd
2115 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2121 # add JSON properties for create and set function
2122 sub json_config_properties
{
2123 my ($prop, $with_disk_alloc) = @_;
2125 my $skip_json_config_opts = {
2129 runningmachine
=> 1,
2134 foreach my $opt (keys %$confdesc) {
2135 next if $skip_json_config_opts->{$opt};
2137 if ($with_disk_alloc && is_valid_drivename
($opt)) {
2138 $prop->{$opt} = $PVE::QemuServer
::Drive
::drivedesc_hash_with_alloc-
>{$opt};
2140 $prop->{$opt} = $confdesc->{$opt};
2147 # Properties that we can read from an OVF file
2148 sub json_ovf_properties
{
2151 for my $device (PVE
::QemuServer
::Drive
::valid_drive_names
()) {
2152 $prop->{$device} = {
2154 format
=> 'pve-volume-id-or-absolute-path',
2155 description
=> "Disk image that gets imported to $device",
2162 description
=> "The number of CPU cores.",
2167 description
=> "Amount of RAM for the VM in MB.",
2172 description
=> "Name of the VM.",
2179 # return copy of $confdesc_cloudinit to generate documentation
2180 sub cloudinit_config_properties
{
2182 return dclone
($confdesc_cloudinit);
2185 sub cloudinit_pending_properties
{
2187 map { $_ => 1 } keys $confdesc_cloudinit->%*,
2190 $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2195 my ($key, $value) = @_;
2197 die "unknown setting '$key'\n" if !$confdesc->{$key};
2199 my $type = $confdesc->{$key}->{type
};
2201 if (!defined($value)) {
2202 die "got undefined value\n";
2205 if ($value =~ m/[\n\r]/) {
2206 die "property contains a line feed\n";
2209 if ($type eq 'boolean') {
2210 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2211 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2212 die "type check ('boolean') failed - got '$value'\n";
2213 } elsif ($type eq 'integer') {
2214 return int($1) if $value =~ m/^(\d+)$/;
2215 die "type check ('integer') failed - got '$value'\n";
2216 } elsif ($type eq 'number') {
2217 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2218 die "type check ('number') failed - got '$value'\n";
2219 } elsif ($type eq 'string') {
2220 if (my $fmt = $confdesc->{$key}->{format
}) {
2221 PVE
::JSONSchema
::check_format
($fmt, $value);
2224 $value =~ s/^\"(.*)\"$/$1/;
2227 die "internal error"
2232 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2234 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2236 if (!$skiplock && !PVE
::QemuConfig-
>has_lock($conf, 'suspended')) {
2237 PVE
::QemuConfig-
>check_lock($conf);
2240 if ($conf->{template
}) {
2241 # check if any base image is still used by a linked clone
2242 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2243 my ($ds, $drive) = @_;
2244 return if drive_is_cdrom
($drive);
2246 my $volid = $drive->{file
};
2247 return if !$volid || $volid =~ m
|^/|;
2249 die "base volume '$volid' is still in use by linked cloned\n"
2250 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2256 my $remove_owned_drive = sub {
2257 my ($ds, $drive) = @_;
2258 return if drive_is_cdrom
($drive, 1);
2260 my $volid = $drive->{file
};
2261 return if !$volid || $volid =~ m
|^/|;
2262 return if $volids->{$volid};
2264 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2265 return if !$path || !$owner || ($owner != $vmid);
2267 $volids->{$volid} = 1;
2268 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2269 warn "Could not remove disk '$volid', check manually: $@" if $@;
2272 # only remove disks owned by this VM (referenced in the config)
2273 my $include_opts = {
2274 include_unused
=> 1,
2275 extra_keys
=> ['vmstate'],
2277 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2279 for my $snap (values %{$conf->{snapshots
}}) {
2280 next if !defined($snap->{vmstate
});
2281 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2282 next if !defined($drive);
2283 $remove_owned_drive->('vmstate', $drive);
2286 PVE
::QemuConfig-
>foreach_volume_full($conf->{pending
}, $include_opts, $remove_owned_drive);
2288 if ($purge_unreferenced) { # also remove unreferenced disk
2289 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2290 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2291 my ($volid, $sid, $volname, $d) = @_;
2292 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2297 eval { delete_ifaces_ipams_ips
($conf, $vmid)};
2300 if (defined $replacement_conf) {
2301 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2303 PVE
::QemuConfig-
>destroy_config($vmid);
2307 sub parse_vm_config
{
2308 my ($filename, $raw, $strict) = @_;
2310 return if !defined($raw);
2313 digest
=> Digest
::SHA
::sha1_hex
($raw),
2319 my $handle_error = sub {
2329 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2330 || die "got strange filename '$filename'";
2336 my $finish_description = sub {
2337 if (defined($descr)) {
2339 $conf->{description
} = $descr;
2345 my @lines = split(/\n/, $raw);
2346 foreach my $line (@lines) {
2347 next if $line =~ m/^\s*$/;
2349 if ($line =~ m/^\[PENDING\]\s*$/i) {
2350 $section = 'pending';
2351 $finish_description->();
2352 $conf = $res->{$section} = {};
2354 } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2355 $section = 'cloudinit';
2356 $finish_description->();
2357 $conf = $res->{$section} = {};
2360 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2362 $finish_description->();
2363 $conf = $res->{snapshots
}->{$section} = {};
2367 if ($line =~ m/^\#(.*)$/) {
2368 $descr = '' if !defined($descr);
2369 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2373 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2374 $descr = '' if !defined($descr);
2375 $descr .= PVE
::Tools
::decode_text
($2);
2376 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2377 $conf->{snapstate
} = $1;
2378 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2381 $conf->{$key} = $value;
2382 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2384 if ($section eq 'pending') {
2385 $conf->{delete} = $value; # we parse this later
2387 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2389 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2392 if ($section eq 'cloudinit') {
2393 # ignore validation only used for informative purpose
2394 $conf->{$key} = $value;
2397 eval { $value = check_type
($key, $value); };
2399 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2401 $key = 'ide2' if $key eq 'cdrom';
2402 my $fmt = $confdesc->{$key}->{format
};
2403 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2404 my $v = parse_drive
($key, $value);
2405 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2406 $v->{file
} = $volid;
2407 $value = print_drive
($v);
2409 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2414 $conf->{$key} = $value;
2417 $handle_error->("vm $vmid - unable to parse config: $line\n");
2421 $finish_description->();
2422 delete $res->{snapstate
}; # just to be sure
2427 sub write_vm_config
{
2428 my ($filename, $conf) = @_;
2430 delete $conf->{snapstate
}; # just to be sure
2432 if ($conf->{cdrom
}) {
2433 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2434 $conf->{ide2
} = $conf->{cdrom
};
2435 delete $conf->{cdrom
};
2438 # we do not use 'smp' any longer
2439 if ($conf->{sockets
}) {
2440 delete $conf->{smp
};
2441 } elsif ($conf->{smp
}) {
2442 $conf->{sockets
} = $conf->{smp
};
2443 delete $conf->{cores
};
2444 delete $conf->{smp
};
2447 my $used_volids = {};
2449 my $cleanup_config = sub {
2450 my ($cref, $pending, $snapname) = @_;
2452 foreach my $key (keys %$cref) {
2453 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2454 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2455 my $value = $cref->{$key};
2456 if ($key eq 'delete') {
2457 die "propertry 'delete' is only allowed in [PENDING]\n"
2459 # fixme: check syntax?
2462 eval { $value = check_type
($key, $value); };
2463 die "unable to parse value of '$key' - $@" if $@;
2465 $cref->{$key} = $value;
2467 if (!$snapname && is_valid_drivename
($key)) {
2468 my $drive = parse_drive
($key, $value);
2469 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2474 &$cleanup_config($conf);
2476 &$cleanup_config($conf->{pending
}, 1);
2478 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2479 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2480 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2483 # remove 'unusedX' settings if we re-add a volume
2484 foreach my $key (keys %$conf) {
2485 my $value = $conf->{$key};
2486 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2487 delete $conf->{$key};
2491 my $generate_raw_config = sub {
2492 my ($conf, $pending) = @_;
2496 # add description as comment to top of file
2497 if (defined(my $descr = $conf->{description
})) {
2499 foreach my $cl (split(/\n/, $descr)) {
2500 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2503 $raw .= "#\n" if $pending;
2507 foreach my $key (sort keys %$conf) {
2508 next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2509 $raw .= "$key: $conf->{$key}\n";
2514 my $raw = &$generate_raw_config($conf);
2516 if (scalar(keys %{$conf->{pending
}})){
2517 $raw .= "\n[PENDING]\n";
2518 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2521 if (scalar(keys %{$conf->{cloudinit
}}) && PVE
::QemuConfig-
>has_cloudinit($conf)){
2522 $raw .= "\n[special:cloudinit]\n";
2523 $raw .= &$generate_raw_config($conf->{cloudinit
});
2526 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2527 $raw .= "\n[$snapname]\n";
2528 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2538 # we use static defaults from our JSON schema configuration
2539 foreach my $key (keys %$confdesc) {
2540 if (defined(my $default = $confdesc->{$key}->{default})) {
2541 $res->{$key} = $default;
2549 my $vmlist = PVE
::Cluster
::get_vmlist
();
2551 return $res if !$vmlist || !$vmlist->{ids
};
2552 my $ids = $vmlist->{ids
};
2553 my $nodename = nodename
();
2555 foreach my $vmid (keys %$ids) {
2556 my $d = $ids->{$vmid};
2557 next if !$d->{node
} || $d->{node
} ne $nodename;
2558 next if !$d->{type
} || $d->{type
} ne 'qemu';
2559 $res->{$vmid}->{exists} = 1;
2564 # test if VM uses local resources (to prevent migration)
2565 sub check_local_resources
{
2566 my ($conf, $noerr) = @_;
2569 my $mapped_res = [];
2571 my $nodelist = PVE
::Cluster
::get_nodelist
();
2572 my $pci_map = PVE
::Mapping
::PCI
::config
();
2573 my $usb_map = PVE
::Mapping
::USB
::config
();
2575 my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2577 my $add_missing_mapping = sub {
2578 my ($type, $key, $id) = @_;
2579 for my $node (@$nodelist) {
2581 if ($type eq 'pci') {
2582 $entry = PVE
::Mapping
::PCI
::get_node_mapping
($pci_map, $id, $node);
2583 } elsif ($type eq 'usb') {
2584 $entry = PVE
::Mapping
::USB
::get_node_mapping
($usb_map, $id, $node);
2586 if (!scalar($entry->@*)) {
2587 push @{$missing_mappings_by_node->{$node}}, $key;
2592 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2593 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2595 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2597 foreach my $k (keys %$conf) {
2598 if ($k =~ m/^usb/) {
2599 my $entry = parse_property_string
('pve-qm-usb', $conf->{$k});
2600 next if $entry->{host
} =~ m/^spice$/i;
2601 if ($entry->{mapping
}) {
2602 $add_missing_mapping->('usb', $k, $entry->{mapping
});
2603 push @$mapped_res, $k;
2606 if ($k =~ m/^hostpci/) {
2607 my $entry = parse_property_string
('pve-qm-hostpci', $conf->{$k});
2608 if ($entry->{mapping
}) {
2609 $add_missing_mapping->('pci', $k, $entry->{mapping
});
2610 push @$mapped_res, $k;
2613 # sockets are safe: they will recreated be on the target side post-migrate
2614 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2615 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2618 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2620 return wantarray ?
(\
@loc_res, $mapped_res, $missing_mappings_by_node) : \
@loc_res;
2623 # check if used storages are available on all nodes (use by migrate)
2624 sub check_storage_availability
{
2625 my ($storecfg, $conf, $node) = @_;
2627 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2628 my ($ds, $drive) = @_;
2630 my $volid = $drive->{file
};
2633 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2636 # check if storage is available on both nodes
2637 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2638 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2640 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2642 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2643 if !$scfg->{content
}->{$vtype};
2647 # list nodes where all VM images are available (used by has_feature API)
2649 my ($conf, $storecfg) = @_;
2651 my $nodelist = PVE
::Cluster
::get_nodelist
();
2652 my $nodehash = { map { $_ => 1 } @$nodelist };
2653 my $nodename = nodename
();
2655 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2656 my ($ds, $drive) = @_;
2658 my $volid = $drive->{file
};
2661 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2663 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2664 if ($scfg->{disable
}) {
2666 } elsif (my $avail = $scfg->{nodes
}) {
2667 foreach my $node (keys %$nodehash) {
2668 delete $nodehash->{$node} if !$avail->{$node};
2670 } elsif (!$scfg->{shared
}) {
2671 foreach my $node (keys %$nodehash) {
2672 delete $nodehash->{$node} if $node ne $nodename
2681 sub check_local_storage_availability
{
2682 my ($conf, $storecfg) = @_;
2684 my $nodelist = PVE
::Cluster
::get_nodelist
();
2685 my $nodehash = { map { $_ => {} } @$nodelist };
2687 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2688 my ($ds, $drive) = @_;
2690 my $volid = $drive->{file
};
2693 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2695 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2697 if ($scfg->{disable
}) {
2698 foreach my $node (keys %$nodehash) {
2699 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2701 } elsif (my $avail = $scfg->{nodes
}) {
2702 foreach my $node (keys %$nodehash) {
2703 if (!$avail->{$node}) {
2704 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2711 foreach my $node (values %$nodehash) {
2712 if (my $unavail = $node->{unavailable_storages
}) {
2713 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2720 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2722 my ($vmid, $nocheck, $node) = @_;
2724 # $nocheck is set when called during a migration, in which case the config
2725 # file might still or already reside on the *other* node
2726 # - because rename has already happened, and current node is source
2727 # - because rename hasn't happened yet, and current node is target
2728 # - because rename has happened, current node is target, but hasn't yet
2730 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2731 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2736 my $vzlist = config_list
();
2738 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2740 while (defined(my $de = $fd->read)) {
2741 next if $de !~ m/^(\d+)\.pid$/;
2743 next if !defined($vzlist->{$vmid});
2744 if (my $pid = check_running
($vmid)) {
2745 $vzlist->{$vmid}->{pid
} = $pid;
2752 our $vmstatus_return_properties = {
2753 vmid
=> get_standard_option
('pve-vmid'),
2755 description
=> "QEMU process status.",
2757 enum
=> ['stopped', 'running'],
2760 description
=> "Maximum memory in bytes.",
2763 renderer
=> 'bytes',
2766 description
=> "Root disk size in bytes.",
2769 renderer
=> 'bytes',
2772 description
=> "VM name.",
2777 description
=> "VM run state from the 'query-status' QMP monitor command.",
2782 description
=> "PID of running qemu process.",
2787 description
=> "Uptime.",
2790 renderer
=> 'duration',
2793 description
=> "Maximum usable CPUs.",
2798 description
=> "The current config lock, if any.",
2803 description
=> "The current configured tags, if any",
2807 'running-machine' => {
2808 description
=> "The currently running machine type (if running).",
2813 description
=> "The currently running QEMU version (if running).",
2819 my $last_proc_pid_stat;
2821 # get VM status information
2822 # This must be fast and should not block ($full == false)
2823 # We only query KVM using QMP if $full == true (this can be slow)
2825 my ($opt_vmid, $full) = @_;
2829 my $storecfg = PVE
::Storage
::config
();
2831 my $list = vzlist
();
2832 my $defaults = load_defaults
();
2834 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2836 my $cpucount = $cpuinfo->{cpus
} || 1;
2838 foreach my $vmid (keys %$list) {
2839 next if $opt_vmid && ($vmid ne $opt_vmid);
2841 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2843 my $d = { vmid
=> int($vmid) };
2844 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2846 # fixme: better status?
2847 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2849 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2850 if (defined($size)) {
2851 $d->{disk
} = 0; # no info available
2852 $d->{maxdisk
} = $size;
2858 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2859 * ($conf->{cores
} || $defaults->{cores
});
2860 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2861 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2863 $d->{name
} = $conf->{name
} || "VM $vmid";
2864 $d->{maxmem
} = get_current_memory
($conf->{memory
})*(1024*1024);
2866 if ($conf->{balloon
}) {
2867 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2868 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2869 : $defaults->{shares
};
2880 $d->{diskwrite
} = 0;
2882 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2884 $d->{serial
} = 1 if conf_has_serial
($conf);
2885 $d->{lock} = $conf->{lock} if $conf->{lock};
2886 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2891 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2892 foreach my $dev (keys %$netdev) {
2893 next if $dev !~ m/^tap([1-9]\d*)i/;
2895 my $d = $res->{$vmid};
2898 $d->{netout
} += $netdev->{$dev}->{receive
};
2899 $d->{netin
} += $netdev->{$dev}->{transmit
};
2902 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2903 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2908 my $ctime = gettimeofday
;
2910 foreach my $vmid (keys %$list) {
2912 my $d = $res->{$vmid};
2913 my $pid = $d->{pid
};
2916 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2917 next if !$pstat; # not running
2919 my $used = $pstat->{utime} + $pstat->{stime
};
2921 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2923 if ($pstat->{vsize
}) {
2924 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2927 my $old = $last_proc_pid_stat->{$pid};
2929 $last_proc_pid_stat->{$pid} = {
2937 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2939 if ($dtime > 1000) {
2940 my $dutime = $used - $old->{used
};
2942 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2943 $last_proc_pid_stat->{$pid} = {
2949 $d->{cpu
} = $old->{cpu
};
2953 return $res if !$full;
2955 my $qmpclient = PVE
::QMPClient-
>new();
2957 my $ballooncb = sub {
2958 my ($vmid, $resp) = @_;
2960 my $info = $resp->{'return'};
2961 return if !$info->{max_mem
};
2963 my $d = $res->{$vmid};
2965 # use memory assigned to VM
2966 $d->{maxmem
} = $info->{max_mem
};
2967 $d->{balloon
} = $info->{actual
};
2969 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2970 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2971 $d->{freemem
} = $info->{free_mem
};
2974 $d->{ballooninfo
} = $info;
2977 my $blockstatscb = sub {
2978 my ($vmid, $resp) = @_;
2979 my $data = $resp->{'return'} || [];
2980 my $totalrdbytes = 0;
2981 my $totalwrbytes = 0;
2983 for my $blockstat (@$data) {
2984 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2985 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2987 $blockstat->{device
} =~ s/drive-//;
2988 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2990 $res->{$vmid}->{diskread
} = $totalrdbytes;
2991 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2994 my $machinecb = sub {
2995 my ($vmid, $resp) = @_;
2996 my $data = $resp->{'return'} || [];
2998 $res->{$vmid}->{'running-machine'} =
2999 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
3002 my $versioncb = sub {
3003 my ($vmid, $resp) = @_;
3004 my $data = $resp->{'return'} // {};
3005 my $version = 'unknown';
3007 if (my $v = $data->{qemu
}) {
3008 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
3011 $res->{$vmid}->{'running-qemu'} = $version;
3014 my $statuscb = sub {
3015 my ($vmid, $resp) = @_;
3017 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3018 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3019 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3020 # this fails if ballon driver is not loaded, so this must be
3021 # the last commnand (following command are aborted if this fails).
3022 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3024 my $status = 'unknown';
3025 if (!defined($status = $resp->{'return'}->{status
})) {
3026 warn "unable to get VM status\n";
3030 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3033 foreach my $vmid (keys %$list) {
3034 next if $opt_vmid && ($vmid ne $opt_vmid);
3035 next if !$res->{$vmid}->{pid
}; # not running
3036 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3039 $qmpclient->queue_execute(undef, 2);
3041 foreach my $vmid (keys %$list) {
3042 next if $opt_vmid && ($vmid ne $opt_vmid);
3043 next if !$res->{$vmid}->{pid
}; #not running
3045 # we can't use the $qmpclient since it might have already aborted on
3046 # 'query-balloon', but this might also fail for older versions...
3047 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
3048 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3051 foreach my $vmid (keys %$list) {
3052 next if $opt_vmid && ($vmid ne $opt_vmid);
3053 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3059 sub conf_has_serial
{
3062 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3063 if ($conf->{"serial$i"}) {
3071 sub conf_has_audio
{
3072 my ($conf, $id) = @_;
3075 my $audio = $conf->{"audio$id"};
3076 return if !defined($audio);
3078 my $audioproperties = parse_property_string
($audio_fmt, $audio);
3079 my $audiodriver = $audioproperties->{driver
} // 'spice';
3082 dev
=> $audioproperties->{device
},
3083 dev_id
=> "audiodev$id",
3084 backend
=> $audiodriver,
3085 backend_id
=> "$audiodriver-backend${id}",
3090 my ($audio, $audiopciaddr, $machine_version) = @_;
3094 my $id = $audio->{dev_id
};
3096 if (min_version
($machine_version, 4, 2)) {
3097 $audiodev = ",audiodev=$audio->{backend_id}";
3100 if ($audio->{dev
} eq 'AC97') {
3101 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3102 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3103 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3104 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3105 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3107 die "unkown audio device '$audio->{dev}', implement me!";
3110 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3118 socket => "/var/run/qemu-server/$vmid.swtpm",
3119 pid
=> "/var/run/qemu-server/$vmid.swtpm.pid",
3123 sub add_tpm_device
{
3124 my ($vmid, $devices, $conf) = @_;
3126 return if !$conf->{tpmstate0
};
3128 my $paths = get_tpm_paths
($vmid);
3130 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3131 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3132 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3136 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3138 return if !$tpmdrive;
3141 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
3142 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
3144 $state = PVE
::Storage
::map_volume
($storecfg, $tpm->{file
});
3146 $state = $tpm->{file
};
3149 my $paths = get_tpm_paths
($vmid);
3151 # during migration, we will get state from remote
3154 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3161 "--create-platform-cert",
3164 "/etc/swtpm_setup.conf", # do not use XDG configs
3166 "0", # force creation as root, error if not possible
3167 "--not-overwrite", # ignore existing state, do not modify
3170 push @$setup_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3171 # TPM 2.0 supports ECC crypto, use if possible
3172 push @$setup_cmd, "--ecc" if $tpm->{version
} eq 'v2.0';
3174 run_command
($setup_cmd, outfunc
=> sub {
3175 print "swtpm_setup: $1\n";
3179 # Used to distinguish different invocations in the log.
3180 my $log_prefix = "[id=" . int(time()) . "] ";
3182 my $emulator_cmd = [
3186 "backend-uri=file://$state,mode=0600",
3188 "type=unixio,path=$paths->{socket},mode=0600",
3190 "file=$paths->{pid}",
3191 "--terminate", # terminate on QEMU disconnect
3194 "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3196 push @$emulator_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3197 run_command
($emulator_cmd, outfunc
=> sub { print $1; });
3199 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3200 while (! -e
$paths->{pid
}) {
3201 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3205 # return untainted PID of swtpm daemon so it can be killed on error
3206 file_read_firstline
($paths->{pid
}) =~ m/(\d+)/;
3210 sub vga_conf_has_spice
{
3213 my $vgaconf = parse_vga
($vga);
3214 my $vgatype = $vgaconf->{type
};
3215 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3222 return $conf->{arch
} // get_host_arch
();
3225 my $default_machines = {
3230 sub get_installed_machine_version
{
3231 my ($kvmversion) = @_;
3232 $kvmversion = kvm_user_version
() if !defined($kvmversion);
3233 $kvmversion =~ m/^(\d+\.\d+)/;
3237 sub windows_get_pinned_machine_version
{
3238 my ($machine, $base_version, $kvmversion) = @_;
3240 my $pin_version = $base_version;
3241 if (!defined($base_version) ||
3242 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3244 $pin_version = get_installed_machine_version
($kvmversion);
3246 if (!$machine || $machine eq 'pc') {
3247 $machine = "pc-i440fx-$pin_version";
3248 } elsif ($machine eq 'q35') {
3249 $machine = "pc-q35-$pin_version";
3250 } elsif ($machine eq 'virt') {
3251 $machine = "virt-$pin_version";
3253 warn "unknown machine type '$machine', not touching that!\n";
3259 sub get_vm_machine
{
3260 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3262 my $machine = $forcemachine || $conf->{machine
};
3264 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3265 $kvmversion //= kvm_user_version
();
3266 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3267 # layout which confuses windows quite a bit and may result in various regressions..
3268 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3269 if (windows_version
($conf->{ostype
})) {
3270 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3273 $machine ||= $default_machines->{$arch};
3274 if ($add_pve_version) {
3275 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3276 $machine .= "+pve$pvever";
3280 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3281 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3282 $machine = $1 if $is_pxe;
3284 # for version-pinned machines that do not include a pve-version (e.g.
3285 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3286 $machine .= '+pve0';
3288 $machine .= '.pxe' if $is_pxe;
3294 sub get_ovmf_files
($$$) {
3295 my ($arch, $efidisk, $smm) = @_;
3297 my $types = $OVMF->{$arch}
3298 or die "no OVMF images known for architecture '$arch'\n";
3300 my $type = 'default';
3301 if ($arch eq 'x86_64') {
3302 if (defined($efidisk->{efitype
}) && $efidisk->{efitype
} eq '4m') {
3303 $type = $smm ?
"4m" : "4m-no-smm";
3304 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3306 # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3310 my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3311 die "EFI base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3312 die "EFI vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
3314 return ($ovmf_code, $ovmf_vars);
3318 aarch64
=> '/usr/bin/qemu-system-aarch64',
3319 x86_64
=> '/usr/bin/qemu-system-x86_64',
3321 sub get_command_for_arch
($) {
3323 return '/usr/bin/kvm' if is_native_arch
($arch);
3325 my $cmd = $Arch2Qemu->{$arch}
3326 or die "don't know how to emulate architecture '$arch'\n";
3330 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3331 # to use in a QEMU command line (-cpu element), first array_intersect the result
3332 # of query_supported_ with query_understood_. This is necessary because:
3334 # a) query_understood_ returns flags the host cannot use and
3335 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3336 # flags, but CPU settings - with most of them being flags. Those settings
3337 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3339 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3340 # expensive. If you need the value returned from this, you can get it much
3341 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3342 # $accel being 'kvm' or 'tcg'.
3344 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3345 # changes, automatically populating pmxcfs.
3347 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3348 # since kvm and tcg machines support different flags
3350 sub query_supported_cpu_flags
{
3353 $arch //= get_host_arch
();
3354 my $default_machine = $default_machines->{$arch};
3358 # FIXME: Once this is merged, the code below should work for ARM as well:
3359 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3360 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3363 my $kvm_supported = defined(kvm_version
());
3364 my $qemu_cmd = get_command_for_arch
($arch);
3366 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3368 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3369 my $query_supported_run_qemu = sub {
3375 '-machine', $default_machine,
3377 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3378 '-mon', 'chardev=qmp,mode=control',
3379 '-pidfile', $pidfile,
3384 push @$cmd, '-accel', 'tcg';
3387 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3388 die "QEMU flag querying VM exited with code " . $rc if $rc;
3391 my $cmd_result = mon_cmd
(
3393 'query-cpu-model-expansion',
3395 model
=> { name
=> 'host' }
3398 my $props = $cmd_result->{model
}->{props
};
3399 foreach my $prop (keys %$props) {
3400 next if $props->{$prop} ne '1';
3401 # QEMU returns some flags multiple times, with '_', '.' or '-'
3402 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3403 # We only keep those with underscores, to match /proc/cpuinfo
3404 $prop =~ s/\.|-/_/g;
3405 $flags->{$prop} = 1;
3410 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3411 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3415 return [ sort keys %$flags ];
3418 # We need to query QEMU twice, since KVM and TCG have different supported flags
3419 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3420 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3421 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3423 if ($kvm_supported) {
3424 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3425 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3432 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3433 my $understood_cpu_flag_dir = "/usr/share/kvm";
3434 sub query_understood_cpu_flags
{
3435 my $arch = get_host_arch
();
3436 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3438 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3441 my $raw = file_get_contents
($filepath);
3442 $raw =~ s/^\s+|\s+$//g;
3443 my @flags = split(/\s+/, $raw);
3448 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3449 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3450 my sub should_disable_smm
{
3451 my ($conf, $vga, $machine) = @_;
3453 return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3455 return (!defined($conf->{bios
}) || $conf->{bios
} eq 'seabios') &&
3456 $vga->{type
} && $vga->{type
} =~ m/^(serial\d+|none)$/;
3459 my sub print_ovmf_drive_commandlines
{
3460 my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3462 my $d = $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
3464 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch, $d, $q35);
3466 my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3468 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3469 my ($path, $format) = $d->@{'file', 'format'};
3471 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3472 if (!defined($format)) {
3473 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3474 $format = qemu_img_format
($scfg, $volname);
3476 } elsif (!defined($format)) {
3477 die "efidisk format must be specified\n";
3479 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3480 if ($path =~ m/^rbd:/) {
3481 $var_drive_str .= ',cache=writeback';
3482 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3484 $var_drive_str .= ",format=$format,file=$path";
3486 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3487 $var_drive_str .= ',readonly=on' if drive_is_read_only
($conf, $d);
3489 log_warn
("no efidisk configured! Using temporary efivars disk.");
3490 my $path = "/tmp/$vmid-ovmf.fd";
3491 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3492 $var_drive_str .= ",format=raw,file=$path";
3493 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $version_guard->(4, 1, 2);
3496 return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3499 sub config_to_command
{
3500 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3501 $live_restore_backing) = @_;
3503 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3506 my $ostype = $conf->{ostype
};
3507 my $winversion = windows_version
($ostype);
3508 my $kvm = $conf->{kvm
};
3509 my $nodename = nodename
();
3511 my $arch = get_vm_arch
($conf);
3512 my $kvm_binary = get_command_for_arch
($arch);
3513 my $kvmver = kvm_user_version
($kvm_binary);
3515 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3516 $kvmver //= "undefined";
3517 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3520 my $add_pve_version = min_version
($kvmver, 4, 1);
3522 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3523 my $machine_version = extract_version
($machine_type, $kvmver);
3524 $kvm //= 1 if is_native_arch
($arch);
3526 $machine_version =~ m/(\d+)\.(\d+)/;
3527 my ($machine_major, $machine_minor) = ($1, $2);
3529 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3530 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3531 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3532 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3533 ." please upgrade node '$nodename'\n"
3534 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3535 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3536 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3537 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3538 ." node '$nodename'\n";
3541 # if a specific +pve version is required for a feature, use $version_guard
3542 # instead of min_version to allow machines to be run with the minimum
3544 my $required_pve_version = 0;
3545 my $version_guard = sub {
3546 my ($major, $minor, $pve) = @_;
3547 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3548 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3549 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3550 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3554 if ($kvm && !defined kvm_version
()) {
3555 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3556 ." or enable in BIOS.\n";
3559 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3560 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3561 my $use_old_bios_files = undef;
3562 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3565 if ($conf->{affinity
}) {
3566 push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity
};
3569 push @$cmd, $kvm_binary;
3571 push @$cmd, '-id', $vmid;
3573 my $vmname = $conf->{name
} || "vm$vmid";
3575 push @$cmd, '-name', "$vmname,debug-threads=on";
3577 push @$cmd, '-no-shutdown';
3581 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3582 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3583 push @$cmd, '-mon', "chardev=qmp,mode=control";
3585 if (min_version
($machine_version, 2, 12)) {
3586 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3587 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3590 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3592 push @$cmd, '-daemonize';
3594 if ($conf->{smbios1
}) {
3595 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3596 if ($smbios_conf->{base64
}) {
3597 # Do not pass base64 flag to qemu
3598 delete $smbios_conf->{base64
};
3599 my $smbios_string = "";
3600 foreach my $key (keys %$smbios_conf) {
3602 if ($key eq "uuid") {
3603 $value = $smbios_conf->{uuid
}
3605 $value = decode_base64
($smbios_conf->{$key});
3607 # qemu accepts any binary data, only commas need escaping by double comma
3609 $smbios_string .= "," . $key . "=" . $value if $value;
3611 push @$cmd, '-smbios', "type=1" . $smbios_string;
3613 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3617 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3618 die "OVMF (UEFI) BIOS is not supported on 32-bit CPU types\n"
3619 if !$forcecpu && get_cpu_bitness
($conf->{cpu
}, $arch) == 32;
3621 my ($code_drive_str, $var_drive_str) =
3622 print_ovmf_drive_commandlines
($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3623 push $cmd->@*, '-drive', $code_drive_str;
3624 push $cmd->@*, '-drive', $var_drive_str;
3627 if ($q35) { # tell QEMU to load q35 config early
3628 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3629 if (min_version
($machine_version, 4, 0)) {
3630 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3632 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3636 if (defined(my $fixups = qemu_created_version_fixups
($conf, $forcemachine, $kvmver))) {
3637 push @$cmd, $fixups->@*;
3640 if ($conf->{vmgenid
}) {
3641 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3644 # add usb controllers
3645 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3646 $conf, $bridges, $arch, $machine_type, $machine_version);
3647 push @$devices, @usbcontrollers if @usbcontrollers;
3648 my $vga = parse_vga
($conf->{vga
});
3650 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3651 $vga->{type
} = 'qxl' if $qxlnum;
3653 if (!$vga->{type
}) {
3654 if ($arch eq 'aarch64') {
3655 $vga->{type
} = 'virtio';
3656 } elsif (min_version
($machine_version, 2, 9)) {
3657 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3659 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3663 # enable absolute mouse coordinates (needed by vnc)
3664 my $tablet = $conf->{tablet
};
3665 if (!defined($tablet)) {
3666 $tablet = $defaults->{tablet
};
3667 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3668 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3672 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3673 my $kbd = print_keyboarddevice_full
($conf, $arch);
3674 push @$devices, '-device', $kbd if defined($kbd);
3677 my $bootorder = device_bootorder
($conf);
3679 # host pci device passthrough
3680 my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3681 $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3684 my $usb_dev_features = {};
3685 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3687 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3688 $conf, $usb_dev_features, $bootorder, $machine_version);
3689 push @$devices, @usbdevices if @usbdevices;
3692 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3693 my $path = $conf->{"serial$i"} or next;
3694 if ($path eq 'socket') {
3695 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3696 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3697 # On aarch64, serial0 is the UART device. QEMU only allows
3698 # connecting UART devices via the '-serial' command line, as
3699 # the device has a fixed slot on the hardware...
3700 if ($arch eq 'aarch64' && $i == 0) {
3701 push @$devices, '-serial', "chardev:serial$i";
3703 push @$devices, '-device', "isa-serial,chardev=serial$i";
3706 die "no such serial device\n" if ! -c
$path;
3707 push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3708 push @$devices, '-device', "isa-serial,chardev=serial$i";
3713 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3714 if (my $path = $conf->{"parallel$i"}) {
3715 die "no such parallel device\n" if ! -c
$path;
3716 my $devtype = $path =~ m!^/dev/usb/lp! ?
'serial' : 'parallel';
3717 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3718 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3722 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3723 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3724 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3725 push @$devices, @$audio_devs;
3728 # Add a TPM only if the VM is not a template,
3729 # to support backing up template VMs even if the TPM disk is write-protected.
3730 add_tpm_device
($vmid, $devices, $conf) if (!PVE
::QemuConfig-
>is_template($conf));
3733 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3734 $sockets = $conf->{sockets
} if $conf->{sockets
};
3736 my $cores = $conf->{cores
} || 1;
3738 my $maxcpus = $sockets * $cores;
3740 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3742 my $allowed_vcpus = $cpuinfo->{cpus
};
3744 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3746 if ($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3747 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3748 for (my $i = 2; $i <= $vcpus; $i++) {
3749 my $cpustr = print_cpu_device
($conf, $arch, $i);
3750 push @$cmd, '-device', $cpustr;
3755 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3757 push @$cmd, '-nodefaults';
3759 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3761 push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3763 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3765 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3766 push @$devices, '-device', print_vga_device
(
3767 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3769 push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type
} eq 'virtio-gl'; # VIRGL
3771 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3772 push @$cmd, '-vnc', "unix:$socket,password=on";
3774 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3775 push @$cmd, '-nographic';
3779 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3780 my $useLocaltime = $conf->{localtime};
3782 if ($winversion >= 5) { # windows
3783 $useLocaltime = 1 if !defined($conf->{localtime});
3785 # use time drift fix when acpi is enabled
3786 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3787 $tdf = 1 if !defined($conf->{tdf
});
3791 if ($winversion >= 6) {
3792 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3793 push @$machineFlags, 'hpet=off';
3796 push @$rtcFlags, 'driftfix=slew' if $tdf;
3798 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3799 push @$rtcFlags, "base=$conf->{startdate}";
3800 } elsif ($useLocaltime) {
3801 push @$rtcFlags, 'base=localtime';
3805 push @$cmd, '-cpu', $forcecpu;
3807 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3810 PVE
::QemuServer
::Memory
::config
(
3811 $conf, $vmid, $sockets, $cores, $hotplug_features->{memory
}, $cmd);
3813 push @$cmd, '-S' if $conf->{freeze
};
3815 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3817 my $guest_agent = parse_guest_agent
($conf);
3819 if ($guest_agent->{enabled
}) {
3820 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3821 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3823 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3824 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3825 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3826 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3827 } elsif ($guest_agent->{type
} eq 'isa') {
3828 push @$devices, '-device', "isa-serial,chardev=qga0";
3832 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3833 if ($rng && $version_guard->(4, 1, 2)) {
3834 check_rng_source
($rng->{source
});
3836 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3837 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3838 my $limiter_str = "";
3840 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3843 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3844 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3845 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3850 assert_clipboard_config
($vga);
3851 my $is_spice = $qxlnum || $vga->{type
} =~ /^virtio/;
3853 if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3856 for (my $i = 1; $i < $qxlnum; $i++){
3857 push @$devices, '-device', print_vga_device
(
3858 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3861 # assume other OS works like Linux
3862 my ($ram, $vram) = ("134217728", "67108864");
3863 if ($vga->{memory
}) {
3864 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3865 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3867 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3868 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3872 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3874 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3875 if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3876 push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3878 push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3880 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3883 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3884 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3885 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3887 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3888 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3890 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3891 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3892 if ($spice_enhancement->{foldersharing
}) {
3893 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3894 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3897 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3898 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3899 if $spice_enhancement->{videostreaming
};
3900 push @$devices, '-spice', "$spice_opts";
3904 # enable balloon by default, unless explicitly disabled
3905 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3906 my $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3907 my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3908 $ballooncmd .= ",free-page-reporting=on" if min_version
($machine_version, 6, 2);
3909 push @$devices, '-device', $ballooncmd;
3912 if ($conf->{watchdog
}) {
3913 my $wdopts = parse_watchdog
($conf->{watchdog
});
3914 my $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3915 my $watchdog = $wdopts->{model
} || 'i6300esb';
3916 push @$devices, '-device', "$watchdog$pciaddr";
3917 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3921 my $scsicontroller = {};
3922 my $ahcicontroller = {};
3923 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3925 # Add iscsi initiator name if available
3926 if (my $initiator = get_initiator_name
()) {
3927 push @$devices, '-iscsi', "initiator-name=$initiator";
3930 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3931 my ($ds, $drive) = @_;
3933 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3934 check_volume_storage_type
($storecfg, $drive->{file
});
3935 push @$vollist, $drive->{file
};
3938 # ignore efidisk here, already added in bios/fw handling code above
3939 return if $drive->{interface
} eq 'efidisk';
3941 return if $drive->{interface
} eq 'tpmstate';
3943 $use_virtio = 1 if $ds =~ m/^virtio/;
3945 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3947 if ($drive->{interface
} eq 'virtio'){
3948 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3951 if ($drive->{interface
} eq 'scsi') {
3953 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3955 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3956 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3958 my $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3959 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3962 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3963 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3964 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3965 } elsif ($drive->{iothread
}) {
3967 "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3972 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3973 $queues = ",num_queues=$drive->{queues}";
3976 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3977 if !$scsicontroller->{$controller};
3978 $scsicontroller->{$controller}=1;
3981 if ($drive->{interface
} eq 'sata') {
3982 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3983 my $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3984 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3985 if !$ahcicontroller->{$controller};
3986 $ahcicontroller->{$controller}=1;
3989 my $live_restore = $live_restore_backing->{$ds};
3990 my $live_blockdev_name = undef;
3991 if ($live_restore) {
3992 $live_blockdev_name = $live_restore->{name
};
3993 push @$devices, '-blockdev', $live_restore->{blockdev
};
3996 my $drive_cmd = print_drive_commandline_full
(
3997 $storecfg, $vmid, $drive, $live_blockdev_name, min_version
($kvmver, 6, 0));
3999 # extra protection for templates, but SATA and IDE don't support it..
4000 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
4002 push @$devices, '-drive',$drive_cmd;
4003 push @$devices, '-device', print_drivedevice_full
(
4004 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4007 for (my $i = 0; $i < $MAX_NETS; $i++) {
4008 my $netname = "net$i";
4010 next if !$conf->{$netname};
4011 my $d = parse_net
($conf->{$netname});
4013 # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4015 $use_virtio = 1 if $d->{model
} eq 'virtio';
4017 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
4019 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
4020 push @$devices, '-netdev', $netdevfull;
4022 my $netdevicefull = print_netdevice_full
(
4023 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4025 push @$devices, '-device', $netdevicefull;
4028 if ($conf->{ivshmem
}) {
4029 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4033 $bus = print_pcie_addr
("ivshmem");
4035 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4038 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4039 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4041 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4042 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4043 .",size=$ivshmem->{size}M";
4046 # pci.4 is nested in pci.1
4047 $bridges->{1} = 1 if $bridges->{4};
4049 if (!$q35) { # add pci bridges
4050 if (min_version
($machine_version, 2, 3)) {
4054 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4057 for my $k (sort {$b cmp $a} keys %$bridges) {
4058 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4061 if ($k == 2 && $legacy_igd) {
4064 my $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
4065 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4067 if ($q35) { # add after -readconfig pve-q35.cfg
4068 splice @$devices, 2, 0, '-device', $devstr;
4070 unshift @$devices, '-device', $devstr if $k > 0;
4075 push @$machineFlags, 'accel=tcg';
4078 push @$machineFlags, 'smm=off' if should_disable_smm
($conf, $vga, $machine_type);
4080 my $machine_type_min = $machine_type;
4081 if ($add_pve_version) {
4082 $machine_type_min =~ s/\+pve\d+$//;
4083 $machine_type_min .= "+pve$required_pve_version";
4085 push @$machineFlags, "type=${machine_type_min}";
4087 push @$cmd, @$devices;
4088 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4089 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4090 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4092 if (my $vmstate = $conf->{vmstate
}) {
4093 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4094 push @$vollist, $vmstate;
4095 push @$cmd, '-loadstate', $statepath;
4096 print "activating and using '$vmstate' as vmstate\n";
4099 if (PVE
::QemuConfig-
>is_template($conf)) {
4100 # needed to workaround base volumes being read-only
4101 push @$cmd, '-snapshot';
4105 if ($conf->{args
}) {
4106 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4110 return wantarray ?
($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4113 sub check_rng_source
{
4116 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4117 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4120 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4121 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
4122 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4123 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4124 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4125 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4133 my $res = mon_cmd
($vmid, 'query-spice');
4135 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4138 sub vm_devices_list
{
4141 my $res = mon_cmd
($vmid, 'query-pci');
4142 my $devices_to_check = [];
4144 foreach my $pcibus (@$res) {
4145 push @$devices_to_check, @{$pcibus->{devices
}},
4148 while (@$devices_to_check) {
4150 for my $d (@$devices_to_check) {
4151 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4152 next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices
};
4154 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4155 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4157 $devices_to_check = $to_check;
4160 my $resblock = mon_cmd
($vmid, 'query-block');
4161 foreach my $block (@$resblock) {
4162 if($block->{device
} =~ m/^drive-(\S+)/){
4167 my $resmice = mon_cmd
($vmid, 'query-mice');
4168 foreach my $mice (@$resmice) {
4169 if ($mice->{name
} eq 'QEMU HID Tablet') {
4170 $devices->{tablet
} = 1;
4175 # for usb devices there is no query-usb
4176 # but we can iterate over the entries in
4177 # qom-list path=/machine/peripheral
4178 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4179 foreach my $per (@$resperipheral) {
4180 if ($per->{name
} =~ m/^usb(?:redirdev)?\d+$/) {
4181 $devices->{$per->{name
}} = 1;
4189 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4191 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4193 my $devices_list = vm_devices_list
($vmid);
4194 return 1 if defined($devices_list->{$deviceid});
4196 # add PCI bridge if we need it for the device
4197 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4199 if ($deviceid eq 'tablet') {
4200 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4201 } elsif ($deviceid eq 'keyboard') {
4202 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4203 } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4205 qemu_spice_usbredir_chardev_add
($vmid, "usbredirchardev$id");
4206 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_spice_usbdevice
($id, "xhci", $id + 1));
4207 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4208 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device, {}, $1 + 1));
4209 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4210 qemu_iothread_add
($vmid, $deviceid, $device);
4212 qemu_driveadd
($storecfg, $vmid, $device);
4213 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4215 qemu_deviceadd
($vmid, $devicefull);
4216 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4218 eval { qemu_drivedel
($vmid, $deviceid); };
4222 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4223 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4224 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4225 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4227 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4229 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4230 qemu_iothread_add
($vmid, $deviceid, $device);
4231 $devicefull .= ",iothread=iothread-$deviceid";
4234 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4235 $devicefull .= ",num_queues=$device->{queues}";
4238 qemu_deviceadd
($vmid, $devicefull);
4239 qemu_deviceaddverify
($vmid, $deviceid);
4240 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4241 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4242 qemu_driveadd
($storecfg, $vmid, $device);
4244 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4245 eval { qemu_deviceadd
($vmid, $devicefull); };
4247 eval { qemu_drivedel
($vmid, $deviceid); };
4251 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4252 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4254 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4255 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type);
4256 my $use_old_bios_files = undef;
4257 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4259 my $netdevicefull = print_netdevice_full
(
4260 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4261 qemu_deviceadd
($vmid, $netdevicefull);
4263 qemu_deviceaddverify
($vmid, $deviceid);
4264 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4267 eval { qemu_netdevdel
($vmid, $deviceid); };
4271 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4273 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4274 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4276 qemu_deviceadd
($vmid, $devicefull);
4277 qemu_deviceaddverify
($vmid, $deviceid);
4279 die "can't hotplug device '$deviceid'\n";
4285 # fixme: this should raise exceptions on error!
4286 sub vm_deviceunplug
{
4287 my ($vmid, $conf, $deviceid) = @_;
4289 my $devices_list = vm_devices_list
($vmid);
4290 return 1 if !defined($devices_list->{$deviceid});
4292 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4293 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4295 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4296 qemu_devicedel
($vmid, $deviceid);
4297 } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4298 qemu_devicedel
($vmid, $deviceid);
4299 qemu_devicedelverify
($vmid, $deviceid);
4300 } elsif ($deviceid =~ m/^usb\d+$/) {
4301 qemu_devicedel
($vmid, $deviceid);
4302 qemu_devicedelverify
($vmid, $deviceid);
4303 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4304 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4306 qemu_devicedel
($vmid, $deviceid);
4307 qemu_devicedelverify
($vmid, $deviceid);
4308 qemu_drivedel
($vmid, $deviceid);
4309 qemu_iothread_del
($vmid, $deviceid, $device);
4310 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4311 qemu_devicedel
($vmid, $deviceid);
4312 qemu_devicedelverify
($vmid, $deviceid);
4313 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4314 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4316 qemu_devicedel
($vmid, $deviceid);
4317 qemu_devicedelverify
($vmid, $deviceid);
4318 qemu_drivedel
($vmid, $deviceid);
4319 qemu_deletescsihw
($conf, $vmid, $deviceid);
4321 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4322 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4323 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4324 qemu_devicedel
($vmid, $deviceid);
4325 qemu_devicedelverify
($vmid, $deviceid);
4326 qemu_netdevdel
($vmid, $deviceid);
4328 die "can't unplug device '$deviceid'\n";
4334 sub qemu_spice_usbredir_chardev_add
{
4335 my ($vmid, $id) = @_;
4337 mon_cmd
($vmid, "chardev-add" , (
4348 sub qemu_iothread_add
{
4349 my ($vmid, $deviceid, $device) = @_;
4351 if ($device->{iothread
}) {
4352 my $iothreads = vm_iothreads_list
($vmid);
4353 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4357 sub qemu_iothread_del
{
4358 my ($vmid, $deviceid, $device) = @_;
4360 if ($device->{iothread
}) {
4361 my $iothreads = vm_iothreads_list
($vmid);
4362 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4367 my ($storecfg, $vmid, $device) = @_;
4369 my $kvmver = get_running_qemu_version
($vmid);
4370 my $io_uring = min_version
($kvmver, 6, 0);
4371 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4372 $drive =~ s/\\/\\\\/g;
4373 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4375 # If the command succeeds qemu prints: "OK
"
4376 return 1 if $ret =~ m/OK/s;
4378 die "adding drive failed
: $ret\n";
4382 my ($vmid, $deviceid) = @_;
4384 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4387 return 1 if $ret eq "";
4389 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4390 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4392 die "deleting drive
$deviceid failed
: $ret\n";
4395 sub qemu_deviceaddverify {
4396 my ($vmid, $deviceid) = @_;
4398 for (my $i = 0; $i <= 5; $i++) {
4399 my $devices_list = vm_devices_list($vmid);
4400 return 1 if defined($devices_list->{$deviceid});
4404 die "error on hotplug device
'$deviceid'\n";
4408 sub qemu_devicedelverify {
4409 my ($vmid, $deviceid) = @_;
4411 # need to verify that the device is correctly removed as device_del
4412 # is async and empty return is not reliable
4414 for (my $i = 0; $i <= 5; $i++) {
4415 my $devices_list = vm_devices_list($vmid);
4416 return 1 if !defined($devices_list->{$deviceid});
4420 die "error on hot-unplugging device
'$deviceid'\n";
4423 sub qemu_findorcreatescsihw {
4424 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4426 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4428 my $scsihwid="$controller_prefix$controller";
4429 my $devices_list = vm_devices_list($vmid);
4431 if (!defined($devices_list->{$scsihwid})) {
4432 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4438 sub qemu_deletescsihw {
4439 my ($conf, $vmid, $opt) = @_;
4441 my $device = parse_drive($opt, $conf->{$opt});
4443 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4444 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4448 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4450 my $devices_list = vm_devices_list($vmid);
4451 foreach my $opt (keys %{$devices_list}) {
4452 if (is_valid_drivename($opt)) {
4453 my $drive = parse_drive($opt, $conf->{$opt});
4454 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4460 my $scsihwid="scsihw
$controller";
4462 vm_deviceunplug($vmid, $conf, $scsihwid);
4467 sub qemu_add_pci_bridge {
4468 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4474 print_pci_addr($device, $bridges, $arch, $machine_type);
4476 while (my ($k, $v) = each %$bridges) {
4479 return 1 if !defined($bridgeid) || $bridgeid < 1;
4481 my $bridge = "pci
.$bridgeid";
4482 my $devices_list = vm_devices_list($vmid);
4484 if (!defined($devices_list->{$bridge})) {
4485 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4491 sub qemu_set_link_status {
4492 my ($vmid, $device, $up) = @_;
4494 mon_cmd($vmid, "set_link
", name => $device,
4495 up => $up ? JSON::true : JSON::false);
4498 sub qemu_netdevadd {
4499 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4501 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4502 my %options = split(/[=,]/, $netdev);
4504 if (defined(my $vhost = $options{vhost})) {
4505 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4508 if (defined(my $queues = $options{queues})) {
4509 $options{queues} = $queues + 0;
4512 mon_cmd($vmid, "netdev_add
", %options);
4516 sub qemu_netdevdel {
4517 my ($vmid, $deviceid) = @_;
4519 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4522 sub qemu_usb_hotplug {
4523 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4527 # remove the old one first
4528 vm_deviceunplug($vmid, $conf, $deviceid);
4530 # check if xhci controller is necessary and available
4531 my $devicelist = vm_devices_list($vmid);
4533 if (!$devicelist->{xhci}) {
4534 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4535 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4539 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4542 sub qemu_cpu_hotplug {
4543 my ($vmid, $conf, $vcpus) = @_;
4545 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4548 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4549 $sockets = $conf->{sockets} if $conf->{sockets};
4550 my $cores = $conf->{cores} || 1;
4551 my $maxcpus = $sockets * $cores;
4553 $vcpus = $maxcpus if !$vcpus;
4555 die "you can
't add more vcpus than maxcpus\n"
4556 if $vcpus > $maxcpus;
4558 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4560 if ($vcpus < $currentvcpus) {
4562 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4564 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4565 qemu_devicedel($vmid, "cpu$i");
4567 my $currentrunningvcpus = undef;
4569 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4570 last if scalar(@{$currentrunningvcpus}) == $i-1;
4571 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4575 #update conf after each succesfull cpu unplug
4576 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4577 PVE::QemuConfig->write_config($vmid, $conf);
4580 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4586 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4587 die "vcpus in running vm does not match its configuration\n"
4588 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4590 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4591 my $arch = get_vm_arch($conf);
4593 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4594 my $cpustr = print_cpu_device($conf, $arch, $i);
4595 qemu_deviceadd($vmid, $cpustr);
4598 my $currentrunningvcpus = undef;
4600 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4601 last if scalar(@{$currentrunningvcpus}) == $i;
4602 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4606 #update conf after each succesfull cpu hotplug
4607 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4608 PVE::QemuConfig->write_config($vmid, $conf);
4612 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4613 mon_cmd($vmid, "cpu-add", id => int($i));
4618 sub qemu_block_set_io_throttle {
4619 my ($vmid, $deviceid,
4620 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4621 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4622 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4623 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4625 return if !check_running($vmid) ;
4627 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4629 bps_rd => int($bps_rd),
4630 bps_wr => int($bps_wr),
4632 iops_rd => int($iops_rd),
4633 iops_wr => int($iops_wr),
4634 bps_max => int($bps_max),
4635 bps_rd_max => int($bps_rd_max),
4636 bps_wr_max => int($bps_wr_max),
4637 iops_max => int($iops_max),
4638 iops_rd_max => int($iops_rd_max),
4639 iops_wr_max => int($iops_wr_max),
4640 bps_max_length => int($bps_max_length),
4641 bps_rd_max_length => int($bps_rd_max_length),
4642 bps_wr_max_length => int($bps_wr_max_length),
4643 iops_max_length => int($iops_max_length),
4644 iops_rd_max_length => int($iops_rd_max_length),
4645 iops_wr_max_length => int($iops_wr_max_length),
4650 sub qemu_block_resize {
4651 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4653 my $running = check_running($vmid);
4655 PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4657 return if !$running;
4659 my $padding = (1024 - $size % 1024) % 1024;
4660 $size = $size + $padding;
4665 device => $deviceid,
4671 sub qemu_volume_snapshot {
4672 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4674 my $running = check_running($vmid);
4676 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4677 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4679 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4683 sub qemu_volume_snapshot_delete {
4684 my ($vmid, $storecfg, $volid, $snap) = @_;
4686 my $running = check_running($vmid);
4687 my $attached_deviceid;
4690 my $conf = PVE::QemuConfig->load_config($vmid);
4691 PVE::QemuConfig->foreach_volume($conf, sub {
4692 my ($ds, $drive) = @_;
4693 $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
4697 if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
4700 'blockdev-snapshot-delete-internal-sync
',
4701 device => $attached_deviceid,
4705 PVE::Storage::volume_snapshot_delete(
4706 $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
4710 sub set_migration_caps {
4711 my ($vmid, $savevm) = @_;
4713 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4715 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4716 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4721 "auto-converge" => 1,
4723 "x-rdma-pin-all" => 0,
4726 "dirty-bitmaps" => $dirty_bitmaps,
4729 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4731 for my $supported_capability (@$supported_capabilities) {
4733 capability => $supported_capability->{capability},
4734 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4738 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4742 my ($conf, $func, @param) = @_;
4746 my $test_volid = sub {
4747 my ($key, $drive, $snapname, $pending) = @_;
4749 my $volid = $drive->{file};
4752 $volhash->{$volid}->{cdrom} //= 1;
4753 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4755 my $replicate = $drive->{replicate} // 1;
4756 $volhash->{$volid}->{replicate} //= 0;
4757 $volhash->{$volid}->{replicate} = 1 if $replicate;
4759 $volhash->{$volid}->{shared} //= 0;
4760 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4762 $volhash->{$volid}->{is_unused} //= 0;
4763 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4765 $volhash->{$volid}->{is_attached} //= 0;
4766 $volhash->{$volid}->{is_attached} = 1
4767 if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4769 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4770 if defined($snapname);
4772 $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4774 my $size = $drive->{size};
4775 $volhash->{$volid}->{size} //= $size if $size;
4777 $volhash->{$volid}->{is_vmstate} //= 0;
4778 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4780 $volhash->{$volid}->{is_tpmstate} //= 0;
4781 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0
';
4783 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4786 my $include_opts = {
4787 extra_keys => ['vmstate
'],
4788 include_unused => 1,
4791 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4793 PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4794 if defined($conf->{pending}) && $conf->{pending}->%*;
4796 foreach my $snapname (keys %{$conf->{snapshots}}) {
4797 my $snap = $conf->{snapshots}->{$snapname};
4798 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4801 foreach my $volid (keys %$volhash) {
4802 &$func($volid, $volhash->{$volid}, @param);
4806 my $fast_plug_option = {
4810 'migrate_downtime
' => 1,
4811 'migrate_speed
' => 1,
4818 'vmstatestorage
' => 1,
4821 for my $opt (keys %$confdesc_cloudinit) {
4822 $fast_plug_option->{$opt} = 1;
4825 # hotplug changes in [PENDING]
4826 # $selection hash can be used to only apply specified options, for
4827 # example: { cores => 1 } (only apply changed 'cores
')
4828 # $errors ref is used to return error messages
4829 sub vmconfig_hotplug_pending {
4830 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4832 my $defaults = load_defaults();
4833 my $arch = get_vm_arch($conf);
4834 my $machine_type = get_vm_machine($conf, undef, $arch);
4836 # commit values which do not have any impact on running VM first
4837 # Note: those option cannot raise errors, we we do not care about
4838 # $selection and always apply them.
4840 my $add_error = sub {
4841 my ($opt, $msg) = @_;
4842 $errors->{$opt} = "hotplug problem - $msg";
4845 my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4847 my $cloudinit_record_changed = sub {
4848 my ($conf, $opt, $old, $new) = @_;
4849 return if !$cloudinit_pending_properties->{$opt};
4851 my $ci = ($conf->{cloudinit} //= {});
4853 my $recorded = $ci->{$opt};
4854 my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4856 if (defined($new)) {
4857 if (defined($old)) {
4858 # an existing value is being modified
4859 if (defined($recorded)) {
4860 # the value was already not in sync
4861 if ($new eq $recorded) {
4862 # a value is being reverted to the cloud-init state:
4864 delete $added{$opt};
4866 # the value was changed multiple times, do nothing
4868 } elsif ($added{$opt}) {
4869 # the value had been marked as added and is being changed, do nothing
4871 # the value is new, record it:
4875 # a new value is being added
4876 if (defined($recorded)) {
4877 # it was already not in sync
4878 if ($new eq $recorded) {
4879 # a value is being reverted to the cloud-init state:
4881 delete $added{$opt};
4883 # the value had temporarily been removed, do nothing
4885 } elsif ($added{$opt}) {
4886 # the value had been marked as added already, do nothing
4888 # the value is new, add it
4892 } elsif (!defined($old)) {
4893 # a non-existent value is being removed? ignore...
4895 # a value is being deleted
4896 if (defined($recorded)) {
4897 # a value was already recorded, just keep it
4898 } elsif ($added{$opt}) {
4899 # the value was marked as added, remove it
4900 delete $added{$opt};
4902 # a previously unrecorded value is being removed, record the old value:
4907 my $added = join(',', sort keys %added);
4908 $ci->{added} = $added if length($added);
4912 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4913 if ($fast_plug_option->{$opt}) {
4914 my $new = delete $conf->{pending}->{$opt};
4915 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4916 $conf->{$opt} = $new;
4922 PVE::QemuConfig->write_config($vmid, $conf);
4925 my $ostype = $conf->{ostype};
4926 my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4927 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4928 my $usb_hotplug = $hotplug_features->{usb}
4929 && min_version($version, 7, 1)
4930 && defined($ostype) && ($ostype eq 'l26
' || windows_version($ostype) > 7);
4932 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4933 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4935 foreach my $opt (sort keys %$pending_delete_hash) {
4936 next if $selection && !$selection->{$opt};
4937 my $force = $pending_delete_hash->{$opt}->{force};
4939 if ($opt eq 'hotplug
') {
4940 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4941 } elsif ($opt eq 'tablet
') {
4942 die "skip\n" if !$hotplug_features->{usb};
4943 if ($defaults->{tablet}) {
4944 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4945 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4946 if $arch eq 'aarch64
';
4948 vm_deviceunplug($vmid, $conf, 'tablet
');
4949 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4951 } elsif ($opt =~ m/^usb(\d+)$/) {
4953 die "skip\n" if !$usb_hotplug;
4954 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4955 vm_deviceunplug
($vmid, $conf, $opt);
4956 } elsif ($opt eq 'vcpus') {
4957 die "skip\n" if !$hotplug_features->{cpu
};
4958 qemu_cpu_hotplug
($vmid, $conf, undef);
4959 } elsif ($opt eq 'balloon') {
4960 # enable balloon device is not hotpluggable
4961 die "skip\n" if defined($conf->{balloon
}) && $conf->{balloon
} == 0;
4962 # here we reset the ballooning value to memory
4963 my $balloon = get_current_memory
($conf->{memory
});
4964 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
4965 } elsif ($fast_plug_option->{$opt}) {
4967 } elsif ($opt =~ m/^net(\d+)$/) {
4968 die "skip\n" if !$hotplug_features->{network
};
4969 vm_deviceunplug
($vmid, $conf, $opt);
4971 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
4972 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
});
4974 } elsif (is_valid_drivename
($opt)) {
4975 die "skip\n" if !$hotplug_features->{disk
} || $opt =~ m/(ide|sata)(\d+)/;
4976 vm_deviceunplug
($vmid, $conf, $opt);
4977 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
4978 } elsif ($opt =~ m/^memory$/) {
4979 die "skip\n" if !$hotplug_features->{memory
};
4980 PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf);
4981 } elsif ($opt eq 'cpuunits') {
4982 $cgroup->change_cpu_shares(undef);
4983 } elsif ($opt eq 'cpulimit') {
4984 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4990 &$add_error($opt, $err) if $err ne "skip\n";
4992 my $old = delete $conf->{$opt};
4993 $cloudinit_record_changed->($conf, $opt, $old, undef);
4994 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
4999 foreach my $opt (keys %{$conf->{pending
}}) {
5000 next if $selection && !$selection->{$opt};
5001 my $value = $conf->{pending
}->{$opt};
5003 if ($opt eq 'hotplug') {
5004 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug
} =~ /memory/);
5005 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug
} =~ /cpu/);
5006 } elsif ($opt eq 'tablet') {
5007 die "skip\n" if !$hotplug_features->{usb
};
5009 vm_deviceplug
($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5010 vm_deviceplug
($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5011 if $arch eq 'aarch64';
5012 } elsif ($value == 0) {
5013 vm_deviceunplug
($vmid, $conf, 'tablet');
5014 vm_deviceunplug
($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5016 } elsif ($opt =~ m/^usb(\d+)$/) {
5018 die "skip\n" if !$usb_hotplug;
5019 my $d = eval { parse_property_string
('pve-qm-usb', $value) };
5021 if ($d->{host
} =~ m/^spice$/i) {
5022 $id = "usbredirdev$index";
5024 qemu_usb_hotplug
($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5025 } elsif ($opt eq 'vcpus') {
5026 die "skip\n" if !$hotplug_features->{cpu
};
5027 qemu_cpu_hotplug
($vmid, $conf, $value);
5028 } elsif ($opt eq 'balloon') {
5029 # enable/disable balloning device is not hotpluggable
5030 my $old_balloon_enabled = !!(!defined($conf->{balloon
}) || $conf->{balloon
});
5031 my $new_balloon_enabled = !!(!defined($conf->{pending
}->{balloon
}) || $conf->{pending
}->{balloon
});
5032 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5034 # allow manual ballooning if shares is set to zero
5035 if ((defined($conf->{shares
}) && ($conf->{shares
} == 0))) {
5036 my $memory = get_current_memory
($conf->{memory
});
5037 my $balloon = $conf->{pending
}->{balloon
} || $memory;
5038 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5040 } elsif ($opt =~ m/^net(\d+)$/) {
5041 # some changes can be done without hotplug
5042 vmconfig_update_net
($storecfg, $conf, $hotplug_features->{network
},
5043 $vmid, $opt, $value, $arch, $machine_type);
5044 } elsif (is_valid_drivename
($opt)) {
5045 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5046 # some changes can be done without hotplug
5047 my $drive = parse_drive
($opt, $value);
5048 if (drive_is_cloudinit
($drive)) {
5049 $cloudinit_opt = [$opt, $drive];
5050 # apply all the other changes first, then generate the cloudinit disk
5053 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5054 $vmid, $opt, $value, $arch, $machine_type);
5055 } elsif ($opt =~ m/^memory$/) { #dimms
5056 die "skip\n" if !$hotplug_features->{memory
};
5057 $value = PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $value);
5058 } elsif ($opt eq 'cpuunits') {
5059 my $new_cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{pending
}->{$opt}); #clamp
5060 $cgroup->change_cpu_shares($new_cpuunits);
5061 } elsif ($opt eq 'cpulimit') {
5062 my $cpulimit = $conf->{pending
}->{$opt} == 0 ?
-1 : int($conf->{pending
}->{$opt} * 100000);
5063 $cgroup->change_cpu_quota($cpulimit, 100000);
5064 } elsif ($opt eq 'agent') {
5065 vmconfig_update_agent
($conf, $opt, $value);
5067 die "skip\n"; # skip non-hot-pluggable options
5071 &$add_error($opt, $err) if $err ne "skip\n";
5073 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5074 $conf->{$opt} = $value;
5075 delete $conf->{pending
}->{$opt};
5079 if (defined($cloudinit_opt)) {
5080 my ($opt, $drive) = @$cloudinit_opt;
5081 my $value = $conf->{pending
}->{$opt};
5083 my $temp = {%$conf, $opt => $value};
5084 PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($temp, $vmid);
5085 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5086 $vmid, $opt, $value, $arch, $machine_type);
5089 &$add_error($opt, $err) if $err ne "skip\n";
5091 $conf->{$opt} = $value;
5092 delete $conf->{pending
}->{$opt};
5096 # unplug xhci controller if no usb device is left
5099 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
5100 next if !defined($conf->{"usb$i"});
5105 vm_deviceunplug
($vmid, $conf, 'xhci');
5109 PVE
::QemuConfig-
>write_config($vmid, $conf);
5111 if ($hotplug_features->{cloudinit
} && PVE
::QemuServer
::Cloudinit
::has_changes
($conf)) {
5112 PVE
::QemuServer
::vmconfig_update_cloudinit_drive
($storecfg, $conf, $vmid);
5116 sub try_deallocate_drive
{
5117 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5119 if (($force || $key =~ /^unused/) && !drive_is_cdrom
($drive, 1)) {
5120 my $volid = $drive->{file
};
5121 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
5122 my $sid = PVE
::Storage
::parse_volume_id
($volid);
5123 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5125 # check if the disk is really unused
5126 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5127 if PVE
::QemuServer
::Drive
::is_volume_in_use
($storecfg, $conf, $key, $volid);
5128 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5131 # If vm is not owner of this disk remove from config
5139 sub vmconfig_delete_or_detach_drive
{
5140 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5142 my $drive = parse_drive
($opt, $conf->{$opt});
5144 my $rpcenv = PVE
::RPCEnvironment
::get
();
5145 my $authuser = $rpcenv->get_user();
5148 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5149 try_deallocate_drive
($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5151 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $drive);
5157 sub vmconfig_apply_pending
{
5158 my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5160 return if !scalar(keys %{$conf->{pending
}});
5162 my $add_apply_error = sub {
5163 my ($opt, $msg) = @_;
5164 my $err_msg = "unable to apply pending change $opt : $msg";
5165 $errors->{$opt} = $err_msg;
5171 my $pending_delete_hash = PVE
::QemuConfig-
>parse_pending_delete($conf->{pending
}->{delete});
5172 foreach my $opt (sort keys %$pending_delete_hash) {
5173 my $force = $pending_delete_hash->{$opt}->{force
};
5175 if ($opt =~ m/^unused/) {
5176 die "internal error";
5177 } elsif (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5178 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5179 } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5181 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
5182 eval { PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
}) };
5188 $add_apply_error->($opt, $err);
5190 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5191 delete $conf->{$opt};
5195 PVE
::QemuConfig-
>cleanup_pending($conf);
5197 my $generate_cloudinit = $skip_cloud_init ?
0 : undef;
5199 foreach my $opt (keys %{$conf->{pending
}}) { # add/change
5200 next if $opt eq 'delete'; # just to be sure
5202 if (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5203 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, parse_drive
($opt, $conf->{$opt}))
5204 } elsif (defined($conf->{pending
}->{$opt}) && $opt =~ m/^net\d+$/) {
5205 return if !$have_sdn; # return from eval if SDN is not available
5207 my $new_net = PVE
::QemuServer
::parse_net
($conf->{pending
}->{$opt});
5208 if ($conf->{$opt}) {
5209 my $old_net = PVE
::QemuServer
::parse_net
($conf->{$opt});
5211 if (defined($old_net->{bridge
}) && defined($old_net->{macaddr
}) && (
5212 safe_string_ne
($old_net->{bridge
}, $new_net->{bridge
}) ||
5213 safe_string_ne
($old_net->{macaddr
}, $new_net->{macaddr
})
5215 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($old_net->{bridge
}, $old_net->{macaddr
}, $conf->{name
});
5218 #fixme: reuse ip if mac change && same bridge
5219 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($new_net->{bridge
}, $conf->{name
}, $new_net->{macaddr
}, $vmid, undef, 1);
5223 $add_apply_error->($opt, $err);
5226 if (is_valid_drivename
($opt)) {
5227 my $drive = parse_drive
($opt, $conf->{pending
}->{$opt});
5228 $generate_cloudinit //= 1 if drive_is_cloudinit
($drive);
5231 $conf->{$opt} = delete $conf->{pending
}->{$opt};
5235 # write all changes at once to avoid unnecessary i/o
5236 PVE
::QemuConfig-
>write_config($vmid, $conf);
5237 if ($generate_cloudinit) {
5238 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5239 # After successful generation and if there were changes to be applied, update the
5240 # config to drop the {cloudinit} entry.
5241 PVE
::QemuConfig-
>write_config($vmid, $conf);
5246 sub vmconfig_update_net
{
5247 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5249 my $newnet = parse_net
($value);
5251 if ($conf->{$opt}) {
5252 my $oldnet = parse_net
($conf->{$opt});
5254 if (safe_string_ne
($oldnet->{model
}, $newnet->{model
}) ||
5255 safe_string_ne
($oldnet->{macaddr
}, $newnet->{macaddr
}) ||
5256 safe_num_ne
($oldnet->{queues
}, $newnet->{queues
}) ||
5257 safe_num_ne
($oldnet->{mtu
}, $newnet->{mtu
}) ||
5258 !($newnet->{bridge
} && $oldnet->{bridge
})
5259 ) { # bridge/nat mode change
5261 # for non online change, we try to hot-unplug
5262 die "skip\n" if !$hotplug;
5263 vm_deviceunplug
($vmid, $conf, $opt);
5266 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($oldnet->{bridge
}, $oldnet->{macaddr
}, $conf->{name
});
5271 die "internal error" if $opt !~ m/net(\d+)/;
5272 my $iface = "tap${vmid}i$1";
5274 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5275 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
}) ||
5276 safe_string_ne
($oldnet->{trunks
}, $newnet->{trunks
}) ||
5277 safe_num_ne
($oldnet->{firewall
}, $newnet->{firewall
})
5279 PVE
::Network
::tap_unplug
($iface);
5281 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5282 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5283 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
})
5285 qemu_set_link_status
($vmid, $opt, 0);
5288 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
})) {
5290 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($oldnet->{bridge
}, $oldnet->{macaddr
}, $conf->{name
});
5291 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($newnet->{bridge
}, $conf->{name
}, $newnet->{macaddr
}, $vmid, undef, 1);
5296 PVE
::Network
::SDN
::Zones
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5298 PVE
::Network
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5301 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5302 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5303 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
})
5305 qemu_set_link_status
($vmid, $opt, 1);
5308 } elsif (safe_num_ne
($oldnet->{rate
}, $newnet->{rate
})) {
5309 # Rate can be applied on its own but any change above needs to
5310 # include the rate in tap_plug since OVS resets everything.
5311 PVE
::Network
::tap_rate_limit
($iface, $newnet->{rate
});
5314 if (safe_string_ne
($oldnet->{link_down
}, $newnet->{link_down
})) {
5315 qemu_set_link_status
($vmid, $opt, !$newnet->{link_down
});
5324 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($newnet->{bridge
}, $conf->{name
}, $newnet->{macaddr
}, $vmid, undef, 1);
5325 PVE
::Network
::SDN
::Vnets
::add_dhcp_mapping
($newnet->{bridge
}, $newnet->{macaddr
}, $vmid, $conf->{name
});
5327 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5333 sub vmconfig_update_agent
{
5334 my ($conf, $opt, $value) = @_;
5336 die "skip\n" if !$conf->{$opt};
5338 my $hotplug_options = { fstrim_cloned_disks
=> 1 };
5340 my $old_agent = parse_guest_agent
($conf);
5341 my $agent = parse_guest_agent
({$opt => $value});
5343 for my $option (keys %$agent) { # added/changed options
5344 next if defined($hotplug_options->{$option});
5345 die "skip\n" if safe_string_ne
($agent->{$option}, $old_agent->{$option});
5348 for my $option (keys %$old_agent) { # removed options
5349 next if defined($hotplug_options->{$option});
5350 die "skip\n" if safe_string_ne
($old_agent->{$option}, $agent->{$option});
5353 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5356 sub vmconfig_update_disk
{
5357 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5359 my $drive = parse_drive
($opt, $value);
5361 if ($conf->{$opt} && (my $old_drive = parse_drive
($opt, $conf->{$opt}))) {
5362 my $media = $drive->{media
} || 'disk';
5363 my $oldmedia = $old_drive->{media
} || 'disk';
5364 die "unable to change media type\n" if $media ne $oldmedia;
5366 if (!drive_is_cdrom
($old_drive)) {
5368 if ($drive->{file
} ne $old_drive->{file
}) {
5370 die "skip\n" if !$hotplug;
5372 # unplug and register as unused
5373 vm_deviceunplug
($vmid, $conf, $opt);
5374 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive)
5377 # update existing disk
5379 # skip non hotpluggable value
5380 if (safe_string_ne
($drive->{aio
}, $old_drive->{aio
}) ||
5381 safe_string_ne
($drive->{discard
}, $old_drive->{discard
}) ||
5382 safe_string_ne
($drive->{iothread
}, $old_drive->{iothread
}) ||
5383 safe_string_ne
($drive->{queues
}, $old_drive->{queues
}) ||
5384 safe_string_ne
($drive->{product
}, $old_drive->{product
}) ||
5385 safe_string_ne
($drive->{cache
}, $old_drive->{cache
}) ||
5386 safe_string_ne
($drive->{ssd
}, $old_drive->{ssd
}) ||
5387 safe_string_ne
($drive->{vendor
}, $old_drive->{vendor
}) ||
5388 safe_string_ne
($drive->{ro
}, $old_drive->{ro
})) {
5393 if (safe_num_ne
($drive->{mbps
}, $old_drive->{mbps
}) ||
5394 safe_num_ne
($drive->{mbps_rd
}, $old_drive->{mbps_rd
}) ||
5395 safe_num_ne
($drive->{mbps_wr
}, $old_drive->{mbps_wr
}) ||
5396 safe_num_ne
($drive->{iops
}, $old_drive->{iops
}) ||
5397 safe_num_ne
($drive->{iops_rd
}, $old_drive->{iops_rd
}) ||
5398 safe_num_ne
($drive->{iops_wr
}, $old_drive->{iops_wr
}) ||
5399 safe_num_ne
($drive->{mbps_max
}, $old_drive->{mbps_max
}) ||
5400 safe_num_ne
($drive->{mbps_rd_max
}, $old_drive->{mbps_rd_max
}) ||
5401 safe_num_ne
($drive->{mbps_wr_max
}, $old_drive->{mbps_wr_max
}) ||
5402 safe_num_ne
($drive->{iops_max
}, $old_drive->{iops_max
}) ||
5403 safe_num_ne
($drive->{iops_rd_max
}, $old_drive->{iops_rd_max
}) ||
5404 safe_num_ne
($drive->{iops_wr_max
}, $old_drive->{iops_wr_max
}) ||
5405 safe_num_ne
($drive->{bps_max_length
}, $old_drive->{bps_max_length
}) ||
5406 safe_num_ne
($drive->{bps_rd_max_length
}, $old_drive->{bps_rd_max_length
}) ||
5407 safe_num_ne
($drive->{bps_wr_max_length
}, $old_drive->{bps_wr_max_length
}) ||
5408 safe_num_ne
($drive->{iops_max_length
}, $old_drive->{iops_max_length
}) ||
5409 safe_num_ne
($drive->{iops_rd_max_length
}, $old_drive->{iops_rd_max_length
}) ||
5410 safe_num_ne
($drive->{iops_wr_max_length
}, $old_drive->{iops_wr_max_length
})) {
5412 qemu_block_set_io_throttle
(
5414 ($drive->{mbps
} || 0)*1024*1024,
5415 ($drive->{mbps_rd
} || 0)*1024*1024,
5416 ($drive->{mbps_wr
} || 0)*1024*1024,
5417 $drive->{iops
} || 0,
5418 $drive->{iops_rd
} || 0,
5419 $drive->{iops_wr
} || 0,
5420 ($drive->{mbps_max
} || 0)*1024*1024,
5421 ($drive->{mbps_rd_max
} || 0)*1024*1024,
5422 ($drive->{mbps_wr_max
} || 0)*1024*1024,
5423 $drive->{iops_max
} || 0,
5424 $drive->{iops_rd_max
} || 0,
5425 $drive->{iops_wr_max
} || 0,
5426 $drive->{bps_max_length
} || 1,
5427 $drive->{bps_rd_max_length
} || 1,
5428 $drive->{bps_wr_max_length
} || 1,
5429 $drive->{iops_max_length
} || 1,
5430 $drive->{iops_rd_max_length
} || 1,
5431 $drive->{iops_wr_max_length
} || 1,
5441 if ($drive->{file
} eq 'none') {
5442 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5443 if (drive_is_cloudinit
($old_drive)) {
5444 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive);
5447 my $path = get_iso_path
($storecfg, $vmid, $drive->{file
});
5449 # force eject if locked
5450 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5453 mon_cmd
($vmid, "blockdev-change-medium",
5454 id
=> "$opt", filename
=> "$path");
5462 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5464 PVE
::Storage
::activate_volumes
($storecfg, [$drive->{file
}]) if $drive->{file
} !~ m
|^/dev/.+|;
5465 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5468 sub vmconfig_update_cloudinit_drive
{
5469 my ($storecfg, $conf, $vmid) = @_;
5471 my $cloudinit_ds = undef;
5472 my $cloudinit_drive = undef;
5474 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5475 my ($ds, $drive) = @_;
5476 if (PVE
::QemuServer
::drive_is_cloudinit
($drive)) {
5477 $cloudinit_ds = $ds;
5478 $cloudinit_drive = $drive;
5482 return if !$cloudinit_drive;
5484 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5485 PVE
::QemuConfig-
>write_config($vmid, $conf);
5488 my $running = PVE
::QemuServer
::check_running
($vmid);
5491 my $path = PVE
::Storage
::path
($storecfg, $cloudinit_drive->{file
});
5493 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$cloudinit_ds");
5494 mon_cmd
($vmid, "blockdev-change-medium", id
=> "$cloudinit_ds", filename
=> "$path");
5499 # called in locked context by incoming migration
5500 sub vm_migrate_get_nbd_disks
{
5501 my ($storecfg, $conf, $replicated_volumes) = @_;
5503 my $local_volumes = {};
5504 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5505 my ($ds, $drive) = @_;
5507 return if drive_is_cdrom
($drive);
5508 return if $ds eq 'tpmstate0';
5510 my $volid = $drive->{file
};
5514 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid);
5516 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5517 return if $scfg->{shared
};
5519 my $format = qemu_img_format
($scfg, $volname);
5521 # replicated disks re-use existing state via bitmap
5522 my $use_existing = $replicated_volumes->{$volid} ?
1 : 0;
5523 $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5525 return $local_volumes;
5528 # called in locked context by incoming migration
5529 sub vm_migrate_alloc_nbd_disks
{
5530 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5533 foreach my $opt (sort keys %$source_volumes) {
5534 my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5536 if ($use_existing) {
5537 $nbd->{$opt}->{drivestr
} = print_drive
($drive);
5538 $nbd->{$opt}->{volid
} = $volid;
5539 $nbd->{$opt}->{replicated
} = 1;
5543 $storeid = PVE
::JSONSchema
::map_id
($storagemap, $storeid);
5545 # order of precedence, filtered by whether storage supports it:
5546 # 1. explicit requested format
5547 # 2. default format of storage
5548 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5549 $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5551 my $size = $drive->{size
} / 1024;
5552 my $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $format, undef, $size);
5553 my $newdrive = $drive;
5554 $newdrive->{format
} = $format;
5555 $newdrive->{file
} = $newvolid;
5556 my $drivestr = print_drive
($newdrive);
5557 $nbd->{$opt}->{drivestr
} = $drivestr;
5558 $nbd->{$opt}->{volid
} = $newvolid;
5564 # see vm_start_nolock for parameters, additionally:
5566 # storagemap = parsed storage map for allocating NBD disks
5568 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5570 return PVE
::QemuConfig-
>lock_config($vmid, sub {
5571 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migrate_opts->{migratedfrom
});
5573 die "you can't start a vm if it's a template\n"
5574 if !$params->{skiptemplate
} && PVE
::QemuConfig-
>is_template($conf);
5576 my $has_suspended_lock = PVE
::QemuConfig-
>has_lock($conf, 'suspended');
5577 my $has_backup_lock = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5579 my $running = check_running
($vmid, undef, $migrate_opts->{migratedfrom
});
5581 if ($has_backup_lock && $running) {
5582 # a backup is currently running, attempt to start the guest in the
5583 # existing QEMU instance
5584 return vm_resume
($vmid);
5587 PVE
::QemuConfig-
>check_lock($conf)
5588 if !($params->{skiplock
} || $has_suspended_lock);
5590 $params->{resume
} = $has_suspended_lock || defined($conf->{vmstate
});
5592 die "VM $vmid already running\n" if $running;
5594 if (my $storagemap = $migrate_opts->{storagemap
}) {
5595 my $replicated = $migrate_opts->{replicated_volumes
};
5596 my $disks = vm_migrate_get_nbd_disks
($storecfg, $conf, $replicated);
5597 $migrate_opts->{nbd
} = vm_migrate_alloc_nbd_disks
($storecfg, $vmid, $disks, $storagemap);
5599 foreach my $opt (keys %{$migrate_opts->{nbd
}}) {
5600 $conf->{$opt} = $migrate_opts->{nbd
}->{$opt}->{drivestr
};
5604 return vm_start_nolock
($storecfg, $vmid, $conf, $params, $migrate_opts);
5610 # statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5611 # skiplock => 0/1, skip checking for config lock
5612 # skiptemplate => 0/1, skip checking whether VM is template
5613 # forcemachine => to force QEMU machine (rollback/migration)
5614 # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5615 # timeout => in seconds
5616 # paused => start VM in paused state (backup)
5617 # resume => resume from hibernation
5618 # live-restore-backing => {
5620 # name => blockdev-name,
5621 # blockdev => "arg to the -blockdev command instantiating device named 'name'",
5626 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5627 # migratedfrom => source node
5628 # spice_ticket => used for spice migration, passed via tunnel/stdin
5629 # network => CIDR of migration network
5630 # type => secure/insecure - tunnel over encrypted connection or plain-text
5631 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5632 # replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5633 # offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5634 # contained in config
5635 sub vm_start_nolock
{
5636 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5638 my $statefile = $params->{statefile
};
5639 my $resume = $params->{resume
};
5641 my $migratedfrom = $migrate_opts->{migratedfrom
};
5642 my $migration_type = $migrate_opts->{type
};
5646 # clean up leftover reboot request files
5647 eval { clear_reboot_request
($vmid); };
5650 if (!$statefile && scalar(keys %{$conf->{pending
}})) {
5651 vmconfig_apply_pending
($vmid, $conf, $storecfg);
5652 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5655 # don't regenerate the ISO if the VM is started as part of a live migration
5656 # this way we can reuse the old ISO with the correct config
5657 if (!$migratedfrom) {
5658 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5659 # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5660 # $conf->{cloudinit}, so we could just not do this?
5661 # But we do it above, so for now let's be consistent.
5662 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5666 # override offline migrated volumes, conf is out of date still
5667 if (my $offline_volumes = $migrate_opts->{offline_volumes
}) {
5668 for my $key (sort keys $offline_volumes->%*) {
5669 my $parsed = parse_drive
($key, $conf->{$key});
5670 $parsed->{file
} = $offline_volumes->{$key};
5671 $conf->{$key} = print_drive
($parsed);
5675 my $defaults = load_defaults
();
5677 # set environment variable useful inside network script
5678 # for remote migration the config is available on the target node!
5679 if (!$migrate_opts->{remote_node
}) {
5680 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom;
5683 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5685 my $forcemachine = $params->{forcemachine
};
5686 my $forcecpu = $params->{forcecpu
};
5688 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5689 $forcemachine = $conf->{runningmachine
};
5690 $forcecpu = $conf->{runningcpu
};
5691 print "Resuming suspended VM\n";
5694 my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command
($storecfg, $vmid,
5695 $conf, $defaults, $forcemachine, $forcecpu, $params->{'live-restore-backing'});
5698 my $get_migration_ip = sub {
5699 my ($nodename) = @_;
5701 return $migration_ip if defined($migration_ip);
5703 my $cidr = $migrate_opts->{network
};
5705 if (!defined($cidr)) {
5706 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5707 $cidr = $dc_conf->{migration
}->{network
};
5710 if (defined($cidr)) {
5711 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5713 die "could not get IP: no address configured on local " .
5714 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5716 die "could not get IP: multiple addresses configured on local " .
5717 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5719 $migration_ip = @$ips[0];
5722 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5723 if !defined($migration_ip);
5725 return $migration_ip;
5729 if ($statefile eq 'tcp') {
5730 my $migrate = $res->{migrate
} = { proto
=> 'tcp' };
5731 $migrate->{addr
} = "localhost";
5732 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5733 my $nodename = nodename
();
5735 if (!defined($migration_type)) {
5736 if (defined($datacenterconf->{migration
}->{type
})) {
5737 $migration_type = $datacenterconf->{migration
}->{type
};
5739 $migration_type = 'secure';
5743 if ($migration_type eq 'insecure') {
5744 $migrate->{addr
} = $get_migration_ip->($nodename);
5745 $migrate->{addr
} = "[$migrate->{addr}]" if Net
::IP
::ip_is_ipv6
($migrate->{addr
});
5748 # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5750 push @$cmd, '-incoming', 'defer';
5753 } elsif ($statefile eq 'unix') {
5754 # should be default for secure migrations as a ssh TCP forward
5755 # tunnel is not deterministic reliable ready and fails regurarly
5756 # to set up in time, so use UNIX socket forwards
5757 my $migrate = $res->{migrate
} = { proto
=> 'unix' };
5758 $migrate->{addr
} = "/run/qemu-server/$vmid.migrate";
5759 unlink $migrate->{addr
};
5761 $migrate->{uri
} = "unix:$migrate->{addr}";
5762 push @$cmd, '-incoming', $migrate->{uri
};
5765 } elsif (-e
$statefile) {
5766 push @$cmd, '-loadstate', $statefile;
5768 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5769 push @$vollist, $statefile;
5770 push @$cmd, '-loadstate', $statepath;
5772 } elsif ($params->{paused
}) {
5776 my $memory = get_current_memory
($conf->{memory
});
5777 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $memory, $resume);
5779 my $pci_reserve_list = [];
5780 for my $device (values $pci_devices->%*) {
5781 next if $device->{mdev
}; # we don't reserve for mdev devices
5782 push $pci_reserve_list->@*, map { $_->{id
} } $device->{ids
}->@*;
5785 # reserve all PCI IDs before actually doing anything with them
5786 PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, $start_timeout);
5790 for my $id (sort keys %$pci_devices) {
5791 my $d = $pci_devices->{$id};
5792 my ($index) = ($id =~ m/^hostpci(\d+)$/);
5795 for my $dev ($d->{ids
}->@*) {
5796 my $info = eval { PVE
::QemuServer
::PCI
::prepare_pci_device
($vmid, $dev->{id
}, $index, $d->{mdev
}) };
5799 $chosen_mdev = $info;
5800 last if $chosen_mdev; # if successful, we're done
5806 next if !$d->{mdev
};
5807 die "could not create mediated device\n" if !defined($chosen_mdev);
5809 # nvidia grid needs the uuid of the mdev as qemu parameter
5810 if (!defined($uuid) && $chosen_mdev->{vendor
} =~ m/^(0x)?10de$/) {
5811 if (defined($conf->{smbios1
})) {
5812 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
5813 $uuid = $smbios_conf->{uuid
} if defined($smbios_conf->{uuid
});
5815 $uuid = PVE
::QemuServer
::PCI
::generate_mdev_uuid
($vmid, $index) if !defined($uuid);
5818 push @$cmd, '-uuid', $uuid if defined($uuid);
5821 eval { cleanup_pci_devices
($vmid, $conf) };
5826 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5829 my %silence_std_outs = (outfunc
=> sub {}, errfunc
=> sub {});
5830 eval { run_command
(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5831 eval { run_command
(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5832 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5833 # timeout should be more than enough here...
5834 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 20);
5836 my $cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{cpuunits
});
5839 timeout
=> $statefile ?
undef : $start_timeout,
5844 # when migrating, prefix QEMU output so other side can pick up any
5845 # errors that might occur and show the user
5846 if ($migratedfrom) {
5847 $run_params{quiet
} = 1;
5848 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5851 my %systemd_properties = (
5852 Slice
=> 'qemu.slice',
5853 KillMode
=> 'process',
5855 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5858 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5859 $systemd_properties{CPUWeight
} = $cpuunits;
5861 $systemd_properties{CPUShares
} = $cpuunits;
5864 if (my $cpulimit = $conf->{cpulimit
}) {
5865 $systemd_properties{CPUQuota
} = int($cpulimit * 100);
5867 $systemd_properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5869 my $run_qemu = sub {
5870 PVE
::Tools
::run_fork
sub {
5871 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5874 if ((my $tpm = $conf->{tpmstate0
}) && !PVE
::QemuConfig-
>is_template($conf)) {
5875 # start the TPM emulator so QEMU can connect on start
5876 $tpmpid = start_swtpm
($storecfg, $vmid, $tpm, $migratedfrom);
5879 my $exitcode = run_command
($cmd, %run_params);
5882 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5883 kill 'TERM', $tpmpid;
5885 die "QEMU exited with code $exitcode\n";
5890 if ($conf->{hugepages
}) {
5893 my $hotplug_features =
5894 parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
5895 my $hugepages_topology =
5896 PVE
::QemuServer
::Memory
::hugepages_topology
($conf, $hotplug_features->{memory
});
5898 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5900 PVE
::QemuServer
::Memory
::hugepages_mount
();
5901 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5903 eval { $run_qemu->() };
5905 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5906 if !$conf->{keephugepages
};
5910 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5911 if !$conf->{keephugepages
};
5913 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5916 eval { $run_qemu->() };
5920 # deactivate volumes if start fails
5921 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5923 eval { cleanup_pci_devices
($vmid, $conf) };
5926 die "start failed: $err";
5929 # re-reserve all PCI IDs now that we can know the actual VM PID
5930 my $pid = PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
5931 eval { PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, undef, $pid) };
5934 if (defined(my $migrate = $res->{migrate
})) {
5935 if ($migrate->{proto
} eq 'tcp') {
5936 my $nodename = nodename
();
5937 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5938 $migrate->{port
} = PVE
::Tools
::next_migrate_port
($pfamily);
5939 $migrate->{uri
} = "tcp:$migrate->{addr}:$migrate->{port}";
5940 mon_cmd
($vmid, "migrate-incoming", uri
=> $migrate->{uri
});
5942 print "migration listens on $migrate->{uri}\n";
5943 } elsif ($statefile) {
5944 eval { mon_cmd
($vmid, "cont"); };
5948 #start nbd server for storage migration
5949 if (my $nbd = $migrate_opts->{nbd
}) {
5950 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5952 my $migrate_storage_uri;
5953 # nbd_protocol_version > 0 for unix socket support
5954 if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5955 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5956 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
5957 $migrate_storage_uri = "nbd:unix:$socket_path";
5958 $res->{migrate
}->{unix_sockets
} = [$socket_path];
5960 my $nodename = nodename
();
5961 my $localip = $get_migration_ip->($nodename);
5962 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5963 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5965 mon_cmd
($vmid, "nbd-server-start", addr
=> {
5968 host
=> "${localip}",
5969 port
=> "${storage_migrate_port}",
5972 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5973 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5976 my $block_info = mon_cmd
($vmid, "query-block");
5977 $block_info = { map { $_->{device
} => $_ } $block_info->@* };
5979 foreach my $opt (sort keys %$nbd) {
5980 my $drivestr = $nbd->{$opt}->{drivestr
};
5981 my $volid = $nbd->{$opt}->{volid
};
5983 my $block_node = $block_info->{"drive-$opt"}->{inserted
}->{'node-name'};
5989 'node-name' => $block_node,
5990 writable
=> JSON
::true
,
5992 name
=> "drive-$opt", # NBD export name
5995 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5996 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5997 print "re-using replicated volume: $opt - $volid\n"
5998 if $nbd->{$opt}->{replicated
};
6000 $res->{drives
}->{$opt} = $nbd->{$opt};
6001 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
6005 if ($migratedfrom) {
6007 set_migration_caps
($vmid);
6012 print "spice listens on port $spice_port\n";
6013 $res->{spice_port
} = $spice_port;
6014 if ($migrate_opts->{spice_ticket
}) {
6015 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
6016 $migrate_opts->{spice_ticket
});
6017 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
6022 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
6023 if !$statefile && $conf->{balloon
};
6025 foreach my $opt (keys %$conf) {
6026 next if $opt !~ m/^net\d+$/;
6027 my $nicconf = parse_net
($conf->{$opt});
6028 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
6030 add_nets_bridge_fdb
($conf, $vmid);
6033 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
6038 path
=> "machine/peripheral/balloon0",
6039 property
=> "guest-stats-polling-interval",
6043 log_warn
("could not set polling interval for ballooning - $@") if $@;
6047 print "Resumed VM, removing state\n";
6048 if (my $vmstate = $conf->{vmstate
}) {
6049 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6050 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6052 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6053 PVE
::QemuConfig-
>write_config($vmid, $conf);
6056 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
6058 my ($current_machine, $is_deprecated) =
6059 PVE
::QemuServer
::Machine
::get_current_qemu_machine
($vmid);
6060 if ($is_deprecated) {
6062 "current machine version '$current_machine' is deprecated - see the documentation and ".
6063 "change to a newer one",
6070 sub vm_commandline
{
6071 my ($storecfg, $vmid, $snapname) = @_;
6073 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6075 my ($forcemachine, $forcecpu);
6077 my $snapshot = $conf->{snapshots
}->{$snapname};
6078 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6080 # check for machine or CPU overrides in snapshot
6081 $forcemachine = $snapshot->{runningmachine
};
6082 $forcecpu = $snapshot->{runningcpu
};
6084 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
6089 my $defaults = load_defaults
();
6091 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6093 return PVE
::Tools
::cmd2string
($cmd);
6097 my ($vmid, $skiplock) = @_;
6099 PVE
::QemuConfig-
>lock_config($vmid, sub {
6101 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6103 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6105 mon_cmd
($vmid, "system_reset");
6109 sub get_vm_volumes
{
6113 foreach_volid
($conf, sub {
6114 my ($volid, $attr) = @_;
6116 return if $volid =~ m
|^/|;
6118 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6121 push @$vollist, $volid;
6127 sub cleanup_pci_devices
{
6128 my ($vmid, $conf) = @_;
6130 foreach my $key (keys %$conf) {
6131 next if $key !~ m/^hostpci(\d+)$/;
6132 my $hostpciindex = $1;
6133 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
6134 my $d = parse_hostpci
($conf->{$key});
6136 # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6137 # don't want to break ABI just for this two liner
6138 my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6140 # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6141 # out when we do it first. so wait for up to 10 seconds and then try it manually
6142 if ($d->{ids
}->[0]->[0]->{vendor
} =~ m/^(0x)?10de$/ && -e
$dev_sysfs_dir) {
6144 while (-e
$dev_sysfs_dir && $count < 10) {
6148 print "waited $count seconds for mediated device driver finishing clean up\n";
6151 if (-e
$dev_sysfs_dir) {
6152 print "actively clean up mediated device with UUID $uuid\n";
6153 PVE
::SysFSTools
::file_write
("$dev_sysfs_dir/remove", "1");
6157 PVE
::QemuServer
::PCI
::remove_pci_reservation
($vmid);
6160 sub vm_stop_cleanup
{
6161 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6166 my $vollist = get_vm_volumes
($conf);
6167 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6169 if (my $tpmdrive = $conf->{tpmstate0
}) {
6170 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
6171 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
6173 PVE
::Storage
::unmap_volume
($storecfg, $tpm->{file
});
6178 foreach my $ext (qw(mon qmp pid vnc qga)) {
6179 unlink "/var/run/qemu-server/${vmid}.$ext";
6182 if ($conf->{ivshmem
}) {
6183 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
6184 # just delete it for now, VMs which have this already open do not
6185 # are affected, but new VMs will get a separated one. If this
6186 # becomes an issue we either add some sort of ref-counting or just
6187 # add a "don't delete on stop" flag to the ivshmem format.
6188 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
6191 cleanup_pci_devices
($vmid, $conf);
6193 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
6195 warn $@ if $@; # avoid errors - just warn
6198 # call only in locked context
6200 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6202 my $pid = check_running
($vmid, $nocheck);
6207 $conf = PVE
::QemuConfig-
>load_config($vmid);
6208 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6209 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
6210 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
6211 $timeout = $opts->{down
} if $opts->{down
};
6213 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
6218 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
6219 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
6221 mon_cmd
($vmid, "system_powerdown");
6224 mon_cmd
($vmid, "quit");
6230 $timeout = 60 if !defined($timeout);
6233 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6238 if ($count >= $timeout) {
6240 warn "VM still running - terminating now with SIGTERM\n";
6243 die "VM quit/powerdown failed - got timeout\n";
6246 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6250 if (!check_running
($vmid, $nocheck)) {
6251 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6255 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6258 die "VM quit/powerdown failed\n";
6266 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6271 if ($count >= $timeout) {
6272 warn "VM still running - terminating now with SIGKILL\n";
6277 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6280 # Note: use $nocheck to skip tests if VM configuration file exists.
6281 # We need that when migration VMs to other nodes (files already moved)
6282 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6284 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6286 $force = 1 if !defined($force) && !$shutdown;
6289 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
6290 kill 15, $pid if $pid;
6291 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
6292 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
6296 PVE
::QemuConfig-
>lock_config($vmid, sub {
6297 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6302 my ($vmid, $timeout) = @_;
6304 PVE
::QemuConfig-
>lock_config($vmid, sub {
6307 # only reboot if running, as qmeventd starts it again on a stop event
6308 return if !check_running
($vmid);
6310 create_reboot_request
($vmid);
6312 my $storecfg = PVE
::Storage
::config
();
6313 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
6317 # avoid that the next normal shutdown will be confused for a reboot
6318 clear_reboot_request
($vmid);
6324 # note: if using the statestorage parameter, the caller has to check privileges
6326 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6333 PVE
::QemuConfig-
>lock_config($vmid, sub {
6335 $conf = PVE
::QemuConfig-
>load_config($vmid);
6337 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
6338 PVE
::QemuConfig-
>check_lock($conf)
6339 if !($skiplock || $is_backing_up);
6341 die "cannot suspend to disk during backup\n"
6342 if $is_backing_up && $includestate;
6344 if ($includestate) {
6345 $conf->{lock} = 'suspending';
6346 my $date = strftime
("%Y-%m-%d", localtime(time()));
6347 $storecfg = PVE
::Storage
::config
();
6348 if (!$statestorage) {
6349 $statestorage = find_vmstate_storage
($conf, $storecfg);
6350 # check permissions for the storage
6351 my $rpcenv = PVE
::RPCEnvironment
::get
();
6352 if ($rpcenv->{type
} ne 'cli') {
6353 my $authuser = $rpcenv->get_user();
6354 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6359 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
6360 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6361 $path = PVE
::Storage
::path
($storecfg, $vmstate);
6362 PVE
::QemuConfig-
>write_config($vmid, $conf);
6364 mon_cmd
($vmid, "stop");
6368 if ($includestate) {
6370 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
6373 set_migration_caps
($vmid, 1);
6374 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
6376 my $state = mon_cmd
($vmid, "query-savevm");
6377 if (!$state->{status
}) {
6378 die "savevm not active\n";
6379 } elsif ($state->{status
} eq 'active') {
6382 } elsif ($state->{status
} eq 'completed') {
6383 print "State saved, quitting\n";
6385 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
6386 die "query-savevm failed with error '$state->{error}'\n"
6388 die "query-savevm returned status '$state->{status}'\n";
6394 PVE
::QemuConfig-
>lock_config($vmid, sub {
6395 $conf = PVE
::QemuConfig-
>load_config($vmid);
6397 # cleanup, but leave suspending lock, to indicate something went wrong
6399 mon_cmd
($vmid, "savevm-end");
6400 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6401 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6402 delete $conf->@{qw(vmstate runningmachine runningcpu)};
6403 PVE
::QemuConfig-
>write_config($vmid, $conf);
6409 die "lock changed unexpectedly\n"
6410 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
6412 mon_cmd
($vmid, "quit");
6413 $conf->{lock} = 'suspended';
6414 PVE
::QemuConfig-
>write_config($vmid, $conf);
6419 # $nocheck is set when called as part of a migration - in this context the
6420 # location of the config file (source or target node) is not deterministic,
6421 # since migration cannot wait for pmxcfs to process the rename
6423 my ($vmid, $skiplock, $nocheck) = @_;
6425 PVE
::QemuConfig-
>lock_config($vmid, sub {
6426 my $res = mon_cmd
($vmid, 'query-status');
6427 my $resume_cmd = 'cont';
6431 $conf = eval { PVE
::QemuConfig-
>load_config($vmid) }; # try on target node
6433 my $vmlist = PVE
::Cluster
::get_vmlist
();
6434 if (exists($vmlist->{ids
}->{$vmid})) {
6435 my $node = $vmlist->{ids
}->{$vmid}->{node
};
6436 $conf = eval { PVE
::QemuConfig-
>load_config($vmid, $node) }; # try on source node
6439 PVE
::Cluster
::cfs_update
(); # vmlist was wrong, invalidate cache
6440 $conf = PVE
::QemuConfig-
>load_config($vmid); # last try on target node again
6444 $conf = PVE
::QemuConfig-
>load_config($vmid);
6447 if ($res->{status
}) {
6448 return if $res->{status
} eq 'running'; # job done, go home
6449 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
6450 $reset = 1 if $res->{status
} eq 'shutdown';
6454 PVE
::QemuConfig-
>check_lock($conf)
6455 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
6459 # required if a VM shuts down during a backup and we get a resume
6460 # request before the backup finishes for example
6461 mon_cmd
($vmid, "system_reset");
6464 add_nets_bridge_fdb
($conf, $vmid) if $resume_cmd eq 'cont';
6466 mon_cmd
($vmid, $resume_cmd);
6471 my ($vmid, $skiplock, $key) = @_;
6473 PVE
::QemuConfig-
>lock_config($vmid, sub {
6475 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6477 # there is no qmp command, so we use the human monitor command
6478 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
6479 die $res if $res ne '';
6483 sub check_bridge_access
{
6484 my ($rpcenv, $authuser, $conf) = @_;
6486 return 1 if $authuser eq 'root@pam';
6488 for my $opt (sort keys $conf->%*) {
6489 next if $opt !~ m/^net\d+$/;
6490 my $net = parse_net
($conf->{$opt});
6491 my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6492 PVE
::GuestHelpers
::check_vnet_access
($rpcenv, $authuser, $bridge, $tag, $trunks);
6497 sub check_mapping_access
{
6498 my ($rpcenv, $user, $conf) = @_;
6500 for my $opt (keys $conf->%*) {
6501 if ($opt =~ m/^usb\d+$/) {
6502 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-usb', $conf->{$opt});
6503 if (my $host = $device->{host
}) {
6504 die "only root can set '$opt' config for real devices\n"
6505 if $host !~ m/^spice$/i && $user ne 'root@pam';
6506 } elsif ($device->{mapping
}) {
6507 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6509 die "either 'host' or 'mapping' must be set.\n";
6511 } elsif ($opt =~ m/^hostpci\d+$/) {
6512 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-hostpci', $conf->{$opt});
6513 if ($device->{host
}) {
6514 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6515 } elsif ($device->{mapping
}) {
6516 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6518 die "either 'host' or 'mapping' must be set.\n";
6524 sub check_restore_permissions
{
6525 my ($rpcenv, $user, $conf) = @_;
6527 check_bridge_access
($rpcenv, $user, $conf);
6528 check_mapping_access
($rpcenv, $user, $conf);
6530 # vzdump restore implementaion
6532 sub tar_archive_read_firstfile
{
6533 my $archive = shift;
6535 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
6537 # try to detect archive type first
6538 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6539 die "unable to open file '$archive'\n";
6540 my $firstfile = <$fh>;
6544 die "ERROR: archive contaions no data\n" if !$firstfile;
6550 sub tar_restore_cleanup
{
6551 my ($storecfg, $statfile) = @_;
6553 print STDERR
"starting cleanup\n";
6555 if (my $fd = IO
::File-
>new($statfile, "r")) {
6556 while (defined(my $line = <$fd>)) {
6557 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6560 if ($volid =~ m
|^/|) {
6561 unlink $volid || die 'unlink failed\n';
6563 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6565 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6567 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6569 print STDERR
"unable to parse line in statfile - $line";
6576 sub restore_file_archive
{
6577 my ($archive, $vmid, $user, $opts) = @_;
6579 return restore_vma_archive
($archive, $vmid, $user, $opts)
6582 my $info = PVE
::Storage
::archive_info
($archive);
6583 my $format = $opts->{format
} // $info->{format
};
6584 my $comp = $info->{compression
};
6586 # try to detect archive format
6587 if ($format eq 'tar') {
6588 return restore_tar_archive
($archive, $vmid, $user, $opts);
6590 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
6594 # hepler to remove disks that will not be used after restore
6595 my $restore_cleanup_oldconf = sub {
6596 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6598 my $kept_disks = {};
6600 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
6601 my ($ds, $drive) = @_;
6603 return if drive_is_cdrom
($drive, 1);
6605 my $volid = $drive->{file
};
6606 return if !$volid || $volid =~ m
|^/|;
6608 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
6609 return if !$path || !$owner || ($owner != $vmid);
6611 # Note: only delete disk we want to restore
6612 # other volumes will become unused
6613 if ($virtdev_hash->{$ds}) {
6614 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
6619 $kept_disks->{$volid} = 1;
6623 # after the restore we have no snapshots anymore
6624 for my $snapname (keys $oldconf->{snapshots
}->%*) {
6625 my $snap = $oldconf->{snapshots
}->{$snapname};
6626 if ($snap->{vmstate
}) {
6627 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
6633 for my $volid (keys $kept_disks->%*) {
6634 eval { PVE
::Storage
::volume_snapshot_delete
($storecfg, $volid, $snapname); };
6640 # Helper to parse vzdump backup device hints
6642 # $rpcenv: Environment, used to ckeck storage permissions
6643 # $user: User ID, to check storage permissions
6644 # $storecfg: Storage configuration
6645 # $fh: the file handle for reading the configuration
6646 # $devinfo: should contain device sizes for all backu-up'ed devices
6647 # $options: backup options (pool, default storage)
6649 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6650 my $parse_backup_hints = sub {
6651 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6653 my $check_storage = sub { # assert if an image can be allocate
6654 my ($storeid, $scfg) = @_;
6655 die "Content type 'images' is not available on storage '$storeid'\n"
6656 if !$scfg->{content
}->{images
};
6657 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6658 if $user ne 'root@pam';
6661 my $virtdev_hash = {};
6662 while (defined(my $line = <$fh>)) {
6663 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6664 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6665 die "archive does not contain data for drive '$virtdev'\n"
6666 if !$devinfo->{$devname};
6668 if (defined($options->{storage
})) {
6669 $storeid = $options->{storage
} || 'local';
6670 } elsif (!$storeid) {
6673 $format = 'raw' if !$format;
6674 $devinfo->{$devname}->{devname
} = $devname;
6675 $devinfo->{$devname}->{virtdev
} = $virtdev;
6676 $devinfo->{$devname}->{format
} = $format;
6677 $devinfo->{$devname}->{storeid
} = $storeid;
6679 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6680 $check_storage->($storeid, $scfg); # permission and content type check
6682 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6683 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6685 my $drive = parse_drive
($virtdev, $2);
6687 if (drive_is_cloudinit
($drive)) {
6688 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6689 $storeid = $options->{storage
} if defined ($options->{storage
});
6690 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6691 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6693 $check_storage->($storeid, $scfg); # permission and content type check
6695 $virtdev_hash->{$virtdev} = {
6697 storeid
=> $storeid,
6698 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6705 return $virtdev_hash;
6708 # Helper to allocate and activate all volumes required for a restore
6710 # $storecfg: Storage configuration
6711 # $virtdev_hash: as returned by parse_backup_hints()
6713 # Returns: { $virtdev => $volid }
6714 my $restore_allocate_devices = sub {
6715 my ($storecfg, $virtdev_hash, $vmid) = @_;
6718 foreach my $virtdev (sort keys %$virtdev_hash) {
6719 my $d = $virtdev_hash->{$virtdev};
6720 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6721 my $storeid = $d->{storeid
};
6722 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6724 # test if requested format is supported
6725 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6726 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6727 $d->{format
} = $defFormat if !$supported;
6730 if ($d->{is_cloudinit
}) {
6731 $name = "vm-$vmid-cloudinit";
6732 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6733 if ($scfg->{path
}) {
6734 $name .= ".$d->{format}";
6738 my $volid = PVE
::Storage
::vdisk_alloc
(
6739 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6741 print STDERR
"new volume ID is '$volid'\n";
6742 $d->{volid
} = $volid;
6744 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6746 $map->{$virtdev} = $volid;
6752 sub restore_update_config_line
{
6753 my ($cookie, $map, $line, $unique) = @_;
6755 return '' if $line =~ m/^\#qmdump\#/;
6756 return '' if $line =~ m/^\#vzdump\#/;
6757 return '' if $line =~ m/^lock:/;
6758 return '' if $line =~ m/^unused\d+:/;
6759 return '' if $line =~ m/^parent:/;
6763 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6764 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6765 # try to convert old 1.X settings
6766 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6767 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6768 my ($model, $macaddr) = split(/\=/, $devconfig);
6769 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6772 bridge
=> "vmbr$ind",
6773 macaddr
=> $macaddr,
6775 my $netstr = print_net
($net);
6777 $res .= "net$cookie->{netcount}: $netstr\n";
6778 $cookie->{netcount
}++;
6780 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6781 my ($id, $netstr) = ($1, $2);
6782 my $net = parse_net
($netstr);
6783 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6784 $netstr = print_net
($net);
6785 $res .= "$id: $netstr\n";
6786 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6789 my $di = parse_drive
($virtdev, $value);
6790 if (defined($di->{backup
}) && !$di->{backup
}) {
6792 } elsif ($map->{$virtdev}) {
6793 delete $di->{format
}; # format can change on restore
6794 $di->{file
} = $map->{$virtdev};
6795 $value = print_drive
($di);
6796 $res .= "$virtdev: $value\n";
6800 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6802 if ($vmgenid ne '0') {
6803 # always generate a new vmgenid if there was a valid one setup
6804 $vmgenid = generate_uuid
();
6806 $res .= "vmgenid: $vmgenid\n";
6807 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6808 my ($uuid, $uuid_str);
6809 UUID
::generate
($uuid);
6810 UUID
::unparse
($uuid, $uuid_str);
6811 my $smbios1 = parse_smbios1
($2);
6812 $smbios1->{uuid
} = $uuid_str;
6813 $res .= $1.print_smbios1
($smbios1)."\n";
6821 my $restore_deactivate_volumes = sub {
6822 my ($storecfg, $virtdev_hash) = @_;
6825 for my $dev (values $virtdev_hash->%*) {
6826 push $vollist->@*, $dev->{volid
} if $dev->{volid
};
6829 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
6830 print STDERR
$@ if $@;
6833 my $restore_destroy_volumes = sub {
6834 my ($storecfg, $virtdev_hash) = @_;
6836 for my $dev (values $virtdev_hash->%*) {
6837 my $volid = $dev->{volid
} or next;
6839 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6840 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6842 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6846 sub restore_merge_config
{
6847 my ($filename, $backup_conf_raw, $override_conf) = @_;
6849 my $backup_conf = parse_vm_config
($filename, $backup_conf_raw);
6850 for my $key (keys $override_conf->%*) {
6851 $backup_conf->{$key} = $override_conf->{$key};
6854 return $backup_conf;
6858 my ($cfg, $vmid) = @_;
6860 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6862 my $volid_hash = {};
6863 foreach my $storeid (keys %$info) {
6864 foreach my $item (@{$info->{$storeid}}) {
6865 next if !($item->{volid
} && $item->{size
});
6866 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6867 $volid_hash->{$item->{volid
}} = $item;
6874 sub update_disk_config
{
6875 my ($vmid, $conf, $volid_hash) = @_;
6878 my $prefix = "VM $vmid";
6880 # used and unused disks
6881 my $referenced = {};
6883 # Note: it is allowed to define multiple storages with same path (alias), so
6884 # we need to check both 'volid' and real 'path' (two different volid can point
6885 # to the same path).
6887 my $referencedpath = {};
6890 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6891 my ($opt, $drive) = @_;
6893 my $volid = $drive->{file
};
6895 my $volume = $volid_hash->{$volid};
6897 # mark volid as "in-use" for next step
6898 $referenced->{$volid} = 1;
6899 if ($volume && (my $path = $volume->{path
})) {
6900 $referencedpath->{$path} = 1;
6903 return if drive_is_cdrom
($drive);
6906 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6907 if (defined($updated)) {
6909 $conf->{$opt} = print_drive
($updated);
6910 print "$prefix ($opt): $msg\n";
6914 # remove 'unusedX' entry if volume is used
6915 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6916 my ($opt, $drive) = @_;
6918 my $volid = $drive->{file
};
6922 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6923 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6924 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6926 delete $conf->{$opt};
6929 $referenced->{$volid} = 1;
6930 $referencedpath->{$path} = 1 if $path;
6933 foreach my $volid (sort keys %$volid_hash) {
6934 next if $volid =~ m/vm-$vmid-state-/;
6935 next if $referenced->{$volid};
6936 my $path = $volid_hash->{$volid}->{path
};
6937 next if !$path; # just to be sure
6938 next if $referencedpath->{$path};
6940 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6941 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6942 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6949 my ($vmid, $nolock, $dryrun) = @_;
6951 my $cfg = PVE
::Storage
::config
();
6953 print "rescan volumes...\n";
6954 my $volid_hash = scan_volids
($cfg, $vmid);
6956 my $updatefn = sub {
6959 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6961 PVE
::QemuConfig-
>check_lock($conf);
6964 foreach my $volid (keys %$volid_hash) {
6965 my $info = $volid_hash->{$volid};
6966 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6969 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6971 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6974 if (defined($vmid)) {
6978 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6981 my $vmlist = config_list
();
6982 foreach my $vmid (keys %$vmlist) {
6986 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6992 sub restore_proxmox_backup_archive
{
6993 my ($archive, $vmid, $user, $options) = @_;
6995 my $storecfg = PVE
::Storage
::config
();
6997 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6998 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7000 my $fingerprint = $scfg->{fingerprint
};
7001 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
7003 my $repo = PVE
::PBSClient
::get_repository
($scfg);
7004 my $namespace = $scfg->{namespace
};
7006 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7007 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
7008 local $ENV{PBS_PASSWORD
} = $password;
7009 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
7011 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7012 PVE
::Storage
::parse_volname
($storecfg, $archive);
7014 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7016 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7018 my $tmpdir = "/var/tmp/vzdumptmp$$";
7022 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7023 # disable interrupts (always do cleanups)
7027 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7029 # Note: $oldconf is undef if VM does not exists
7030 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7031 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7032 my $new_conf_raw = '';
7034 my $rpcenv = PVE
::RPCEnvironment
::get
();
7035 my $devinfo = {}; # info about drives included in backup
7036 my $virtdev_hash = {}; # info about allocated drives
7044 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7046 my $cfgfn = "$tmpdir/qemu-server.conf";
7047 my $firewall_config_fn = "$tmpdir/fw.conf";
7048 my $index_fn = "$tmpdir/index.json";
7050 my $cmd = "restore";
7052 my $param = [$pbs_backup_name, "index.json", $index_fn];
7053 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7054 my $index = PVE
::Tools
::file_get_contents
($index_fn);
7055 $index = decode_json
($index);
7057 foreach my $info (@{$index->{files
}}) {
7058 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
7060 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
7061 $devinfo->{$devname}->{size
} = $1;
7063 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7068 my $is_qemu_server_backup = scalar(
7069 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
7071 if (!$is_qemu_server_backup) {
7072 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7074 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
7076 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7077 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7079 if ($has_firewall_config) {
7080 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7081 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7083 my $pve_firewall_dir = '/etc/pve/firewall';
7084 mkdir $pve_firewall_dir; # make sure the dir exists
7085 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7088 my $fh = IO
::File-
>new($cfgfn, "r") ||
7089 die "unable to read qemu-server.conf - $!\n";
7091 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7093 # fixme: rate limit?
7095 # create empty/temp config
7096 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
7098 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7101 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7103 foreach my $virtdev (sort keys %$virtdev_hash) {
7104 my $d = $virtdev_hash->{$virtdev};
7105 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7107 # this fails if storage is unavailable
7108 my $volid = $d->{volid
};
7109 my $path = PVE
::Storage
::path
($storecfg, $volid);
7111 # for live-restore we only want to preload the efidisk and TPM state
7112 next if $options->{live
} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7115 if (defined(my $ns = $scfg->{namespace
})) {
7116 @ns_arg = ('--ns', $ns);
7119 my $pbs_restore_cmd = [
7120 '/usr/bin/pbs-restore',
7121 '--repository', $repo,
7124 "$d->{devname}.img.fidx",
7129 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
7130 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
7132 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
7133 push @$pbs_restore_cmd, '--skip-zero';
7136 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
7137 print "restore proxmox backup image: $dbg_cmdstring\n";
7138 run_command
($pbs_restore_cmd);
7141 $fh->seek(0, 0) || die "seek failed - $!\n";
7143 my $cookie = { netcount
=> 0 };
7144 while (defined(my $line = <$fh>)) {
7145 $new_conf_raw .= restore_update_config_line
(
7157 if ($err || !$options->{live
}) {
7158 $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7164 $restore_destroy_volumes->($storecfg, $virtdev_hash);
7168 if ($options->{live
}) {
7169 # keep lock during live-restore
7170 $new_conf_raw .= "\nlock: create";
7173 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $options->{override_conf
});
7174 check_restore_permissions
($rpcenv, $user, $new_conf);
7175 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7177 eval { rescan
($vmid, 1); };
7180 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
7182 if ($options->{live
}) {
7188 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
7190 my $conf = PVE
::QemuConfig-
>load_config($vmid);
7191 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
7193 # these special drives are already restored before start
7194 delete $devinfo->{'drive-efidisk0'};
7195 delete $devinfo->{'drive-tpmstate0-backup'};
7199 keyfile
=> $keyfile,
7200 snapshot
=> $pbs_backup_name,
7201 namespace
=> $namespace,
7203 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7205 PVE
::QemuConfig-
>remove_lock($vmid, "create");
7209 sub pbs_live_restore
{
7210 my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7212 print "starting VM for live-restore\n";
7213 print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7215 my $live_restore_backing = {};
7216 for my $ds (keys %$restored_disks) {
7217 $ds =~ m/^drive-(.*)$/;
7221 repository
=> $opts->{repo
},
7222 snapshot
=> $opts->{snapshot
},
7223 archive
=> "$ds.img.fidx",
7225 $pbs_conf->{keyfile
} = $opts->{keyfile
} if -e
$opts->{keyfile
};
7226 $pbs_conf->{namespace
} = $opts->{namespace
} if defined($opts->{namespace
});
7228 my $drive = parse_drive
($confname, $conf->{$confname});
7229 print "restoring '$ds' to '$drive->{file}'\n";
7231 my $pbs_name = "drive-${confname}-pbs";
7232 $live_restore_backing->{$confname} = {
7234 blockdev
=> print_pbs_blockdev
($pbs_conf, $pbs_name),
7238 my $drives_streamed = 0;
7240 # make sure HA doesn't interrupt our restore by stopping the VM
7241 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
7242 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7245 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7246 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7247 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'live-restore-backing' => $live_restore_backing}, {});
7249 my $qmeventd_fd = register_qmeventd_handle
($vmid);
7251 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7252 # this will effectively collapse the backing image chain consisting of
7253 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7254 # removes itself once all backing images vanish with 'auto-remove=on')
7256 for my $ds (sort keys %$restored_disks) {
7257 my $job_id = "restore-$ds";
7258 mon_cmd
($vmid, 'block-stream',
7259 'job-id' => $job_id,
7262 $jobs->{$job_id} = {};
7265 mon_cmd
($vmid, 'cont');
7266 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
7268 print "restore-drive jobs finished successfully, removing all tracking block devices"
7269 ." to disconnect from Proxmox Backup Server\n";
7271 for my $ds (sort keys %$restored_disks) {
7272 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7275 close($qmeventd_fd);
7281 warn "An error occurred during live-restore: $err\n";
7282 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
7283 die "live-restore failed\n";
7287 # Inspired by pbs live-restore, this restores with the disks being available as files.
7288 # Theoretically this can also be used to quick-start a full-clone vm if the
7289 # disks are all available as files.
7291 # The mapping should provide a path by config entry, such as
7292 # `{ scsi0 => { format => <qcow2|raw|...>, path => "/path/to/file", sata1 => ... } }`
7294 # This is used when doing a `create` call with the `--live-import` parameter,
7295 # where the disks get an `import-from=` property. The non-live part is
7296 # therefore already handled in the `$create_disks()` call happening in the
7298 sub live_import_from_files
{
7299 my ($mapping, $vmid, $conf, $restore_options) = @_;
7301 die "only live-restore is implemented for restirng from files\n"
7302 if !$restore_options->{live
};
7304 my $live_restore_backing = {};
7305 for my $dev (keys %$mapping) {
7306 die "disk not support for live-restoring: '$dev'\n"
7307 if !is_valid_drivename
($dev) || $dev =~ /^(?:efidisk|tpmstate)/;
7309 die "mapping contains disk '$dev' which does not exist in the config\n"
7310 if !exists($conf->{$dev});
7312 my $info = $mapping->{$dev};
7313 my ($format, $path) = $info->@{qw(format path)};
7314 die "missing path for '$dev' mapping\n" if !$path;
7315 die "missing format for '$dev' mapping\n" if !$format;
7316 die "invalid format '$format' for '$dev' mapping\n"
7317 if !grep { $format eq $_ } qw(raw qcow2 vmdk);
7319 $live_restore_backing->{$dev} = {
7320 name
=> "drive-$dev-restore",
7321 blockdev
=> "driver=$format,node-name=drive-$dev-restore"
7323 . ",file.driver=file,file.filename=$path"
7327 my $storecfg = PVE
::Storage
::config
();
7330 # make sure HA doesn't interrupt our restore by stopping the VM
7331 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
7332 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7335 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'live-restore-backing' => $live_restore_backing}, {});
7337 # prevent shutdowns from qmeventd when the VM powers off from the inside
7338 my $qmeventd_fd = register_qmeventd_handle
($vmid);
7340 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7341 # this will effectively collapse the backing image chain consisting of
7342 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7343 # removes itself once all backing images vanish with 'auto-remove=on')
7345 for my $ds (sort keys %$live_restore_backing) {
7346 my $job_id = "restore-$ds";
7347 mon_cmd
($vmid, 'block-stream',
7348 'job-id' => $job_id,
7349 device
=> "drive-$ds",
7351 $jobs->{$job_id} = {};
7354 mon_cmd
($vmid, 'cont');
7355 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
7357 print "restore-drive jobs finished successfully, removing all tracking block devices\n";
7359 for my $ds (sort keys %$live_restore_backing) {
7360 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "drive-$ds-restore");
7363 close($qmeventd_fd);
7369 warn "An error occurred during live-restore: $err\n";
7370 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
7371 die "live-restore failed\n";
7374 PVE
::QemuConfig-
>remove_lock($vmid, "import");
7377 sub restore_vma_archive
{
7378 my ($archive, $vmid, $user, $opts, $comp) = @_;
7380 my $readfrom = $archive;
7382 my $cfg = PVE
::Storage
::config
();
7384 my $bwlimit = $opts->{bwlimit
};
7386 my $dbg_cmdstring = '';
7387 my $add_pipe = sub {
7389 push @$commands, $cmd;
7390 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7391 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
7396 if ($archive eq '-') {
7399 # If we use a backup from a PVE defined storage we also consider that
7400 # storage's rate limit:
7401 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
7402 if (defined($volid)) {
7403 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
7404 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
7406 print STDERR
"applying read rate limit: $readlimit\n";
7407 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7408 $add_pipe->($cstream);
7414 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
7415 my $cmd = $info->{decompressor
};
7416 push @$cmd, $readfrom;
7420 my $tmpdir = "/var/tmp/vzdumptmp$$";
7423 # disable interrupts (always do cleanups)
7427 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
7429 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7430 POSIX
::mkfifo
($mapfifo, 0600);
7432 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7434 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7436 my $devinfo = {}; # info about drives included in backup
7437 my $virtdev_hash = {}; # info about allocated drives
7439 my $rpcenv = PVE
::RPCEnvironment
::get
();
7441 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7443 # Note: $oldconf is undef if VM does not exist
7444 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7445 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7446 my $new_conf_raw = '';
7450 my $print_devmap = sub {
7451 my $cfgfn = "$tmpdir/qemu-server.conf";
7453 # we can read the config - that is already extracted
7454 my $fh = IO
::File-
>new($cfgfn, "r") ||
7455 die "unable to read qemu-server.conf - $!\n";
7457 my $fwcfgfn = "$tmpdir/qemu-server.fw";
7459 my $pve_firewall_dir = '/etc/pve/firewall';
7460 mkdir $pve_firewall_dir; # make sure the dir exists
7461 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7464 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7466 foreach my $info (values %{$virtdev_hash}) {
7467 my $storeid = $info->{storeid
};
7468 next if defined($storage_limits{$storeid});
7470 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
7471 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7472 $storage_limits{$storeid} = $limit * 1024;
7475 foreach my $devname (keys %$devinfo) {
7476 die "found no device mapping information for device '$devname'\n"
7477 if !$devinfo->{$devname}->{virtdev
};
7480 # create empty/temp config
7482 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
7483 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7487 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7489 # print restore information to $fifofh
7490 foreach my $virtdev (sort keys %$virtdev_hash) {
7491 my $d = $virtdev_hash->{$virtdev};
7492 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7494 my $storeid = $d->{storeid
};
7495 my $volid = $d->{volid
};
7498 if (my $limit = $storage_limits{$storeid}) {
7499 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7502 my $write_zeros = 1;
7503 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
7507 my $path = PVE
::Storage
::path
($cfg, $volid);
7509 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7511 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7514 $fh->seek(0, 0) || die "seek failed - $!\n";
7516 my $cookie = { netcount
=> 0 };
7517 while (defined(my $line = <$fh>)) {
7518 $new_conf_raw .= restore_update_config_line
(
7537 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7538 local $SIG{ALRM
} = sub { die "got timeout\n"; };
7540 $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7547 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7548 my ($dev_id, $size, $devname) = ($1, $2, $3);
7549 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
7550 } elsif ($line =~ m/^CTIME: /) {
7551 # we correctly received the vma config, so we can disable
7552 # the timeout now for disk allocation
7553 alarm($oldtimeout || 0);
7554 $oldtimeout = undef;
7556 print $fifofh "done\n";
7562 print "restore vma archive: $dbg_cmdstring\n";
7563 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
7567 alarm($oldtimeout) if $oldtimeout;
7569 $restore_deactivate_volumes->($cfg, $virtdev_hash);
7571 close($fifofh) if $fifofh;
7576 $restore_destroy_volumes->($cfg, $virtdev_hash);
7580 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $opts->{override_conf
});
7581 check_restore_permissions
($rpcenv, $user, $new_conf);
7582 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7584 eval { rescan
($vmid, 1); };
7587 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
7590 sub restore_tar_archive
{
7591 my ($archive, $vmid, $user, $opts) = @_;
7593 if (scalar(keys $opts->{override_conf
}->%*) > 0) {
7594 my $keystring = join(' ', keys $opts->{override_conf
}->%*);
7595 die "cannot pass along options ($keystring) when restoring from tar archive\n";
7598 if ($archive ne '-') {
7599 my $firstfile = tar_archive_read_firstfile
($archive);
7600 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7601 if $firstfile ne 'qemu-server.conf';
7604 my $storecfg = PVE
::Storage
::config
();
7606 # avoid zombie disks when restoring over an existing VM -> cleanup first
7607 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7608 # skiplock=1 because qmrestore has set the 'create' lock itself already
7609 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
7610 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
7612 my $tocmd = "/usr/lib/qemu-server/qmextract";
7614 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
7615 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
7616 $tocmd .= ' --prealloc' if $opts->{prealloc
};
7617 $tocmd .= ' --info' if $opts->{info
};
7619 # tar option "xf" does not autodetect compression when read from STDIN,
7620 # so we pipe to zcat
7621 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
7622 PVE
::Tools
::shellquote
("--to-command=$tocmd");
7624 my $tmpdir = "/var/tmp/vzdumptmp$$";
7627 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
7628 local $ENV{VZDUMP_VMID
} = $vmid;
7629 local $ENV{VZDUMP_USER
} = $user;
7631 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7632 my $new_conf_raw = '';
7634 # disable interrupts (always do cleanups)
7638 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7646 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7648 if ($archive eq '-') {
7649 print "extracting archive from STDIN\n";
7650 run_command
($cmd, input
=> "<&STDIN");
7652 print "extracting archive '$archive'\n";
7656 return if $opts->{info
};
7660 my $statfile = "$tmpdir/qmrestore.stat";
7661 if (my $fd = IO
::File-
>new($statfile, "r")) {
7662 while (defined (my $line = <$fd>)) {
7663 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7664 $map->{$1} = $2 if $1;
7666 print STDERR
"unable to parse line in statfile - $line\n";
7672 my $confsrc = "$tmpdir/qemu-server.conf";
7674 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7676 my $cookie = { netcount
=> 0 };
7677 while (defined (my $line = <$srcfd>)) {
7678 $new_conf_raw .= restore_update_config_line
(
7689 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
7695 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7697 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7699 eval { rescan
($vmid, 1); };
7703 sub foreach_storage_used_by_vm
{
7704 my ($conf, $func) = @_;
7708 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7709 my ($ds, $drive) = @_;
7710 return if drive_is_cdrom
($drive);
7712 my $volid = $drive->{file
};
7714 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
7715 $sidhash->{$sid} = $sid if $sid;
7718 foreach my $sid (sort keys %$sidhash) {
7723 my $qemu_snap_storage = {
7726 sub do_snapshots_with_qemu
{
7727 my ($storecfg, $volid, $deviceid) = @_;
7729 return if $deviceid =~ m/tpmstate0/;
7731 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
7732 my $scfg = $storecfg->{ids
}->{$storage_name};
7733 die "could not find storage '$storage_name'\n" if !defined($scfg);
7735 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
7739 if ($volid =~ m/\.(qcow2|qed)$/){
7746 sub qga_check_running
{
7747 my ($vmid, $nowarn) = @_;
7749 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
7751 warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7757 sub template_create
{
7758 my ($vmid, $conf, $disk) = @_;
7760 my $storecfg = PVE
::Storage
::config
();
7762 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7763 my ($ds, $drive) = @_;
7765 return if drive_is_cdrom
($drive);
7766 return if $disk && $ds ne $disk;
7768 my $volid = $drive->{file
};
7769 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
7771 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
7772 $drive->{file
} = $voliddst;
7773 $conf->{$ds} = print_drive
($drive);
7774 PVE
::QemuConfig-
>write_config($vmid, $conf);
7778 sub convert_iscsi_path
{
7781 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
7786 my $initiator_name = get_initiator_name
();
7788 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7789 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7792 die "cannot convert iscsi path '$path', unkown format\n";
7795 sub qemu_img_convert
{
7796 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7798 my $storecfg = PVE
::Storage
::config
();
7799 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
7800 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
7802 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7806 my $src_is_iscsi = 0;
7810 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
7811 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7812 $src_format = qemu_img_format
($src_scfg, $src_volname);
7813 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
7814 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
7815 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
7816 } elsif (-f
$src_volid || -b
$src_volid) {
7817 $src_path = $src_volid;
7818 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7823 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7825 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7826 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7827 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7828 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7831 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7832 push @$cmd, '-l', "snapshot.name=$snapname"
7833 if $snapname && $src_format && $src_format eq "qcow2";
7834 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7835 push @$cmd, '-T', $cachemode if defined($cachemode);
7836 push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7838 if ($src_is_iscsi) {
7839 push @$cmd, '--image-opts';
7840 $src_path = convert_iscsi_path
($src_path);
7841 } elsif ($src_format) {
7842 push @$cmd, '-f', $src_format;
7845 if ($dst_is_iscsi) {
7846 push @$cmd, '--target-image-opts';
7847 $dst_path = convert_iscsi_path
($dst_path);
7849 push @$cmd, '-O', $dst_format;
7852 push @$cmd, $src_path;
7854 if (!$dst_is_iscsi && $is_zero_initialized) {
7855 push @$cmd, "zeroinit:$dst_path";
7857 push @$cmd, $dst_path;
7862 if($line =~ m/\((\S+)\/100\
%\)/){
7864 my $transferred = int($size * $percent / 100);
7865 my $total_h = render_bytes
($size, 1);
7866 my $transferred_h = render_bytes
($transferred, 1);
7868 print "transferred $transferred_h of $total_h ($percent%)\n";
7873 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7875 die "copy failed: $err" if $err;
7878 sub qemu_img_format
{
7879 my ($scfg, $volname) = @_;
7881 # FIXME: this entire function is kind of weird given that `parse_volname`
7882 # also already gives us a format?
7883 my $is_path_storage = $scfg->{path
} || $scfg->{type
} eq 'esxi';
7885 if ($is_path_storage && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7892 sub qemu_drive_mirror
{
7893 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7895 $jobs = {} if !$jobs;
7899 $jobs->{"drive-$drive"} = {};
7901 if ($dst_volid =~ /^nbd:/) {
7902 $qemu_target = $dst_volid;
7905 my $storecfg = PVE
::Storage
::config
();
7906 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7908 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7910 $format = qemu_img_format
($dst_scfg, $dst_volname);
7912 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7914 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7917 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7918 $opts->{format
} = $format if $format;
7920 if (defined($src_bitmap)) {
7921 $opts->{sync
} = 'incremental';
7922 $opts->{bitmap
} = $src_bitmap;
7923 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7926 if (defined($bwlimit)) {
7927 $opts->{speed
} = $bwlimit * 1024;
7928 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7930 print "drive mirror is starting for drive-$drive\n";
7933 # if a job already runs for this device we get an error, catch it for cleanup
7934 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7936 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7938 die "mirroring error: $err\n";
7941 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7944 # $completion can be either
7945 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7946 # 'cancel': wait until all jobs are ready, block-job-cancel them
7947 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7948 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7949 sub qemu_drive_mirror_monitor
{
7950 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7952 $completion //= 'complete';
7956 my $err_complete = 0;
7958 my $starttime = time ();
7960 die "block job ('$op') timed out\n" if $err_complete > 300;
7962 my $stats = mon_cmd
($vmid, "query-block-jobs");
7965 my $running_jobs = {};
7966 for my $stat (@$stats) {
7967 next if $stat->{type
} ne $op;
7968 $running_jobs->{$stat->{device
}} = $stat;
7971 my $readycounter = 0;
7973 for my $job_id (sort keys %$jobs) {
7974 my $job = $running_jobs->{$job_id};
7976 my $vanished = !defined($job);
7977 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7978 if($complete || ($vanished && $completion eq 'auto')) {
7979 print "$job_id: $op-job finished\n";
7980 delete $jobs->{$job_id};
7984 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7986 my $busy = $job->{busy
};
7987 my $ready = $job->{ready
};
7988 if (my $total = $job->{len
}) {
7989 my $transferred = $job->{offset
} || 0;
7990 my $remaining = $total - $transferred;
7991 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7993 my $duration = $ctime - $starttime;
7994 my $total_h = render_bytes
($total, 1);
7995 my $transferred_h = render_bytes
($transferred, 1);
7997 my $status = sprintf(
7998 "transferred $transferred_h of $total_h ($percent%%) in %s",
7999 render_duration
($duration),
8004 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
8006 $status .= ", ready";
8009 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
8010 $jobs->{$job_id}->{ready
} = $ready;
8013 $readycounter++ if $job->{ready
};
8016 last if scalar(keys %$jobs) == 0;
8018 if ($readycounter == scalar(keys %$jobs)) {
8019 print "all '$op' jobs are ready\n";
8021 # do the complete later (or has already been done)
8022 last if $completion eq 'skip' || $completion eq 'auto';
8024 if ($vmiddst && $vmiddst != $vmid) {
8025 my $agent_running = $qga && qga_check_running
($vmid);
8026 if ($agent_running) {
8027 print "freeze filesystem\n";
8028 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
8031 print "suspend vm\n";
8032 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
8036 # if we clone a disk for a new target vm, we don't switch the disk
8037 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
8039 if ($agent_running) {
8040 print "unfreeze filesystem\n";
8041 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
8044 print "resume vm\n";
8045 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
8052 for my $job_id (sort keys %$jobs) {
8053 # try to switch the disk if source and destination are on the same guest
8054 print "$job_id: Completing block job_id...\n";
8057 if ($completion eq 'complete') {
8058 $op = 'block-job-complete';
8059 } elsif ($completion eq 'cancel') {
8060 $op = 'block-job-cancel';
8062 die "invalid completion value: $completion\n";
8064 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
8065 if ($@ =~ m/cannot be completed/) {
8066 print "$job_id: block job cannot be completed, trying again.\n";
8069 print "$job_id: Completed successfully.\n";
8070 $jobs->{$job_id}->{complete
} = 1;
8081 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
8082 die "block job ($op) error: $err";
8086 sub qemu_blockjobs_cancel
{
8087 my ($vmid, $jobs) = @_;
8089 foreach my $job (keys %$jobs) {
8090 print "$job: Cancelling block job\n";
8091 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
8092 $jobs->{$job}->{cancel
} = 1;
8096 my $stats = mon_cmd
($vmid, "query-block-jobs");
8098 my $running_jobs = {};
8099 foreach my $stat (@$stats) {
8100 $running_jobs->{$stat->{device
}} = $stat;
8103 foreach my $job (keys %$jobs) {
8105 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
8106 print "$job: Done.\n";
8107 delete $jobs->{$job};
8111 last if scalar(keys %$jobs) == 0;
8117 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8118 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8119 my sub clone_disk_check_io_uring
{
8120 my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8122 return if !$use_drive_mirror;
8124 # Don't complain when not changing storage.
8125 # Assume if it works for the source, it'll work for the target too.
8126 return if $src_storeid eq $dst_storeid;
8128 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
8129 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
8131 my $cache_direct = drive_uses_cache_direct
($src_drive);
8133 my $src_uses_io_uring;
8134 if ($src_drive->{aio
}) {
8135 $src_uses_io_uring = $src_drive->{aio
} eq 'io_uring';
8137 $src_uses_io_uring = storage_allows_io_uring_default
($src_scfg, $cache_direct);
8140 die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8141 if $src_uses_io_uring && !storage_allows_io_uring_default
($dst_scfg, $cache_direct);
8145 my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8147 my ($vmid, $running) = $source->@{qw(vmid running)};
8148 my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8150 my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8151 my ($storage, $format) = $dest->@{qw(storage format)};
8153 my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8155 if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8156 die "cloning from/to EFI disk requires EFI disk\n"
8157 if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8158 die "cloning from/to TPM state requires TPM state\n"
8159 if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8161 # This would lead to two device nodes in QEMU pointing to the same backing image!
8162 die "cannot change drive name when cloning disk from/to the same VM\n"
8163 if $use_drive_mirror && $vmid == $newvmid;
8166 die "cannot move TPM state while VM is running\n"
8167 if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8171 print "create " . ($full ?
'full' : 'linked') . " clone of drive ";
8172 print "$src_drivename " if $src_drivename;
8173 print "($drive->{file})\n";
8176 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
8177 push @$newvollist, $newvolid;
8179 my ($src_storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
8180 my $storeid = $storage || $src_storeid;
8182 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
8186 if (drive_is_cloudinit
($drive)) {
8187 $name = "vm-$newvmid-cloudinit";
8188 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8189 if ($scfg->{path
}) {
8190 $name .= ".$dst_format";
8193 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
8194 } elsif ($dst_drivename eq 'efidisk0') {
8195 $size = $efisize or die "internal error - need to specify EFI disk size\n";
8196 } elsif ($dst_drivename eq 'tpmstate0') {
8197 $dst_format = 'raw';
8198 $size = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8200 clone_disk_check_io_uring
($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8202 $size = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
8204 $newvolid = PVE
::Storage
::vdisk_alloc
(
8205 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8207 push @$newvollist, $newvolid;
8209 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
8211 if (drive_is_cloudinit
($drive)) {
8212 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8213 # if this is the case, we have to complete any block-jobs still there from
8214 # previous drive-mirrors
8215 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8216 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
8221 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
8222 if ($use_drive_mirror) {
8223 qemu_drive_mirror
($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8224 $completion, $qga, $bwlimit);
8226 if ($dst_drivename eq 'efidisk0') {
8227 # the relevant data on the efidisk may be smaller than the source
8228 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8229 # that is given by the OVMF_VARS.fd
8230 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
8231 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
8233 my $src_format = (PVE
::Storage
::parse_volname
($storecfg, $drive->{file
}))[6];
8235 # better for Ceph if block size is not too small, see bug #3324
8238 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8240 if ($src_format eq 'qcow2' && $snapname) {
8241 die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8242 if !min_version
(kvm_user_version
(), 6, 2);
8243 push $cmd->@*, '-l', $snapname;
8245 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8248 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8254 my $size = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
8256 my $disk = dclone
($drive);
8257 delete $disk->{format
};
8258 $disk->{file
} = $newvolid;
8259 $disk->{size
} = $size if defined($size);
8264 sub get_running_qemu_version
{
8266 my $res = mon_cmd
($vmid, "query-version");
8267 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8270 sub qemu_use_old_bios_files
{
8271 my ($machine_type) = @_;
8273 return if !$machine_type;
8275 my $use_old_bios_files = undef;
8277 if ($machine_type =~ m/^(\S+)\.pxe$/) {
8279 $use_old_bios_files = 1;
8281 my $version = extract_version
($machine_type, kvm_user_version
());
8282 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8283 # load new efi bios files on migration. So this hack is required to allow
8284 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8285 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8286 $use_old_bios_files = !min_version
($version, 2, 4);
8289 return ($use_old_bios_files, $machine_type);
8292 sub get_efivars_size
{
8293 my ($conf, $efidisk) = @_;
8295 my $arch = get_vm_arch
($conf);
8296 $efidisk //= $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
8297 my $smm = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
8298 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8299 return -s
$ovmf_vars;
8302 sub update_efidisk_size
{
8305 return if !defined($conf->{efidisk0
});
8307 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
8308 $disk->{size
} = get_efivars_size
($conf);
8309 $conf->{efidisk0
} = print_drive
($disk);
8314 sub update_tpmstate_size
{
8317 my $disk = PVE
::QemuServer
::parse_drive
('tpmstate0', $conf->{tpmstate0
});
8318 $disk->{size
} = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8319 $conf->{tpmstate0
} = print_drive
($disk);
8322 sub create_efidisk
($$$$$$$) {
8323 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8325 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8327 my $vars_size_b = -s
$ovmf_vars;
8328 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
8329 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8330 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
8332 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
8333 my $size = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
8335 return ($volid, $size/1024);
8338 sub vm_iothreads_list
{
8341 my $res = mon_cmd
($vmid, 'query-iothreads');
8344 foreach my $iothread (@$res) {
8345 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
8352 my ($conf, $drive) = @_;
8356 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
8358 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
8364 my $controller = int($drive->{index} / $maxdev);
8365 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
8369 return ($maxdev, $controller, $controller_prefix);
8372 sub resolve_dst_disk_format
{
8373 my ($storecfg, $storeid, $src_volname, $format) = @_;
8374 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
8377 # if no target format is specified, use the source disk format as hint
8379 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8380 $format = qemu_img_format
($scfg, $src_volname);
8386 # test if requested format is supported - else use default
8387 my $supported = grep { $_ eq $format } @$validFormats;
8388 $format = $defFormat if !$supported;
8392 # NOTE: if this logic changes, please update docs & possibly gui logic
8393 sub find_vmstate_storage
{
8394 my ($conf, $storecfg) = @_;
8396 # first, return storage from conf if set
8397 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
8399 my ($target, $shared, $local);
8401 foreach_storage_used_by_vm
($conf, sub {
8403 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
8404 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
8405 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
8408 # second, use shared storage where VM has at least one disk
8409 # third, use local storage where VM has at least one disk
8410 # fall back to local storage
8411 $target = $shared // $local // 'local';
8417 my ($uuid, $uuid_str);
8418 UUID
::generate
($uuid);
8419 UUID
::unparse
($uuid, $uuid_str);
8423 sub generate_smbios1_uuid
{
8424 return "uuid=".generate_uuid
();
8430 mon_cmd
($vmid, 'nbd-server-stop', timeout
=> 25);
8433 sub create_reboot_request
{
8435 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8436 or die "failed to create reboot trigger file: $!\n";
8440 sub clear_reboot_request
{
8442 my $path = "/run/qemu-server/$vmid.reboot";
8445 $res = unlink($path);
8446 die "could not remove reboot request for $vmid: $!"
8447 if !$res && $! != POSIX
::ENOENT
;
8452 sub bootorder_from_legacy
{
8453 my ($conf, $bootcfg) = @_;
8455 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
8456 my $bootindex_hash = {};
8458 foreach my $o (split(//, $boot)) {
8459 $bootindex_hash->{$o} = $i*100;
8465 PVE
::QemuConfig-
>foreach_volume($conf, sub {
8466 my ($ds, $drive) = @_;
8468 if (drive_is_cdrom
($drive, 1)) {
8469 if ($bootindex_hash->{d
}) {
8470 $bootorder->{$ds} = $bootindex_hash->{d
};
8471 $bootindex_hash->{d
} += 1;
8473 } elsif ($bootindex_hash->{c
}) {
8474 $bootorder->{$ds} = $bootindex_hash->{c
}
8475 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
8476 $bootindex_hash->{c
} += 1;
8480 if ($bootindex_hash->{n
}) {
8481 for (my $i = 0; $i < $MAX_NETS; $i++) {
8482 my $netname = "net$i";
8483 next if !$conf->{$netname};
8484 $bootorder->{$netname} = $bootindex_hash->{n
};
8485 $bootindex_hash->{n
} += 1;
8492 # Generate default device list for 'boot: order=' property. Matches legacy
8493 # default boot order, but with explicit device names. This is important, since
8494 # the fallback for when neither 'order' nor the old format is specified relies
8495 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8496 sub get_default_bootdevices
{
8502 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
8503 push @ret, $first if $first;
8506 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
8507 push @ret, $first if $first;
8510 for (my $i = 0; $i < $MAX_NETS; $i++) {
8511 my $netname = "net$i";
8512 next if !$conf->{$netname};
8513 push @ret, $netname;
8520 sub device_bootorder
{
8523 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
8525 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
8528 if (!defined($boot) || $boot->{legacy
}) {
8529 $bootorder = bootorder_from_legacy
($conf, $boot);
8530 } elsif ($boot->{order
}) {
8531 my $i = 100; # start at 100 to allow user to insert devices before us with -args
8532 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
8533 $bootorder->{$dev} = $i++;
8540 sub register_qmeventd_handle
{
8544 my $peer = "/var/run/qmeventd.sock";
8549 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
8551 if ($! != EINTR
&& $! != EAGAIN
) {
8552 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8555 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8556 . "after $count retries\n";
8561 # send handshake to mark VM as backing up
8562 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
8564 # return handle to be closed later when inhibit is no longer required
8568 # bash completion helper
8570 sub complete_backup_archives
{
8571 my ($cmdname, $pname, $cvalue) = @_;
8573 my $cfg = PVE
::Storage
::config
();
8577 if ($cvalue =~ m/^([^:]+):/) {
8581 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
8584 foreach my $id (keys %$data) {
8585 foreach my $item (@{$data->{$id}}) {
8586 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8587 push @$res, $item->{volid
} if defined($item->{volid
});
8594 my $complete_vmid_full = sub {
8597 my $idlist = vmstatus
();
8601 foreach my $id (keys %$idlist) {
8602 my $d = $idlist->{$id};
8603 if (defined($running)) {
8604 next if $d->{template
};
8605 next if $running && $d->{status
} ne 'running';
8606 next if !$running && $d->{status
} eq 'running';
8615 return &$complete_vmid_full();
8618 sub complete_vmid_stopped
{
8619 return &$complete_vmid_full(0);
8622 sub complete_vmid_running
{
8623 return &$complete_vmid_full(1);
8626 sub complete_storage
{
8628 my $cfg = PVE
::Storage
::config
();
8629 my $ids = $cfg->{ids
};
8632 foreach my $sid (keys %$ids) {
8633 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
8634 next if !$ids->{$sid}->{content
}->{images
};
8641 sub complete_migration_storage
{
8642 my ($cmd, $param, $current_value, $all_args) = @_;
8644 my $targetnode = @$all_args[1];
8646 my $cfg = PVE
::Storage
::config
();
8647 my $ids = $cfg->{ids
};
8650 foreach my $sid (keys %$ids) {
8651 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
8652 next if !$ids->{$sid}->{content
}->{images
};
8660 my ($vmid, $include_suspended) = @_;
8661 my $qmpstatus = eval {
8662 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
8663 mon_cmd
($vmid, "query-status");
8666 return $qmpstatus && (
8667 $qmpstatus->{status
} eq "paused" ||
8668 $qmpstatus->{status
} eq "prelaunch" ||
8669 ($include_suspended && $qmpstatus->{status
} eq "suspended")
8673 sub check_volume_storage_type
{
8674 my ($storecfg, $vol) = @_;
8676 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
8677 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8678 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
8680 die "storage '$storeid' does not support content-type '$vtype'\n"
8681 if !$scfg->{content
}->{$vtype};
8686 sub add_nets_bridge_fdb
{
8687 my ($conf, $vmid) = @_;
8689 for my $opt (keys %$conf) {
8690 next if $opt !~ m/^net(\d+)$/;
8691 my $iface = "tap${vmid}i$1";
8692 # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8693 my $net = parse_net
($conf->{$opt}, 1) or next;
8695 my $mac = $net->{macaddr
};
8697 log_warn
("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8698 if !file_read_firstline
("/sys/class/net/$iface/brport/learning");
8702 my $bridge = $net->{bridge
};
8704 log_warn
("Interface '$iface' not attached to any bridge.");
8708 PVE
::Network
::SDN
::Zones
::add_bridge_fdb
($iface, $mac, $bridge);
8709 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8710 PVE
::Network
::add_bridge_fdb
($iface, $mac);
8715 sub del_nets_bridge_fdb
{
8716 my ($conf, $vmid) = @_;
8718 for my $opt (keys %$conf) {
8719 next if $opt !~ m/^net(\d+)$/;
8720 my $iface = "tap${vmid}i$1";
8722 my $net = parse_net
($conf->{$opt}) or next;
8723 my $mac = $net->{macaddr
} or next;
8725 my $bridge = $net->{bridge
};
8727 PVE
::Network
::SDN
::Zones
::del_bridge_fdb
($iface, $mac, $bridge);
8728 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8729 PVE
::Network
::del_bridge_fdb
($iface, $mac);
8734 sub create_ifaces_ipams_ips
{
8735 my ($conf, $vmid) = @_;
8737 return if !$have_sdn;
8739 foreach my $opt (keys %$conf) {
8740 if ($opt =~ m/^net(\d+)$/) {
8741 my $value = $conf->{$opt};
8742 my $net = PVE
::QemuServer
::parse_net
($value);
8743 eval { PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($net->{bridge
}, $conf->{name
}, $net->{macaddr
}, $vmid, undef, 1) };
8749 sub delete_ifaces_ipams_ips
{
8750 my ($conf, $vmid) = @_;
8752 return if !$have_sdn;
8754 foreach my $opt (keys %$conf) {
8755 if ($opt =~ m/^net(\d+)$/) {
8756 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
8757 eval { PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
}) };