1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
22 use List
::Util
qw(first);
25 use Storable
qw(dclone);
26 use Time
::HiRes
qw(gettimeofday usleep);
30 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
33 use PVE
::DataCenterConfig
;
34 use PVE
::Exception
qw(raise raise_param_exc);
35 use PVE
::Format
qw(render_duration render_bytes);
36 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
37 use PVE
::Mapping
::PCI
;
38 use PVE
::Mapping
::USB
;
40 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
43 use PVE
::RESTEnvironment
qw(log_warn);
44 use PVE
::RPCEnvironment
;
48 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
52 use PVE
::QemuServer
::Helpers
qw(config_aware_timeout min_version windows_version);
53 use PVE
::QemuServer
::Cloudinit
;
54 use PVE
::QemuServer
::CGroup
;
55 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
56 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
57 use PVE
::QemuServer
::Machine
;
58 use PVE
::QemuServer
::Memory
qw(get_current_memory);
59 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
60 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
61 use PVE
::QemuServer
::QMPHelpers
qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
62 use PVE
::QemuServer
::USB
;
66 require PVE
::Network
::SDN
::Zones
;
67 require PVE
::Network
::SDN
::Vnets
;
71 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
75 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
76 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
79 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
80 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
83 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
84 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
87 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
88 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
90 # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
91 # anymore. how can we deperacate this sanely without breaking existing instances, or using
92 # older backups and snapshot?
94 "$EDK2_FW_BASE/OVMF_CODE.fd",
95 "$EDK2_FW_BASE/OVMF_VARS.fd",
100 "$EDK2_FW_BASE/AAVMF_CODE.fd",
101 "$EDK2_FW_BASE/AAVMF_VARS.fd",
106 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
108 # Note about locking: we use flock on the config file protect against concurent actions.
109 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
110 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
111 # But you can ignore this kind of lock with the --skiplock flag.
119 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
120 description
=> "Some command save/restore state from this location.",
126 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
127 description
=> "Specifies the QEMU machine type.",
129 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
134 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
137 $nodename_cache //= PVE
::INotify
::nodename
();
138 return $nodename_cache;
145 enum
=> [qw(i6300esb ib700)],
146 description
=> "Watchdog type to emulate.",
147 default => 'i6300esb',
152 enum
=> [qw(reset shutdown poweroff pause debug none)],
153 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
157 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
161 description
=> "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
166 fstrim_cloned_disks
=> {
167 description
=> "Run fstrim after moving a disk or migrating the VM.",
172 'freeze-fs-on-backup' => {
173 description
=> "Freeze/thaw guest filesystems on backup for consistency.",
179 description
=> "Select the agent type",
183 enum
=> [qw(virtio isa)],
189 description
=> "Select the VGA type.",
194 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
197 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
204 description
=> 'Enable a specific clipboard. If not set, depending on the display type the'
205 .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
216 description
=> "The size of the file in MB.",
220 pattern
=> '[a-zA-Z0-9\-]+',
222 format_description
=> 'string',
223 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
230 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
231 description
=> "Configure an audio device."
235 enum
=> ['spice', 'none'],
238 description
=> "Driver backend for the audio device."
242 my $spice_enhancements_fmt = {
247 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
251 enum
=> ['off', 'all', 'filter'],
254 description
=> "Enable video streaming. Uses compression for detected video streams."
261 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
263 description
=> "The file on the host to gather entropy from. In most cases '/dev/urandom'"
264 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
265 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
266 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
267 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
268 ." a hardware RNG from the host.",
272 description
=> "Maximum bytes of entropy allowed to get injected into the guest every"
273 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
274 ." `0` to disable limiting (potentially dangerous!).",
277 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
278 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
279 # reading from /dev/urandom
284 description
=> "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
285 ." the guest to retrieve another 'max_bytes' of entropy.",
291 my $meta_info_fmt = {
294 description
=> "The guest creation timestamp as UNIX epoch time",
300 description
=> "The QEMU (machine) version from the time this VM was created.",
301 pattern
=> '\d+(\.\d+)+',
310 description
=> "Specifies whether a VM will be started during system bootup.",
316 description
=> "Automatic restart after crash (currently ignored).",
321 type
=> 'string', format
=> 'pve-hotplug-features',
322 description
=> "Selectively enable hotplug features. This is a comma separated list of"
323 ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
324 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
325 ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
327 default => 'network,disk,usb',
332 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
338 description
=> "Lock/unlock the VM.",
339 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
344 description
=> "Limit of CPU usage.",
345 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
346 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
354 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
355 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
356 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
357 ." weights of all the other running VMs.",
360 default => 'cgroup v1: 1024, cgroup v2: 100',
365 description
=> "Memory properties.",
366 format
=> $PVE::QemuServer
::Memory
::memory_fmt
371 description
=> "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
377 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the"
378 ." more memory this VM gets. Number is relative to weights of all other running VMs."
379 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
387 description
=> "Keyboard layout for VNC server. This option is generally not required and"
388 ." is often better handled from within the guest OS.",
389 enum
=> PVE
::Tools
::kvmkeymaplist
(),
394 type
=> 'string', format
=> 'dns-name',
395 description
=> "Set a name for the VM. Only used on the configuration web interface.",
400 description
=> "SCSI controller model",
401 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
407 description
=> "Description for the VM. Shown in the web-interface VM's summary."
408 ." This is saved as comment inside the configuration file.",
409 maxLength
=> 1024 * 8,
414 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
415 description
=> "Specify guest operating system.",
416 verbose_description
=> <<EODESC,
417 Specify guest operating system. This is used to enable special
418 optimization/features for specific operating systems:
421 other;; unspecified OS
422 wxp;; Microsoft Windows XP
423 w2k;; Microsoft Windows 2000
424 w2k3;; Microsoft Windows 2003
425 w2k8;; Microsoft Windows 2008
426 wvista;; Microsoft Windows Vista
427 win7;; Microsoft Windows 7
428 win8;; Microsoft Windows 8/2012/2012r2
429 win10;; Microsoft Windows 10/2016/2019
430 win11;; Microsoft Windows 11/2022
431 l24;; Linux 2.4 Kernel
432 l26;; Linux 2.6 - 6.X Kernel
433 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
438 type
=> 'string', format
=> 'pve-qm-boot',
439 description
=> "Specify guest boot order. Use the 'order=' sub-property as usage with no"
440 ." key or 'legacy=' is deprecated.",
444 type
=> 'string', format
=> 'pve-qm-bootdisk',
445 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
446 pattern
=> '(ide|sata|scsi|virtio)\d+',
451 description
=> "The number of CPUs. Please use option -sockets instead.",
458 description
=> "The number of CPU sockets.",
465 description
=> "The number of cores per socket.",
472 description
=> "Enable/disable NUMA.",
478 description
=> "Enable/disable hugepages memory.",
479 enum
=> [qw(any 2 1024)],
485 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
486 ." after VM shutdown and can be used for subsequent starts.",
491 description
=> "Number of hotplugged vcpus.",
498 description
=> "Enable/disable ACPI.",
503 description
=> "Enable/disable communication with the QEMU Guest Agent and its properties.",
505 format
=> $agent_fmt,
510 description
=> "Enable/disable KVM hardware virtualization.",
516 description
=> "Enable/disable time drift fix.",
522 description
=> "Set the real time clock (RTC) to local time. This is enabled by default if"
523 ." the `ostype` indicates a Microsoft Windows OS.",
528 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
532 type
=> 'string', format
=> $vga_fmt,
533 description
=> "Configure the VGA hardware.",
534 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
535 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
536 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
537 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
538 ." display server. For win* OS you can select how many independent displays you want,"
539 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
540 ." using a serial device as terminal.",
544 type
=> 'string', format
=> 'pve-qm-watchdog',
545 description
=> "Create a virtual hardware watchdog device.",
546 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
547 ." action), the watchdog must be periodically polled by an agent inside the guest or"
548 ." else the watchdog will reset the guest (or execute the respective action specified)",
553 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
554 description
=> "Set the initial date of the real time clock. Valid format for date are:"
555 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
556 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
559 startup
=> get_standard_option
('pve-startup-order'),
563 description
=> "Enable/disable Template.",
569 description
=> "Arbitrary arguments passed to kvm.",
570 verbose_description
=> <<EODESCR,
571 Arbitrary arguments passed to kvm, for example:
573 args: -no-reboot -smbios 'type=0,vendor=FOO'
575 NOTE: this option is for experts only.
582 description
=> "Enable/disable the USB tablet device.",
583 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
584 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
585 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
586 ." may consider disabling this to save some context switches. This is turned off by"
587 ." default if you use spice (`qm set <vmid> --vga qxl`).",
592 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
596 migrate_downtime
=> {
599 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
605 type
=> 'string', format
=> 'pve-qm-ide',
606 typetext
=> '<volume>',
607 description
=> "This is an alias for option -ide2",
611 description
=> "Emulated CPU type.",
613 format
=> 'pve-vm-cpu-conf',
615 parent
=> get_standard_option
('pve-snapshot-name', {
617 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
621 description
=> "Timestamp for snapshots.",
627 type
=> 'string', format
=> 'pve-volume-id',
628 description
=> "Reference to a volume which stores the VM state. This is used internally"
631 vmstatestorage
=> get_standard_option
('pve-storage-id', {
632 description
=> "Default storage for VM state volumes/files.",
635 runningmachine
=> get_standard_option
('pve-qemu-machine', {
636 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
640 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
641 ." internally for snapshots.",
644 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
645 format_description
=> 'QEMU -cpu parameter'
647 machine
=> get_standard_option
('pve-qemu-machine'),
649 description
=> "Virtual processor architecture. Defaults to the host.",
652 enum
=> [qw(x86_64 aarch64)],
655 description
=> "Specify SMBIOS type 1 fields.",
656 type
=> 'string', format
=> 'pve-qm-smbios1',
663 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
664 ." remove disk operations.",
670 enum
=> [ qw(seabios ovmf) ],
671 description
=> "Select BIOS implementation.",
672 default => 'seabios',
676 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
677 format_description
=> 'UUID',
678 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
679 ." to disable explicitly.",
680 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
681 ." value identifier to the guest OS. This allows to notify the guest operating system"
682 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
683 ." execution or creation from a template). The guest operating system notices the"
684 ." change, and is then able to react as appropriate by marking its copies of"
685 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
686 ."Note that auto-creation only works when done through API/CLI create or update methods"
687 .", but not when manually editing the config file.",
688 default => "1 (autogenerated)",
693 format
=> 'pve-volume-id',
695 description
=> "Script that will be executed during various steps in the vms lifetime.",
699 format
=> $ivshmem_fmt,
700 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
706 format
=> $audio_fmt,
707 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
710 spice_enhancements
=> {
712 format
=> $spice_enhancements_fmt,
713 description
=> "Configure additional enhancements for SPICE.",
717 type
=> 'string', format
=> 'pve-tag-list',
718 description
=> 'Tags of the VM. This is only meta information.',
724 description
=> "Configure a VirtIO-based Random Number Generator.",
729 format
=> $meta_info_fmt,
730 description
=> "Some (read-only) meta-information about this guest.",
734 type
=> 'string', format
=> 'pve-cpuset',
735 description
=> "List of host cores used to execute guest processes, for example: 0,5,8-11",
744 description
=> 'Specify a custom file containing all meta data passed to the VM via"
745 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
746 format
=> 'pve-volume-id',
747 format_description
=> 'volume',
752 description
=> 'To pass a custom file containing all network data to the VM via cloud-init.',
753 format
=> 'pve-volume-id',
754 format_description
=> 'volume',
759 description
=> 'To pass a custom file containing all user data to the VM via cloud-init.',
760 format
=> 'pve-volume-id',
761 format_description
=> 'volume',
766 description
=> 'To pass a custom file containing all vendor data to the VM via cloud-init.',
767 format
=> 'pve-volume-id',
768 format_description
=> 'volume',
771 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
773 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
774 my $confdesc_cloudinit = {
778 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
779 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
780 .' and `configdrive2` for windows.',
781 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
786 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
787 ." image's configured default user.",
792 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
793 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
794 .' support hashed passwords.',
799 description
=> 'cloud-init: do an automatic package upgrade after the first boot.',
805 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
807 format
=> 'pve-qm-cicustom',
812 description
=> 'cloud-init: Sets DNS search domains for a container. Create will'
813 .' automatically use the setting from the host if neither searchdomain nor nameserver'
818 type
=> 'string', format
=> 'address-list',
819 description
=> 'cloud-init: Sets DNS server IP address for a container. Create will'
820 .' automatically use the setting from the host if neither searchdomain nor nameserver'
826 format
=> 'urlencoded',
827 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
831 # what about other qemu settings ?
833 #machine => 'string',
846 ##soundhw => 'string',
848 while (my ($k, $v) = each %$confdesc) {
849 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
853 my $MAX_SERIAL_PORTS = 4;
854 my $MAX_PARALLEL_PORTS = 3;
856 for (my $i = 0; $i < $PVE::QemuServer
::Memory
::MAX_NUMA
; $i++) {
857 $confdesc->{"numa$i"} = $PVE::QemuServer
::Memory
::numadesc
;
860 my $nic_model_list = [
876 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
878 my $net_fmt_bridge_descr = <<__EOD__;
879 Bridge to attach the network device to. The Proxmox VE standard bridge
882 If you do not specify a bridge, we create a kvm user (NATed) network
883 device, which provides DHCP and DNS services. The following addresses
890 The DHCP server assign addresses to the guest starting from 10.0.2.15.
894 macaddr
=> get_standard_option
('mac-addr', {
895 description
=> "MAC address. That address must be unique withing your network. This is"
896 ." automatically generated if not specified.",
900 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
901 ." very low CPU overhead. If your guest does not support this driver, it is usually"
902 ." best to use 'e1000'.",
903 enum
=> $nic_model_list,
906 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
907 bridge
=> get_standard_option
('pve-bridge-id', {
908 description
=> $net_fmt_bridge_descr,
913 minimum
=> 0, maximum
=> 64,
914 description
=> 'Number of packet queues to be used on the device.',
920 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
925 minimum
=> 1, maximum
=> 4094,
926 description
=> 'VLAN tag to apply to packets on this interface.',
931 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
932 description
=> 'VLAN trunks to pass through this interface.',
933 format_description
=> 'vlanid[;vlanid...]',
938 description
=> 'Whether this interface should be protected by the firewall.',
943 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
948 minimum
=> 1, maximum
=> 65520,
949 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
956 type
=> 'string', format
=> $net_fmt,
957 description
=> "Specify network devices.",
960 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
965 format
=> 'pve-ipv4-config',
966 format_description
=> 'IPv4Format/CIDR',
967 description
=> 'IPv4 address in CIDR format.',
974 format_description
=> 'GatewayIPv4',
975 description
=> 'Default gateway for IPv4 traffic.',
981 format
=> 'pve-ipv6-config',
982 format_description
=> 'IPv6Format/CIDR',
983 description
=> 'IPv6 address in CIDR format.',
990 format_description
=> 'GatewayIPv6',
991 description
=> 'Default gateway for IPv6 traffic.',
996 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
999 type
=> 'string', format
=> 'pve-qm-ipconfig',
1000 description
=> <<'EODESCR',
1001 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1003 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1005 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1006 gateway should be provided.
1007 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1008 cloud-init 19.4 or newer.
1010 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1014 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
1016 for (my $i = 0; $i < $MAX_NETS; $i++) {
1017 $confdesc->{"net$i"} = $netdesc;
1018 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1021 foreach my $key (keys %$confdesc_cloudinit) {
1022 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1025 PVE
::JSONSchema
::register_format
('pve-cpuset', \
&pve_verify_cpuset
);
1026 sub pve_verify_cpuset
{
1027 my ($set_text, $noerr) = @_;
1029 my ($count, $members) = eval { PVE
::CpuSet
::parse_cpuset
($set_text) };
1033 die "unable to parse cpuset option\n";
1036 return PVE
::CpuSet-
>new($members)->short_string();
1039 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1040 sub verify_volume_id_or_qm_path
{
1041 my ($volid, $noerr) = @_;
1043 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1045 return verify_volume_id_or_absolute_path
($volid, $noerr);
1048 PVE
::JSONSchema
::register_format
('pve-volume-id-or-absolute-path', \
&verify_volume_id_or_absolute_path
);
1049 sub verify_volume_id_or_absolute_path
{
1050 my ($volid, $noerr) = @_;
1052 return $volid if $volid =~ m
|^/|;
1054 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1065 pattern
=> '(/dev/.+|socket)',
1066 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1067 verbose_description
=> <<EODESCR,
1068 Create a serial device inside the VM (n is 0 to 3), and pass through a
1069 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1070 host side (use 'qm terminal' to open a terminal connection).
1072 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1073 use with special care.
1075 CAUTION: Experimental! User reported problems with this option.
1082 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1083 description
=> "Map host parallel devices (n is 0 to 2).",
1084 verbose_description
=> <<EODESCR,
1085 Map host parallel devices (n is 0 to 2).
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1090 CAUTION: Experimental! User reported problems with this option.
1094 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1095 $confdesc->{"parallel$i"} = $paralleldesc;
1098 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1099 $confdesc->{"serial$i"} = $serialdesc;
1102 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1103 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1106 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1107 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1110 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
1111 $confdesc->{"usb$i"} = $PVE::QemuServer
::USB
::usbdesc
;
1119 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1120 . " Deprecated, use 'order=' instead.",
1121 pattern
=> '[acdn]{1,4}',
1122 format_description
=> "[acdn]{1,4}",
1124 # note: this is also the fallback if boot: is not given at all
1130 format
=> 'pve-qm-bootdev-list',
1131 format_description
=> "device[;device...]",
1132 description
=> <<EODESC,
1133 The guest will attempt to boot from devices in the order they appear here.
1135 Disks, optical drives and passed-through storage USB devices will be directly
1136 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1137 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1139 Note that only devices in this list will be marked as bootable and thus loaded
1140 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1141 (e.g. software-raid), you need to specify all of them here.
1143 Overrides the deprecated 'legacy=[acdn]*' value when given.
1147 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1149 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1150 sub verify_bootdev
{
1151 my ($dev, $noerr) = @_;
1153 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1154 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && !$special;
1158 return 0 if $dev !~ m/^$base\d+$/;
1159 return 0 if !$confdesc->{$dev};
1163 return $dev if $check->("net");
1164 return $dev if $check->("usb");
1165 return $dev if $check->("hostpci");
1168 die "invalid boot device '$dev'\n";
1171 sub print_bootorder
{
1173 return "" if !@$devs;
1174 my $data = { order
=> join(';', @$devs) };
1175 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1178 my $kvm_api_version = 0;
1181 return $kvm_api_version if $kvm_api_version;
1183 open my $fh, '<', '/dev/kvm' or return;
1185 # 0xae00 => KVM_GET_API_VERSION
1186 $kvm_api_version = ioctl($fh, 0xae00, 0);
1189 return $kvm_api_version;
1192 my $kvm_user_version = {};
1195 sub kvm_user_version
{
1198 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1199 my $st = stat($binary);
1201 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1202 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1203 $cachedmtime == $st->mtime;
1205 $kvm_user_version->{$binary} = 'unknown';
1206 $kvm_mtime->{$binary} = $st->mtime;
1210 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1211 $kvm_user_version->{$binary} = $2;
1215 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1218 return $kvm_user_version->{$binary};
1221 my sub extract_version
{
1222 my ($machine_type, $version) = @_;
1223 $version = kvm_user_version
() if !defined($version);
1224 return PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1227 sub kernel_has_vhost_net
{
1228 return -c
'/dev/vhost-net';
1233 return defined($confdesc->{$key});
1237 sub get_cdrom_path
{
1239 return $cdrom_path if defined($cdrom_path);
1241 $cdrom_path = first
{ -l
$_ } map { "/dev/cdrom$_" } ('', '1', '2');
1243 if (!defined($cdrom_path)) {
1244 log_warn
("no physical CD-ROM available, ignoring");
1252 my ($storecfg, $vmid, $cdrom) = @_;
1254 if ($cdrom eq 'cdrom') {
1255 return get_cdrom_path
();
1256 } elsif ($cdrom eq 'none') {
1258 } elsif ($cdrom =~ m
|^/|) {
1261 return PVE
::Storage
::path
($storecfg, $cdrom);
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id
{
1267 my ($vmid, $file, $media) = @_;
1269 if (!($file eq 'none' || $file eq 'cdrom' ||
1270 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1272 return if $file =~ m
|/|;
1274 if ($media && $media eq 'cdrom') {
1275 $file = "local:iso/$file";
1277 $file = "local:$vmid/$file";
1284 sub verify_media_type
{
1285 my ($opt, $vtype, $media) = @_;
1290 if ($media eq 'disk') {
1292 } elsif ($media eq 'cdrom') {
1295 die "internal error";
1298 return if ($vtype eq $etype);
1300 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1303 sub cleanup_drive_path
{
1304 my ($opt, $storecfg, $drive) = @_;
1306 # try to convert filesystem paths to volume IDs
1308 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1309 ($drive->{file
} !~ m
|^/dev/.+|) &&
1310 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1311 ($drive->{file
} !~ m/^\d+$/)) {
1312 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1313 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1315 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1316 verify_media_type
($opt, $vtype, $drive->{media
});
1317 $drive->{file
} = $volid;
1320 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1323 sub parse_hotplug_features
{
1328 return $res if $data eq '0';
1330 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1332 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1333 if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1336 die "invalid hotplug feature '$feature'\n";
1342 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1343 sub pve_verify_hotplug_features
{
1344 my ($value, $noerr) = @_;
1346 return $value if parse_hotplug_features
($value);
1350 die "unable to parse hotplug option\n";
1353 sub assert_clipboard_config
{
1356 my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1360 && $vga->{'clipboard'} eq 'vnc'
1362 && $vga->{type
} !~ $clipboard_regex
1364 die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1368 sub print_tabletdevice_full
{
1369 my ($conf, $arch) = @_;
1371 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1373 # we use uhci for old VMs because tablet driver was buggy in older qemu
1375 if ($q35 || $arch eq 'aarch64') {
1381 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1384 sub print_keyboarddevice_full
{
1385 my ($conf, $arch) = @_;
1387 return if $arch ne 'aarch64';
1389 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1392 my sub get_drive_id
{
1394 return "$drive->{interface}$drive->{index}";
1397 sub print_drivedevice_full
{
1398 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1403 my $drive_id = get_drive_id
($drive);
1404 if ($drive->{interface
} eq 'virtio') {
1405 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1406 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1407 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1408 } elsif ($drive->{interface
} eq 'scsi') {
1410 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1411 my $unit = $drive->{index} % $maxdev;
1413 my $machine_version = extract_version
($machine_type, kvm_user_version
());
1414 my $devicetype = PVE
::QemuServer
::Drive
::get_scsi_devicetype
(
1415 $drive, $storecfg, $machine_version);
1417 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1418 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1420 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1421 .",lun=$drive->{index}";
1423 $device .= ",drive=drive-$drive_id,id=$drive_id";
1425 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1426 $device .= ",rotation_rate=1";
1428 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1430 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1431 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1432 my $controller = int($drive->{index} / $maxdev);
1433 my $unit = $drive->{index} % $maxdev;
1435 # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1436 # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1437 # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1438 # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1439 # were before. Move odd ones up by 2 where they don't clash.
1440 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) && $drive->{interface
} eq 'ide') {
1441 $controller += 2 * ($unit % 2);
1445 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1447 $device = "ide-$devicetype";
1448 if ($drive->{interface
} eq 'ide') {
1449 $device .= ",bus=ide.$controller,unit=$unit";
1451 $device .= ",bus=ahci$controller.$unit";
1453 $device .= ",drive=drive-$drive_id,id=$drive_id";
1455 if ($devicetype eq 'hd') {
1456 if (my $model = $drive->{model
}) {
1457 $model = URI
::Escape
::uri_unescape
($model);
1458 $device .= ",model=$model";
1460 if ($drive->{ssd
}) {
1461 $device .= ",rotation_rate=1";
1464 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1465 } elsif ($drive->{interface
} eq 'usb') {
1467 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1469 die "unsupported interface type";
1472 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1474 if (my $serial = $drive->{serial
}) {
1475 $serial = URI
::Escape
::uri_unescape
($serial);
1476 $device .= ",serial=$serial";
1483 sub get_initiator_name
{
1486 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1487 while (defined(my $line = <$fh>)) {
1488 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1497 my sub storage_allows_io_uring_default
{
1498 my ($scfg, $cache_direct) = @_;
1500 # io_uring with cache mode writeback or writethrough on krbd will hang...
1501 return if $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1503 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1504 # sometimes, just plain disable...
1505 return if $scfg && $scfg->{type
} eq 'lvm';
1507 # io_uring causes problems when used with CIFS since kernel 5.15
1508 # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1509 return if $scfg && $scfg->{type
} eq 'cifs';
1514 my sub drive_uses_cache_direct
{
1515 my ($drive, $scfg) = @_;
1517 my $cache_direct = 0;
1519 if (my $cache = $drive->{cache
}) {
1520 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1521 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1525 return $cache_direct;
1528 sub print_drive_commandline_full
{
1529 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1532 my $volid = $drive->{file
};
1533 my $format = $drive->{format
};
1534 my $drive_id = get_drive_id
($drive);
1536 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1537 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1539 if (drive_is_cdrom
($drive)) {
1540 $path = get_iso_path
($storecfg, $vmid, $volid);
1541 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1544 $path = PVE
::Storage
::path
($storecfg, $volid);
1545 $format //= qemu_img_format
($scfg, $volname);
1552 my $is_rbd = $path =~ m/^rbd:/;
1555 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1556 foreach my $o (@qemu_drive_options) {
1557 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1560 # snapshot only accepts on|off
1561 if (defined($drive->{snapshot
})) {
1562 my $v = $drive->{snapshot
} ?
'on' : 'off';
1563 $opts .= ",snapshot=$v";
1566 if (defined($drive->{ro
})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1567 $opts .= ",readonly=" . ($drive->{ro
} ?
'on' : 'off');
1570 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1571 my ($dir, $qmpname) = @$type;
1572 if (my $v = $drive->{"mbps$dir"}) {
1573 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1575 if (my $v = $drive->{"mbps${dir}_max"}) {
1576 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1578 if (my $v = $drive->{"bps${dir}_max_length"}) {
1579 $opts .= ",throttling.bps$qmpname-max-length=$v";
1581 if (my $v = $drive->{"iops${dir}"}) {
1582 $opts .= ",throttling.iops$qmpname=$v";
1584 if (my $v = $drive->{"iops${dir}_max"}) {
1585 $opts .= ",throttling.iops$qmpname-max=$v";
1587 if (my $v = $drive->{"iops${dir}_max_length"}) {
1588 $opts .= ",throttling.iops$qmpname-max-length=$v";
1593 $format = "rbd" if $is_rbd;
1594 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1596 $opts .= ",format=alloc-track,file.driver=$format";
1598 $opts .= ",format=$format";
1601 my $cache_direct = drive_uses_cache_direct
($drive, $scfg);
1603 $opts .= ",cache=none" if !$drive->{cache
} && $cache_direct;
1605 if (!$drive->{aio
}) {
1606 if ($io_uring && storage_allows_io_uring_default
($scfg, $cache_direct)) {
1607 # io_uring supports all cache modes
1608 $opts .= ",aio=io_uring";
1610 # aio native works only with O_DIRECT
1612 $opts .= ",aio=native";
1614 $opts .= ",aio=threads";
1619 if (!drive_is_cdrom
($drive)) {
1621 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1622 $detectzeroes = 'off';
1623 } elsif ($drive->{discard
}) {
1624 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1626 # This used to be our default with discard not being specified:
1627 $detectzeroes = 'on';
1630 # note: 'detect-zeroes' works per blockdev and we want it to persist
1631 # after the alloc-track is removed, so put it on 'file' directly
1632 my $dz_param = $pbs_name ?
"file.detect-zeroes" : "detect-zeroes";
1633 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1637 $opts .= ",backing=$pbs_name";
1638 $opts .= ",auto-remove=on";
1641 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1642 my $file_param = "file";
1644 # non-rbd drivers require the underlying file to be a seperate block
1645 # node, so add a second .file indirection
1646 $file_param .= ".file" if !$is_rbd;
1647 $file_param .= ".filename";
1649 my $pathinfo = $path ?
"$file_param=$path," : '';
1651 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1654 sub print_pbs_blockdev
{
1655 my ($pbs_conf, $pbs_name) = @_;
1656 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1657 $blockdev .= ",repository=$pbs_conf->{repository}";
1658 $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace
};
1659 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1660 $blockdev .= ",archive=$pbs_conf->{archive}";
1661 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1665 sub print_netdevice_full
{
1666 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1668 my $device = $net->{model
};
1669 if ($net->{model
} eq 'virtio') {
1670 $device = 'virtio-net-pci';
1673 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1674 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1675 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1676 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1677 # and out of each queue plus one config interrupt and control vector queue
1678 my $vectors = $net->{queues
} * 2 + 2;
1679 $tmpstr .= ",vectors=$vectors,mq=on";
1680 if (min_version
($machine_version, 7, 1)) {
1681 $tmpstr .= ",packed=on";
1685 if (min_version
($machine_version, 7, 1) && $net->{model
} eq 'virtio'){
1686 $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1689 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1691 if (my $mtu = $net->{mtu
}) {
1692 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1693 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1696 } elsif ($mtu < 576) {
1697 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1698 } elsif ($mtu > $bridge_mtu) {
1699 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1701 $tmpstr .= ",host_mtu=$mtu";
1703 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1707 if ($use_old_bios_files) {
1709 if ($device eq 'virtio-net-pci') {
1710 $romfile = 'pxe-virtio.rom';
1711 } elsif ($device eq 'e1000') {
1712 $romfile = 'pxe-e1000.rom';
1713 } elsif ($device eq 'e1000e') {
1714 $romfile = 'pxe-e1000e.rom';
1715 } elsif ($device eq 'ne2k') {
1716 $romfile = 'pxe-ne2k_pci.rom';
1717 } elsif ($device eq 'pcnet') {
1718 $romfile = 'pxe-pcnet.rom';
1719 } elsif ($device eq 'rtl8139') {
1720 $romfile = 'pxe-rtl8139.rom';
1722 $tmpstr .= ",romfile=$romfile" if $romfile;
1728 sub print_netdev_full
{
1729 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1732 if ($netid =~ m/^net(\d+)$/) {
1736 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1738 my $ifname = "tap${vmid}i$i";
1740 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1741 die "interface name '$ifname' is too long (max 15 character)\n"
1742 if length($ifname) >= 16;
1744 my $vhostparam = '';
1745 if (is_native
($arch)) {
1746 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1749 my $vmname = $conf->{name
} || "vm$vmid";
1752 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1754 if ($net->{bridge
}) {
1755 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1756 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1758 $netdev = "type=user,id=$netid,hostname=$vmname";
1761 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1767 'cirrus' => 'cirrus-vga',
1769 'vmware' => 'vmware-svga',
1770 'virtio' => 'virtio-vga',
1771 'virtio-gl' => 'virtio-vga-gl',
1774 sub print_vga_device
{
1775 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1777 my $type = $vga_map->{$vga->{type
}};
1778 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1779 $type = 'virtio-gpu';
1781 my $vgamem_mb = $vga->{memory
};
1783 my $max_outputs = '';
1785 $type = $id ?
'qxl' : 'qxl-vga';
1787 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1788 # set max outputs so linux can have up to 4 qxl displays with one device
1789 if (min_version
($machine_version, 4, 1)) {
1790 $max_outputs = ",max_outputs=4";
1795 die "no devicetype for $vga->{type}\n" if !$type;
1799 if ($vga->{type
} =~ /^virtio/) {
1800 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1801 $memory = ",max_hostmem=$bytes";
1803 # from https://www.spice-space.org/multiple-monitors.html
1804 $memory = ",vgamem_mb=$vga->{memory}";
1805 my $ram = $vgamem_mb * 4;
1806 my $vram = $vgamem_mb * 2;
1807 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1809 $memory = ",vgamem_mb=$vga->{memory}";
1811 } elsif ($qxlnum && $id) {
1812 $memory = ",ram_size=67108864,vram_size=33554432";
1816 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1817 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1820 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1821 my $vgaid = "vga" . ($id // '');
1823 if ($q35 && $vgaid eq 'vga') {
1824 # the first display uses pcie.0 bus on q35 machines
1825 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1827 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1830 if ($vga->{type
} eq 'virtio-gl') {
1831 my $base = '/usr/lib/x86_64-linux-gnu/lib';
1832 die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1833 if !-e
"${base}EGL.so.1" || !-e
"${base}GL.so.1";
1835 die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1836 if !PVE
::Tools
::dir_glob_regex
('/dev/dri/', "renderD.*");
1839 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1842 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1844 my ($data, $disable_mac_autogen) = @_;
1846 my $res = eval { parse_property_string
($net_fmt, $data) };
1851 if (!defined($res->{macaddr
}) && !$disable_mac_autogen) {
1852 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1853 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1858 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1859 sub parse_ipconfig
{
1862 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1868 if ($res->{gw
} && !$res->{ip
}) {
1869 warn 'gateway specified without specifying an IP address';
1872 if ($res->{gw6
} && !$res->{ip6
}) {
1873 warn 'IPv6 gateway specified without specifying an IPv6 address';
1876 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1877 warn 'gateway specified together with DHCP';
1880 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1882 warn "IPv6 gateway specified together with $res->{ip6} address";
1886 if (!$res->{ip
} && !$res->{ip6
}) {
1887 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1896 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1899 sub add_random_macs
{
1900 my ($settings) = @_;
1902 foreach my $opt (keys %$settings) {
1903 next if $opt !~ m/^net(\d+)$/;
1904 my $net = parse_net
($settings->{$opt});
1906 $settings->{$opt} = print_net
($net);
1910 sub vm_is_volid_owner
{
1911 my ($storecfg, $vmid, $volid) = @_;
1913 if ($volid !~ m
|^/|) {
1915 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1916 if ($owner && ($owner == $vmid)) {
1924 sub vmconfig_register_unused_drive
{
1925 my ($storecfg, $vmid, $conf, $drive) = @_;
1927 if (drive_is_cloudinit
($drive)) {
1928 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1930 delete $conf->{cloudinit
};
1931 } elsif (!drive_is_cdrom
($drive)) {
1932 my $volid = $drive->{file
};
1933 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1934 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1939 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1943 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1944 format_description
=> 'UUID',
1945 description
=> "Set SMBIOS1 UUID.",
1950 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1951 format_description
=> 'Base64 encoded string',
1952 description
=> "Set SMBIOS1 version.",
1957 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1958 format_description
=> 'Base64 encoded string',
1959 description
=> "Set SMBIOS1 serial number.",
1964 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1965 format_description
=> 'Base64 encoded string',
1966 description
=> "Set SMBIOS1 manufacturer.",
1971 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1972 format_description
=> 'Base64 encoded string',
1973 description
=> "Set SMBIOS1 product ID.",
1978 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1979 format_description
=> 'Base64 encoded string',
1980 description
=> "Set SMBIOS1 SKU string.",
1985 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1986 format_description
=> 'Base64 encoded string',
1987 description
=> "Set SMBIOS1 family string.",
1992 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2000 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2007 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2010 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2012 sub parse_watchdog
{
2017 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2022 sub parse_guest_agent
{
2025 return {} if !defined($conf->{agent
});
2027 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2030 # if the agent is disabled ignore the other potentially set properties
2031 return {} if !$res->{enabled
};
2036 my ($conf, $key) = @_;
2037 return undef if !defined($conf->{agent
});
2039 my $agent = parse_guest_agent
($conf);
2040 return $agent->{$key};
2046 return {} if !$value;
2047 my $res = eval { parse_property_string
($vga_fmt, $value) };
2057 my $res = eval { parse_property_string
($rng_fmt, $value) };
2062 sub parse_meta_info
{
2067 my $res = eval { parse_property_string
($meta_info_fmt, $value) };
2072 sub new_meta_info_string
{
2073 my () = @_; # for now do not allow to override any value
2075 return PVE
::JSONSchema
::print_property_string
(
2077 'creation-qemu' => kvm_user_version
(),
2078 ctime
=> "". int(time()),
2084 sub qemu_created_version_fixups
{
2085 my ($conf, $forcemachine, $kvmver) = @_;
2087 my $meta = parse_meta_info
($conf->{meta
}) // {};
2088 my $forced_vers = PVE
::QemuServer
::Machine
::extract_version
($forcemachine);
2090 # check if we need to apply some handling for VMs that always use the latest machine version but
2091 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2092 # would be required (we do not want to pin machine version for non-windows OS type)
2094 (!defined($conf->{machine
}) || $conf->{machine
} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2095 && (!defined($meta->{'creation-qemu'}) || !min_version
($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2096 && (!$forced_vers || min_version
($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2097 && min_version
($kvmver, 6, 1) # only need to apply the change since 6.1
2099 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2100 if ($q35 && $conf->{ostype
} && $conf->{ostype
} eq 'l26') {
2101 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2102 # and thus with the predictable interface naming of systemd
2103 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2109 # add JSON properties for create and set function
2110 sub json_config_properties
{
2111 my ($prop, $with_disk_alloc) = @_;
2113 my $skip_json_config_opts = {
2117 runningmachine
=> 1,
2122 foreach my $opt (keys %$confdesc) {
2123 next if $skip_json_config_opts->{$opt};
2125 if ($with_disk_alloc && is_valid_drivename
($opt)) {
2126 $prop->{$opt} = $PVE::QemuServer
::Drive
::drivedesc_hash_with_alloc-
>{$opt};
2128 $prop->{$opt} = $confdesc->{$opt};
2135 # Properties that we can read from an OVF file
2136 sub json_ovf_properties
{
2139 for my $device (PVE
::QemuServer
::Drive
::valid_drive_names
()) {
2140 $prop->{$device} = {
2142 format
=> 'pve-volume-id-or-absolute-path',
2143 description
=> "Disk image that gets imported to $device",
2150 description
=> "The number of CPU cores.",
2155 description
=> "Amount of RAM for the VM in MB.",
2160 description
=> "Name of the VM.",
2167 # return copy of $confdesc_cloudinit to generate documentation
2168 sub cloudinit_config_properties
{
2170 return dclone
($confdesc_cloudinit);
2173 sub cloudinit_pending_properties
{
2175 map { $_ => 1 } keys $confdesc_cloudinit->%*,
2178 $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2183 my ($key, $value) = @_;
2185 die "unknown setting '$key'\n" if !$confdesc->{$key};
2187 my $type = $confdesc->{$key}->{type
};
2189 if (!defined($value)) {
2190 die "got undefined value\n";
2193 if ($value =~ m/[\n\r]/) {
2194 die "property contains a line feed\n";
2197 if ($type eq 'boolean') {
2198 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2199 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2200 die "type check ('boolean') failed - got '$value'\n";
2201 } elsif ($type eq 'integer') {
2202 return int($1) if $value =~ m/^(\d+)$/;
2203 die "type check ('integer') failed - got '$value'\n";
2204 } elsif ($type eq 'number') {
2205 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2206 die "type check ('number') failed - got '$value'\n";
2207 } elsif ($type eq 'string') {
2208 if (my $fmt = $confdesc->{$key}->{format
}) {
2209 PVE
::JSONSchema
::check_format
($fmt, $value);
2212 $value =~ s/^\"(.*)\"$/$1/;
2215 die "internal error"
2220 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2222 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2224 if (!$skiplock && !PVE
::QemuConfig-
>has_lock($conf, 'suspended')) {
2225 PVE
::QemuConfig-
>check_lock($conf);
2228 if ($conf->{template
}) {
2229 # check if any base image is still used by a linked clone
2230 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2231 my ($ds, $drive) = @_;
2232 return if drive_is_cdrom
($drive);
2234 my $volid = $drive->{file
};
2235 return if !$volid || $volid =~ m
|^/|;
2237 die "base volume '$volid' is still in use by linked cloned\n"
2238 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2244 my $remove_owned_drive = sub {
2245 my ($ds, $drive) = @_;
2246 return if drive_is_cdrom
($drive, 1);
2248 my $volid = $drive->{file
};
2249 return if !$volid || $volid =~ m
|^/|;
2250 return if $volids->{$volid};
2252 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2253 return if !$path || !$owner || ($owner != $vmid);
2255 $volids->{$volid} = 1;
2256 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2257 warn "Could not remove disk '$volid', check manually: $@" if $@;
2260 # only remove disks owned by this VM (referenced in the config)
2261 my $include_opts = {
2262 include_unused
=> 1,
2263 extra_keys
=> ['vmstate'],
2265 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2267 for my $snap (values %{$conf->{snapshots
}}) {
2268 next if !defined($snap->{vmstate
});
2269 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2270 next if !defined($drive);
2271 $remove_owned_drive->('vmstate', $drive);
2274 PVE
::QemuConfig-
>foreach_volume_full($conf->{pending
}, $include_opts, $remove_owned_drive);
2276 if ($purge_unreferenced) { # also remove unreferenced disk
2277 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2278 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2279 my ($volid, $sid, $volname, $d) = @_;
2280 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2285 eval { delete_ifaces_ipams_ips
($conf, $vmid)};
2288 if (defined $replacement_conf) {
2289 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2291 PVE
::QemuConfig-
>destroy_config($vmid);
2295 sub parse_vm_config
{
2296 my ($filename, $raw, $strict) = @_;
2298 return if !defined($raw);
2301 digest
=> Digest
::SHA
::sha1_hex
($raw),
2307 my $handle_error = sub {
2317 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2318 || die "got strange filename '$filename'";
2324 my $finish_description = sub {
2325 if (defined($descr)) {
2327 $conf->{description
} = $descr;
2333 my @lines = split(/\n/, $raw);
2334 foreach my $line (@lines) {
2335 next if $line =~ m/^\s*$/;
2337 if ($line =~ m/^\[PENDING\]\s*$/i) {
2338 $section = 'pending';
2339 $finish_description->();
2340 $conf = $res->{$section} = {};
2342 } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2343 $section = 'cloudinit';
2344 $finish_description->();
2345 $conf = $res->{$section} = {};
2348 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2350 $finish_description->();
2351 $conf = $res->{snapshots
}->{$section} = {};
2355 if ($line =~ m/^\#(.*)$/) {
2356 $descr = '' if !defined($descr);
2357 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2361 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2362 $descr = '' if !defined($descr);
2363 $descr .= PVE
::Tools
::decode_text
($2);
2364 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2365 $conf->{snapstate
} = $1;
2366 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2369 $conf->{$key} = $value;
2370 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2372 if ($section eq 'pending') {
2373 $conf->{delete} = $value; # we parse this later
2375 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2377 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2380 if ($section eq 'cloudinit') {
2381 # ignore validation only used for informative purpose
2382 $conf->{$key} = $value;
2385 eval { $value = check_type
($key, $value); };
2387 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2389 $key = 'ide2' if $key eq 'cdrom';
2390 my $fmt = $confdesc->{$key}->{format
};
2391 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2392 my $v = parse_drive
($key, $value);
2393 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2394 $v->{file
} = $volid;
2395 $value = print_drive
($v);
2397 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2402 $conf->{$key} = $value;
2405 $handle_error->("vm $vmid - unable to parse config: $line\n");
2409 $finish_description->();
2410 delete $res->{snapstate
}; # just to be sure
2415 sub write_vm_config
{
2416 my ($filename, $conf) = @_;
2418 delete $conf->{snapstate
}; # just to be sure
2420 if ($conf->{cdrom
}) {
2421 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2422 $conf->{ide2
} = $conf->{cdrom
};
2423 delete $conf->{cdrom
};
2426 # we do not use 'smp' any longer
2427 if ($conf->{sockets
}) {
2428 delete $conf->{smp
};
2429 } elsif ($conf->{smp
}) {
2430 $conf->{sockets
} = $conf->{smp
};
2431 delete $conf->{cores
};
2432 delete $conf->{smp
};
2435 my $used_volids = {};
2437 my $cleanup_config = sub {
2438 my ($cref, $pending, $snapname) = @_;
2440 foreach my $key (keys %$cref) {
2441 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2442 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2443 my $value = $cref->{$key};
2444 if ($key eq 'delete') {
2445 die "propertry 'delete' is only allowed in [PENDING]\n"
2447 # fixme: check syntax?
2450 eval { $value = check_type
($key, $value); };
2451 die "unable to parse value of '$key' - $@" if $@;
2453 $cref->{$key} = $value;
2455 if (!$snapname && is_valid_drivename
($key)) {
2456 my $drive = parse_drive
($key, $value);
2457 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2462 &$cleanup_config($conf);
2464 &$cleanup_config($conf->{pending
}, 1);
2466 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2467 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2468 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2471 # remove 'unusedX' settings if we re-add a volume
2472 foreach my $key (keys %$conf) {
2473 my $value = $conf->{$key};
2474 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2475 delete $conf->{$key};
2479 my $generate_raw_config = sub {
2480 my ($conf, $pending) = @_;
2484 # add description as comment to top of file
2485 if (defined(my $descr = $conf->{description
})) {
2487 foreach my $cl (split(/\n/, $descr)) {
2488 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2491 $raw .= "#\n" if $pending;
2495 foreach my $key (sort keys %$conf) {
2496 next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2497 $raw .= "$key: $conf->{$key}\n";
2502 my $raw = &$generate_raw_config($conf);
2504 if (scalar(keys %{$conf->{pending
}})){
2505 $raw .= "\n[PENDING]\n";
2506 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2509 if (scalar(keys %{$conf->{cloudinit
}}) && PVE
::QemuConfig-
>has_cloudinit($conf)){
2510 $raw .= "\n[special:cloudinit]\n";
2511 $raw .= &$generate_raw_config($conf->{cloudinit
});
2514 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2515 $raw .= "\n[$snapname]\n";
2516 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2526 # we use static defaults from our JSON schema configuration
2527 foreach my $key (keys %$confdesc) {
2528 if (defined(my $default = $confdesc->{$key}->{default})) {
2529 $res->{$key} = $default;
2537 my $vmlist = PVE
::Cluster
::get_vmlist
();
2539 return $res if !$vmlist || !$vmlist->{ids
};
2540 my $ids = $vmlist->{ids
};
2541 my $nodename = nodename
();
2543 foreach my $vmid (keys %$ids) {
2544 my $d = $ids->{$vmid};
2545 next if !$d->{node
} || $d->{node
} ne $nodename;
2546 next if !$d->{type
} || $d->{type
} ne 'qemu';
2547 $res->{$vmid}->{exists} = 1;
2552 # test if VM uses local resources (to prevent migration)
2553 sub check_local_resources
{
2554 my ($conf, $noerr) = @_;
2557 my $mapped_res = [];
2559 my $nodelist = PVE
::Cluster
::get_nodelist
();
2560 my $pci_map = PVE
::Mapping
::PCI
::config
();
2561 my $usb_map = PVE
::Mapping
::USB
::config
();
2563 my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2565 my $add_missing_mapping = sub {
2566 my ($type, $key, $id) = @_;
2567 for my $node (@$nodelist) {
2569 if ($type eq 'pci') {
2570 $entry = PVE
::Mapping
::PCI
::get_node_mapping
($pci_map, $id, $node);
2571 } elsif ($type eq 'usb') {
2572 $entry = PVE
::Mapping
::USB
::get_node_mapping
($usb_map, $id, $node);
2574 if (!scalar($entry->@*)) {
2575 push @{$missing_mappings_by_node->{$node}}, $key;
2580 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2581 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2583 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2585 foreach my $k (keys %$conf) {
2586 if ($k =~ m/^usb/) {
2587 my $entry = parse_property_string
('pve-qm-usb', $conf->{$k});
2588 next if $entry->{host
} =~ m/^spice$/i;
2589 if ($entry->{mapping
}) {
2590 $add_missing_mapping->('usb', $k, $entry->{mapping
});
2591 push @$mapped_res, $k;
2594 if ($k =~ m/^hostpci/) {
2595 my $entry = parse_property_string
('pve-qm-hostpci', $conf->{$k});
2596 if ($entry->{mapping
}) {
2597 $add_missing_mapping->('pci', $k, $entry->{mapping
});
2598 push @$mapped_res, $k;
2601 # sockets are safe: they will recreated be on the target side post-migrate
2602 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2603 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2606 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2608 return wantarray ?
(\
@loc_res, $mapped_res, $missing_mappings_by_node) : \
@loc_res;
2611 # check if used storages are available on all nodes (use by migrate)
2612 sub check_storage_availability
{
2613 my ($storecfg, $conf, $node) = @_;
2615 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2616 my ($ds, $drive) = @_;
2618 my $volid = $drive->{file
};
2621 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2624 # check if storage is available on both nodes
2625 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2626 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2628 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2630 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2631 if !$scfg->{content
}->{$vtype};
2635 # list nodes where all VM images are available (used by has_feature API)
2637 my ($conf, $storecfg) = @_;
2639 my $nodelist = PVE
::Cluster
::get_nodelist
();
2640 my $nodehash = { map { $_ => 1 } @$nodelist };
2641 my $nodename = nodename
();
2643 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2644 my ($ds, $drive) = @_;
2646 my $volid = $drive->{file
};
2649 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2651 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2652 if ($scfg->{disable
}) {
2654 } elsif (my $avail = $scfg->{nodes
}) {
2655 foreach my $node (keys %$nodehash) {
2656 delete $nodehash->{$node} if !$avail->{$node};
2658 } elsif (!$scfg->{shared
}) {
2659 foreach my $node (keys %$nodehash) {
2660 delete $nodehash->{$node} if $node ne $nodename
2669 sub check_local_storage_availability
{
2670 my ($conf, $storecfg) = @_;
2672 my $nodelist = PVE
::Cluster
::get_nodelist
();
2673 my $nodehash = { map { $_ => {} } @$nodelist };
2675 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2676 my ($ds, $drive) = @_;
2678 my $volid = $drive->{file
};
2681 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2683 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2685 if ($scfg->{disable
}) {
2686 foreach my $node (keys %$nodehash) {
2687 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2689 } elsif (my $avail = $scfg->{nodes
}) {
2690 foreach my $node (keys %$nodehash) {
2691 if (!$avail->{$node}) {
2692 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2699 foreach my $node (values %$nodehash) {
2700 if (my $unavail = $node->{unavailable_storages
}) {
2701 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2708 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2710 my ($vmid, $nocheck, $node) = @_;
2712 # $nocheck is set when called during a migration, in which case the config
2713 # file might still or already reside on the *other* node
2714 # - because rename has already happened, and current node is source
2715 # - because rename hasn't happened yet, and current node is target
2716 # - because rename has happened, current node is target, but hasn't yet
2718 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2719 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2724 my $vzlist = config_list
();
2726 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2728 while (defined(my $de = $fd->read)) {
2729 next if $de !~ m/^(\d+)\.pid$/;
2731 next if !defined($vzlist->{$vmid});
2732 if (my $pid = check_running
($vmid)) {
2733 $vzlist->{$vmid}->{pid
} = $pid;
2740 our $vmstatus_return_properties = {
2741 vmid
=> get_standard_option
('pve-vmid'),
2743 description
=> "QEMU process status.",
2745 enum
=> ['stopped', 'running'],
2748 description
=> "Maximum memory in bytes.",
2751 renderer
=> 'bytes',
2754 description
=> "Root disk size in bytes.",
2757 renderer
=> 'bytes',
2760 description
=> "VM name.",
2765 description
=> "VM run state from the 'query-status' QMP monitor command.",
2770 description
=> "PID of running qemu process.",
2775 description
=> "Uptime.",
2778 renderer
=> 'duration',
2781 description
=> "Maximum usable CPUs.",
2786 description
=> "The current config lock, if any.",
2791 description
=> "The current configured tags, if any",
2795 'running-machine' => {
2796 description
=> "The currently running machine type (if running).",
2801 description
=> "The currently running QEMU version (if running).",
2807 my $last_proc_pid_stat;
2809 # get VM status information
2810 # This must be fast and should not block ($full == false)
2811 # We only query KVM using QMP if $full == true (this can be slow)
2813 my ($opt_vmid, $full) = @_;
2817 my $storecfg = PVE
::Storage
::config
();
2819 my $list = vzlist
();
2820 my $defaults = load_defaults
();
2822 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2824 my $cpucount = $cpuinfo->{cpus
} || 1;
2826 foreach my $vmid (keys %$list) {
2827 next if $opt_vmid && ($vmid ne $opt_vmid);
2829 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2831 my $d = { vmid
=> int($vmid) };
2832 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2834 # fixme: better status?
2835 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2837 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2838 if (defined($size)) {
2839 $d->{disk
} = 0; # no info available
2840 $d->{maxdisk
} = $size;
2846 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2847 * ($conf->{cores
} || $defaults->{cores
});
2848 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2849 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2851 $d->{name
} = $conf->{name
} || "VM $vmid";
2852 $d->{maxmem
} = get_current_memory
($conf->{memory
})*(1024*1024);
2854 if ($conf->{balloon
}) {
2855 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2856 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2857 : $defaults->{shares
};
2868 $d->{diskwrite
} = 0;
2870 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2872 $d->{serial
} = 1 if conf_has_serial
($conf);
2873 $d->{lock} = $conf->{lock} if $conf->{lock};
2874 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2879 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2880 foreach my $dev (keys %$netdev) {
2881 next if $dev !~ m/^tap([1-9]\d*)i/;
2883 my $d = $res->{$vmid};
2886 $d->{netout
} += $netdev->{$dev}->{receive
};
2887 $d->{netin
} += $netdev->{$dev}->{transmit
};
2890 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2891 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2896 my $ctime = gettimeofday
;
2898 foreach my $vmid (keys %$list) {
2900 my $d = $res->{$vmid};
2901 my $pid = $d->{pid
};
2904 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2905 next if !$pstat; # not running
2907 my $used = $pstat->{utime} + $pstat->{stime
};
2909 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2911 if ($pstat->{vsize
}) {
2912 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2915 my $old = $last_proc_pid_stat->{$pid};
2917 $last_proc_pid_stat->{$pid} = {
2925 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2927 if ($dtime > 1000) {
2928 my $dutime = $used - $old->{used
};
2930 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2931 $last_proc_pid_stat->{$pid} = {
2937 $d->{cpu
} = $old->{cpu
};
2941 return $res if !$full;
2943 my $qmpclient = PVE
::QMPClient-
>new();
2945 my $ballooncb = sub {
2946 my ($vmid, $resp) = @_;
2948 my $info = $resp->{'return'};
2949 return if !$info->{max_mem
};
2951 my $d = $res->{$vmid};
2953 # use memory assigned to VM
2954 $d->{maxmem
} = $info->{max_mem
};
2955 $d->{balloon
} = $info->{actual
};
2957 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2958 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2959 $d->{freemem
} = $info->{free_mem
};
2962 $d->{ballooninfo
} = $info;
2965 my $blockstatscb = sub {
2966 my ($vmid, $resp) = @_;
2967 my $data = $resp->{'return'} || [];
2968 my $totalrdbytes = 0;
2969 my $totalwrbytes = 0;
2971 for my $blockstat (@$data) {
2972 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2973 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2975 $blockstat->{device
} =~ s/drive-//;
2976 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2978 $res->{$vmid}->{diskread
} = $totalrdbytes;
2979 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2982 my $machinecb = sub {
2983 my ($vmid, $resp) = @_;
2984 my $data = $resp->{'return'} || [];
2986 $res->{$vmid}->{'running-machine'} =
2987 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
2990 my $versioncb = sub {
2991 my ($vmid, $resp) = @_;
2992 my $data = $resp->{'return'} // {};
2993 my $version = 'unknown';
2995 if (my $v = $data->{qemu
}) {
2996 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
2999 $res->{$vmid}->{'running-qemu'} = $version;
3002 my $statuscb = sub {
3003 my ($vmid, $resp) = @_;
3005 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3006 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3007 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3008 # this fails if ballon driver is not loaded, so this must be
3009 # the last commnand (following command are aborted if this fails).
3010 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3012 my $status = 'unknown';
3013 if (!defined($status = $resp->{'return'}->{status
})) {
3014 warn "unable to get VM status\n";
3018 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3021 foreach my $vmid (keys %$list) {
3022 next if $opt_vmid && ($vmid ne $opt_vmid);
3023 next if !$res->{$vmid}->{pid
}; # not running
3024 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3027 $qmpclient->queue_execute(undef, 2);
3029 foreach my $vmid (keys %$list) {
3030 next if $opt_vmid && ($vmid ne $opt_vmid);
3031 next if !$res->{$vmid}->{pid
}; #not running
3033 # we can't use the $qmpclient since it might have already aborted on
3034 # 'query-balloon', but this might also fail for older versions...
3035 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
3036 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3039 foreach my $vmid (keys %$list) {
3040 next if $opt_vmid && ($vmid ne $opt_vmid);
3041 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3047 sub conf_has_serial
{
3050 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3051 if ($conf->{"serial$i"}) {
3059 sub conf_has_audio
{
3060 my ($conf, $id) = @_;
3063 my $audio = $conf->{"audio$id"};
3064 return if !defined($audio);
3066 my $audioproperties = parse_property_string
($audio_fmt, $audio);
3067 my $audiodriver = $audioproperties->{driver
} // 'spice';
3070 dev
=> $audioproperties->{device
},
3071 dev_id
=> "audiodev$id",
3072 backend
=> $audiodriver,
3073 backend_id
=> "$audiodriver-backend${id}",
3078 my ($audio, $audiopciaddr, $machine_version) = @_;
3082 my $id = $audio->{dev_id
};
3084 if (min_version
($machine_version, 4, 2)) {
3085 $audiodev = ",audiodev=$audio->{backend_id}";
3088 if ($audio->{dev
} eq 'AC97') {
3089 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3090 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3091 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3092 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3093 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3095 die "unkown audio device '$audio->{dev}', implement me!";
3098 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3106 socket => "/var/run/qemu-server/$vmid.swtpm",
3107 pid
=> "/var/run/qemu-server/$vmid.swtpm.pid",
3111 sub add_tpm_device
{
3112 my ($vmid, $devices, $conf) = @_;
3114 return if !$conf->{tpmstate0
};
3116 my $paths = get_tpm_paths
($vmid);
3118 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3119 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3120 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3124 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3126 return if !$tpmdrive;
3129 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
3130 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
3132 $state = PVE
::Storage
::map_volume
($storecfg, $tpm->{file
});
3134 $state = $tpm->{file
};
3137 my $paths = get_tpm_paths
($vmid);
3139 # during migration, we will get state from remote
3142 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3149 "--create-platform-cert",
3152 "/etc/swtpm_setup.conf", # do not use XDG configs
3154 "0", # force creation as root, error if not possible
3155 "--not-overwrite", # ignore existing state, do not modify
3158 push @$setup_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3159 # TPM 2.0 supports ECC crypto, use if possible
3160 push @$setup_cmd, "--ecc" if $tpm->{version
} eq 'v2.0';
3162 run_command
($setup_cmd, outfunc
=> sub {
3163 print "swtpm_setup: $1\n";
3167 # Used to distinguish different invocations in the log.
3168 my $log_prefix = "[id=" . int(time()) . "] ";
3170 my $emulator_cmd = [
3174 "backend-uri=file://$state,mode=0600",
3176 "type=unixio,path=$paths->{socket},mode=0600",
3178 "file=$paths->{pid}",
3179 "--terminate", # terminate on QEMU disconnect
3182 "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3184 push @$emulator_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3185 run_command
($emulator_cmd, outfunc
=> sub { print $1; });
3187 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3188 while (! -e
$paths->{pid
}) {
3189 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3193 # return untainted PID of swtpm daemon so it can be killed on error
3194 file_read_firstline
($paths->{pid
}) =~ m/(\d+)/;
3198 sub vga_conf_has_spice
{
3201 my $vgaconf = parse_vga
($vga);
3202 my $vgatype = $vgaconf->{type
};
3203 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3210 return get_host_arch
() eq $arch;
3215 return $conf->{arch
} // get_host_arch
();
3218 my $default_machines = {
3223 sub get_installed_machine_version
{
3224 my ($kvmversion) = @_;
3225 $kvmversion = kvm_user_version
() if !defined($kvmversion);
3226 $kvmversion =~ m/^(\d+\.\d+)/;
3230 sub windows_get_pinned_machine_version
{
3231 my ($machine, $base_version, $kvmversion) = @_;
3233 my $pin_version = $base_version;
3234 if (!defined($base_version) ||
3235 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3237 $pin_version = get_installed_machine_version
($kvmversion);
3239 if (!$machine || $machine eq 'pc') {
3240 $machine = "pc-i440fx-$pin_version";
3241 } elsif ($machine eq 'q35') {
3242 $machine = "pc-q35-$pin_version";
3243 } elsif ($machine eq 'virt') {
3244 $machine = "virt-$pin_version";
3246 warn "unknown machine type '$machine', not touching that!\n";
3252 sub get_vm_machine
{
3253 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3255 my $machine = $forcemachine || $conf->{machine
};
3257 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3258 $kvmversion //= kvm_user_version
();
3259 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3260 # layout which confuses windows quite a bit and may result in various regressions..
3261 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3262 if (windows_version
($conf->{ostype
})) {
3263 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3266 $machine ||= $default_machines->{$arch};
3267 if ($add_pve_version) {
3268 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3269 $machine .= "+pve$pvever";
3273 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3274 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3275 $machine = $1 if $is_pxe;
3277 # for version-pinned machines that do not include a pve-version (e.g.
3278 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3279 $machine .= '+pve0';
3281 $machine .= '.pxe' if $is_pxe;
3287 sub get_ovmf_files
($$$) {
3288 my ($arch, $efidisk, $smm) = @_;
3290 my $types = $OVMF->{$arch}
3291 or die "no OVMF images known for architecture '$arch'\n";
3293 my $type = 'default';
3294 if ($arch eq 'x86_64') {
3295 if (defined($efidisk->{efitype
}) && $efidisk->{efitype
} eq '4m') {
3296 $type = $smm ?
"4m" : "4m-no-smm";
3297 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3299 # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3303 my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3304 die "EFI base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3305 die "EFI vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
3307 return ($ovmf_code, $ovmf_vars);
3311 aarch64
=> '/usr/bin/qemu-system-aarch64',
3312 x86_64
=> '/usr/bin/qemu-system-x86_64',
3314 sub get_command_for_arch
($) {
3316 return '/usr/bin/kvm' if is_native
($arch);
3318 my $cmd = $Arch2Qemu->{$arch}
3319 or die "don't know how to emulate architecture '$arch'\n";
3323 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3324 # to use in a QEMU command line (-cpu element), first array_intersect the result
3325 # of query_supported_ with query_understood_. This is necessary because:
3327 # a) query_understood_ returns flags the host cannot use and
3328 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3329 # flags, but CPU settings - with most of them being flags. Those settings
3330 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3332 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3333 # expensive. If you need the value returned from this, you can get it much
3334 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3335 # $accel being 'kvm' or 'tcg'.
3337 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3338 # changes, automatically populating pmxcfs.
3340 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3341 # since kvm and tcg machines support different flags
3343 sub query_supported_cpu_flags
{
3346 $arch //= get_host_arch
();
3347 my $default_machine = $default_machines->{$arch};
3351 # FIXME: Once this is merged, the code below should work for ARM as well:
3352 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3353 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3356 my $kvm_supported = defined(kvm_version
());
3357 my $qemu_cmd = get_command_for_arch
($arch);
3359 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3361 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3362 my $query_supported_run_qemu = sub {
3368 '-machine', $default_machine,
3370 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3371 '-mon', 'chardev=qmp,mode=control',
3372 '-pidfile', $pidfile,
3377 push @$cmd, '-accel', 'tcg';
3380 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3381 die "QEMU flag querying VM exited with code " . $rc if $rc;
3384 my $cmd_result = mon_cmd
(
3386 'query-cpu-model-expansion',
3388 model
=> { name
=> 'host' }
3391 my $props = $cmd_result->{model
}->{props
};
3392 foreach my $prop (keys %$props) {
3393 next if $props->{$prop} ne '1';
3394 # QEMU returns some flags multiple times, with '_', '.' or '-'
3395 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3396 # We only keep those with underscores, to match /proc/cpuinfo
3397 $prop =~ s/\.|-/_/g;
3398 $flags->{$prop} = 1;
3403 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3404 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3408 return [ sort keys %$flags ];
3411 # We need to query QEMU twice, since KVM and TCG have different supported flags
3412 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3413 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3414 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3416 if ($kvm_supported) {
3417 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3418 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3425 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3426 my $understood_cpu_flag_dir = "/usr/share/kvm";
3427 sub query_understood_cpu_flags
{
3428 my $arch = get_host_arch
();
3429 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3431 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3434 my $raw = file_get_contents
($filepath);
3435 $raw =~ s/^\s+|\s+$//g;
3436 my @flags = split(/\s+/, $raw);
3441 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3442 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3443 my sub should_disable_smm
{
3444 my ($conf, $vga, $machine) = @_;
3446 return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3448 return (!defined($conf->{bios
}) || $conf->{bios
} eq 'seabios') &&
3449 $vga->{type
} && $vga->{type
} =~ m/^(serial\d+|none)$/;
3452 my sub print_ovmf_drive_commandlines
{
3453 my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3455 my $d = $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
3457 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch, $d, $q35);
3459 my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3461 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3462 my ($path, $format) = $d->@{'file', 'format'};
3464 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3465 if (!defined($format)) {
3466 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3467 $format = qemu_img_format
($scfg, $volname);
3469 } elsif (!defined($format)) {
3470 die "efidisk format must be specified\n";
3472 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3473 if ($path =~ m/^rbd:/) {
3474 $var_drive_str .= ',cache=writeback';
3475 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3477 $var_drive_str .= ",format=$format,file=$path";
3479 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3480 $var_drive_str .= ',readonly=on' if drive_is_read_only
($conf, $d);
3482 log_warn
("no efidisk configured! Using temporary efivars disk.");
3483 my $path = "/tmp/$vmid-ovmf.fd";
3484 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3485 $var_drive_str .= ",format=raw,file=$path";
3486 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $version_guard->(4, 1, 2);
3489 return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3492 sub config_to_command
{
3493 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3496 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3499 my $ostype = $conf->{ostype
};
3500 my $winversion = windows_version
($ostype);
3501 my $kvm = $conf->{kvm
};
3502 my $nodename = nodename
();
3504 my $arch = get_vm_arch
($conf);
3505 my $kvm_binary = get_command_for_arch
($arch);
3506 my $kvmver = kvm_user_version
($kvm_binary);
3508 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3509 $kvmver //= "undefined";
3510 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3513 my $add_pve_version = min_version
($kvmver, 4, 1);
3515 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3516 my $machine_version = extract_version
($machine_type, $kvmver);
3517 $kvm //= 1 if is_native
($arch);
3519 $machine_version =~ m/(\d+)\.(\d+)/;
3520 my ($machine_major, $machine_minor) = ($1, $2);
3522 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3523 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3524 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3525 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3526 ." please upgrade node '$nodename'\n"
3527 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3528 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3529 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3530 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3531 ." node '$nodename'\n";
3534 # if a specific +pve version is required for a feature, use $version_guard
3535 # instead of min_version to allow machines to be run with the minimum
3537 my $required_pve_version = 0;
3538 my $version_guard = sub {
3539 my ($major, $minor, $pve) = @_;
3540 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3541 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3542 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3543 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3547 if ($kvm && !defined kvm_version
()) {
3548 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3549 ." or enable in BIOS.\n";
3552 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3553 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3554 my $use_old_bios_files = undef;
3555 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3558 if ($conf->{affinity
}) {
3559 push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity
};
3562 push @$cmd, $kvm_binary;
3564 push @$cmd, '-id', $vmid;
3566 my $vmname = $conf->{name
} || "vm$vmid";
3568 push @$cmd, '-name', "$vmname,debug-threads=on";
3570 push @$cmd, '-no-shutdown';
3574 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3575 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3576 push @$cmd, '-mon', "chardev=qmp,mode=control";
3578 if (min_version
($machine_version, 2, 12)) {
3579 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3580 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3583 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3585 push @$cmd, '-daemonize';
3587 if ($conf->{smbios1
}) {
3588 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3589 if ($smbios_conf->{base64
}) {
3590 # Do not pass base64 flag to qemu
3591 delete $smbios_conf->{base64
};
3592 my $smbios_string = "";
3593 foreach my $key (keys %$smbios_conf) {
3595 if ($key eq "uuid") {
3596 $value = $smbios_conf->{uuid
}
3598 $value = decode_base64
($smbios_conf->{$key});
3600 # qemu accepts any binary data, only commas need escaping by double comma
3602 $smbios_string .= "," . $key . "=" . $value if $value;
3604 push @$cmd, '-smbios', "type=1" . $smbios_string;
3606 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3610 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3611 my ($code_drive_str, $var_drive_str) =
3612 print_ovmf_drive_commandlines
($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3613 push $cmd->@*, '-drive', $code_drive_str;
3614 push $cmd->@*, '-drive', $var_drive_str;
3617 if ($q35) { # tell QEMU to load q35 config early
3618 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3619 if (min_version
($machine_version, 4, 0)) {
3620 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3622 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3626 if (defined(my $fixups = qemu_created_version_fixups
($conf, $forcemachine, $kvmver))) {
3627 push @$cmd, $fixups->@*;
3630 if ($conf->{vmgenid
}) {
3631 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3634 # add usb controllers
3635 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3636 $conf, $bridges, $arch, $machine_type, $machine_version);
3637 push @$devices, @usbcontrollers if @usbcontrollers;
3638 my $vga = parse_vga
($conf->{vga
});
3640 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3641 $vga->{type
} = 'qxl' if $qxlnum;
3643 if (!$vga->{type
}) {
3644 if ($arch eq 'aarch64') {
3645 $vga->{type
} = 'virtio';
3646 } elsif (min_version
($machine_version, 2, 9)) {
3647 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3649 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3653 # enable absolute mouse coordinates (needed by vnc)
3654 my $tablet = $conf->{tablet
};
3655 if (!defined($tablet)) {
3656 $tablet = $defaults->{tablet
};
3657 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3658 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3662 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3663 my $kbd = print_keyboarddevice_full
($conf, $arch);
3664 push @$devices, '-device', $kbd if defined($kbd);
3667 my $bootorder = device_bootorder
($conf);
3669 # host pci device passthrough
3670 my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3671 $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3674 my $usb_dev_features = {};
3675 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3677 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3678 $conf, $usb_dev_features, $bootorder, $machine_version);
3679 push @$devices, @usbdevices if @usbdevices;
3682 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3683 my $path = $conf->{"serial$i"} or next;
3684 if ($path eq 'socket') {
3685 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3686 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3687 # On aarch64, serial0 is the UART device. QEMU only allows
3688 # connecting UART devices via the '-serial' command line, as
3689 # the device has a fixed slot on the hardware...
3690 if ($arch eq 'aarch64' && $i == 0) {
3691 push @$devices, '-serial', "chardev:serial$i";
3693 push @$devices, '-device', "isa-serial,chardev=serial$i";
3696 die "no such serial device\n" if ! -c
$path;
3697 push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3698 push @$devices, '-device', "isa-serial,chardev=serial$i";
3703 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3704 if (my $path = $conf->{"parallel$i"}) {
3705 die "no such parallel device\n" if ! -c
$path;
3706 my $devtype = $path =~ m!^/dev/usb/lp! ?
'serial' : 'parallel';
3707 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3708 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3712 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3713 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3714 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3715 push @$devices, @$audio_devs;
3718 # Add a TPM only if the VM is not a template,
3719 # to support backing up template VMs even if the TPM disk is write-protected.
3720 add_tpm_device
($vmid, $devices, $conf) if (!PVE
::QemuConfig-
>is_template($conf));
3723 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3724 $sockets = $conf->{sockets
} if $conf->{sockets
};
3726 my $cores = $conf->{cores
} || 1;
3728 my $maxcpus = $sockets * $cores;
3730 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3732 my $allowed_vcpus = $cpuinfo->{cpus
};
3734 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3736 if ($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3737 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3738 for (my $i = 2; $i <= $vcpus; $i++) {
3739 my $cpustr = print_cpu_device
($conf,$i);
3740 push @$cmd, '-device', $cpustr;
3745 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3747 push @$cmd, '-nodefaults';
3749 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3751 push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3753 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3755 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3756 push @$devices, '-device', print_vga_device
(
3757 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3759 push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type
} eq 'virtio-gl'; # VIRGL
3761 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3762 push @$cmd, '-vnc', "unix:$socket,password=on";
3764 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3765 push @$cmd, '-nographic';
3769 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3770 my $useLocaltime = $conf->{localtime};
3772 if ($winversion >= 5) { # windows
3773 $useLocaltime = 1 if !defined($conf->{localtime});
3775 # use time drift fix when acpi is enabled
3776 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3777 $tdf = 1 if !defined($conf->{tdf
});
3781 if ($winversion >= 6) {
3782 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3783 push @$machineFlags, 'hpet=off';
3786 push @$rtcFlags, 'driftfix=slew' if $tdf;
3788 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3789 push @$rtcFlags, "base=$conf->{startdate}";
3790 } elsif ($useLocaltime) {
3791 push @$rtcFlags, 'base=localtime';
3795 push @$cmd, '-cpu', $forcecpu;
3797 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3800 PVE
::QemuServer
::Memory
::config
(
3801 $conf, $vmid, $sockets, $cores, $hotplug_features->{memory
}, $cmd);
3803 push @$cmd, '-S' if $conf->{freeze
};
3805 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3807 my $guest_agent = parse_guest_agent
($conf);
3809 if ($guest_agent->{enabled
}) {
3810 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3811 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3813 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3814 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3815 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3816 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3817 } elsif ($guest_agent->{type
} eq 'isa') {
3818 push @$devices, '-device', "isa-serial,chardev=qga0";
3822 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3823 if ($rng && $version_guard->(4, 1, 2)) {
3824 check_rng_source
($rng->{source
});
3826 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3827 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3828 my $limiter_str = "";
3830 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3833 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3834 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3835 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3840 assert_clipboard_config
($vga);
3841 my $is_spice = $qxlnum || $vga->{type
} =~ /^virtio/;
3843 if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3846 for (my $i = 1; $i < $qxlnum; $i++){
3847 push @$devices, '-device', print_vga_device
(
3848 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3851 # assume other OS works like Linux
3852 my ($ram, $vram) = ("134217728", "67108864");
3853 if ($vga->{memory
}) {
3854 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3855 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3857 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3858 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3862 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3864 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3865 if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3866 push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3868 push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3870 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3873 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3874 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3875 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3877 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3878 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3880 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3881 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3882 if ($spice_enhancement->{foldersharing
}) {
3883 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3884 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3887 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3888 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3889 if $spice_enhancement->{videostreaming
};
3890 push @$devices, '-spice', "$spice_opts";
3894 # enable balloon by default, unless explicitly disabled
3895 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3896 my $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3897 my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3898 $ballooncmd .= ",free-page-reporting=on" if min_version
($machine_version, 6, 2);
3899 push @$devices, '-device', $ballooncmd;
3902 if ($conf->{watchdog
}) {
3903 my $wdopts = parse_watchdog
($conf->{watchdog
});
3904 my $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3905 my $watchdog = $wdopts->{model
} || 'i6300esb';
3906 push @$devices, '-device', "$watchdog$pciaddr";
3907 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3911 my $scsicontroller = {};
3912 my $ahcicontroller = {};
3913 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3915 # Add iscsi initiator name if available
3916 if (my $initiator = get_initiator_name
()) {
3917 push @$devices, '-iscsi', "initiator-name=$initiator";
3920 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3921 my ($ds, $drive) = @_;
3923 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3924 check_volume_storage_type
($storecfg, $drive->{file
});
3925 push @$vollist, $drive->{file
};
3928 # ignore efidisk here, already added in bios/fw handling code above
3929 return if $drive->{interface
} eq 'efidisk';
3931 return if $drive->{interface
} eq 'tpmstate';
3933 $use_virtio = 1 if $ds =~ m/^virtio/;
3935 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3937 if ($drive->{interface
} eq 'virtio'){
3938 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3941 if ($drive->{interface
} eq 'scsi') {
3943 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3945 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3946 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3948 my $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3949 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3952 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3953 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3954 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3955 } elsif ($drive->{iothread
}) {
3957 "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3962 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3963 $queues = ",num_queues=$drive->{queues}";
3966 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3967 if !$scsicontroller->{$controller};
3968 $scsicontroller->{$controller}=1;
3971 if ($drive->{interface
} eq 'sata') {
3972 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3973 my $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3974 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3975 if !$ahcicontroller->{$controller};
3976 $ahcicontroller->{$controller}=1;
3979 my $pbs_conf = $pbs_backing->{$ds};
3980 my $pbs_name = undef;
3982 $pbs_name = "drive-$ds-pbs";
3983 push @$devices, '-blockdev', print_pbs_blockdev
($pbs_conf, $pbs_name);
3986 my $drive_cmd = print_drive_commandline_full
(
3987 $storecfg, $vmid, $drive, $pbs_name, min_version
($kvmver, 6, 0));
3989 # extra protection for templates, but SATA and IDE don't support it..
3990 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
3992 push @$devices, '-drive',$drive_cmd;
3993 push @$devices, '-device', print_drivedevice_full
(
3994 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3997 for (my $i = 0; $i < $MAX_NETS; $i++) {
3998 my $netname = "net$i";
4000 next if !$conf->{$netname};
4001 my $d = parse_net
($conf->{$netname});
4003 # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4005 $use_virtio = 1 if $d->{model
} eq 'virtio';
4007 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
4009 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
4010 push @$devices, '-netdev', $netdevfull;
4012 my $netdevicefull = print_netdevice_full
(
4013 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4015 push @$devices, '-device', $netdevicefull;
4018 if ($conf->{ivshmem
}) {
4019 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4023 $bus = print_pcie_addr
("ivshmem");
4025 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4028 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4029 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4031 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4032 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4033 .",size=$ivshmem->{size}M";
4036 # pci.4 is nested in pci.1
4037 $bridges->{1} = 1 if $bridges->{4};
4039 if (!$q35) { # add pci bridges
4040 if (min_version
($machine_version, 2, 3)) {
4044 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4047 for my $k (sort {$b cmp $a} keys %$bridges) {
4048 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4051 if ($k == 2 && $legacy_igd) {
4054 my $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
4055 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4057 if ($q35) { # add after -readconfig pve-q35.cfg
4058 splice @$devices, 2, 0, '-device', $devstr;
4060 unshift @$devices, '-device', $devstr if $k > 0;
4065 push @$machineFlags, 'accel=tcg';
4068 push @$machineFlags, 'smm=off' if should_disable_smm
($conf, $vga, $machine_type);
4070 my $machine_type_min = $machine_type;
4071 if ($add_pve_version) {
4072 $machine_type_min =~ s/\+pve\d+$//;
4073 $machine_type_min .= "+pve$required_pve_version";
4075 push @$machineFlags, "type=${machine_type_min}";
4077 push @$cmd, @$devices;
4078 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4079 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4080 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4082 if (my $vmstate = $conf->{vmstate
}) {
4083 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4084 push @$vollist, $vmstate;
4085 push @$cmd, '-loadstate', $statepath;
4086 print "activating and using '$vmstate' as vmstate\n";
4089 if (PVE
::QemuConfig-
>is_template($conf)) {
4090 # needed to workaround base volumes being read-only
4091 push @$cmd, '-snapshot';
4095 if ($conf->{args
}) {
4096 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4100 return wantarray ?
($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4103 sub check_rng_source
{
4106 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4107 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4110 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4111 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
4112 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4113 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4114 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4115 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4123 my $res = mon_cmd
($vmid, 'query-spice');
4125 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4128 sub vm_devices_list
{
4131 my $res = mon_cmd
($vmid, 'query-pci');
4132 my $devices_to_check = [];
4134 foreach my $pcibus (@$res) {
4135 push @$devices_to_check, @{$pcibus->{devices
}},
4138 while (@$devices_to_check) {
4140 for my $d (@$devices_to_check) {
4141 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4142 next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices
};
4144 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4145 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4147 $devices_to_check = $to_check;
4150 my $resblock = mon_cmd
($vmid, 'query-block');
4151 foreach my $block (@$resblock) {
4152 if($block->{device
} =~ m/^drive-(\S+)/){
4157 my $resmice = mon_cmd
($vmid, 'query-mice');
4158 foreach my $mice (@$resmice) {
4159 if ($mice->{name
} eq 'QEMU HID Tablet') {
4160 $devices->{tablet
} = 1;
4165 # for usb devices there is no query-usb
4166 # but we can iterate over the entries in
4167 # qom-list path=/machine/peripheral
4168 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4169 foreach my $per (@$resperipheral) {
4170 if ($per->{name
} =~ m/^usb(?:redirdev)?\d+$/) {
4171 $devices->{$per->{name
}} = 1;
4179 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4181 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4183 my $devices_list = vm_devices_list
($vmid);
4184 return 1 if defined($devices_list->{$deviceid});
4186 # add PCI bridge if we need it for the device
4187 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4189 if ($deviceid eq 'tablet') {
4190 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4191 } elsif ($deviceid eq 'keyboard') {
4192 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4193 } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4195 qemu_spice_usbredir_chardev_add
($vmid, "usbredirchardev$id");
4196 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_spice_usbdevice
($id, "xhci", $id + 1));
4197 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4198 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device, {}, $1 + 1));
4199 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4200 qemu_iothread_add
($vmid, $deviceid, $device);
4202 qemu_driveadd
($storecfg, $vmid, $device);
4203 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4205 qemu_deviceadd
($vmid, $devicefull);
4206 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4208 eval { qemu_drivedel
($vmid, $deviceid); };
4212 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4213 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4214 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4215 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4217 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4219 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4220 qemu_iothread_add
($vmid, $deviceid, $device);
4221 $devicefull .= ",iothread=iothread-$deviceid";
4224 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4225 $devicefull .= ",num_queues=$device->{queues}";
4228 qemu_deviceadd
($vmid, $devicefull);
4229 qemu_deviceaddverify
($vmid, $deviceid);
4230 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4231 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4232 qemu_driveadd
($storecfg, $vmid, $device);
4234 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4235 eval { qemu_deviceadd
($vmid, $devicefull); };
4237 eval { qemu_drivedel
($vmid, $deviceid); };
4241 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4242 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4244 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4245 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type);
4246 my $use_old_bios_files = undef;
4247 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4249 my $netdevicefull = print_netdevice_full
(
4250 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4251 qemu_deviceadd
($vmid, $netdevicefull);
4253 qemu_deviceaddverify
($vmid, $deviceid);
4254 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4257 eval { qemu_netdevdel
($vmid, $deviceid); };
4261 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4263 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4264 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4266 qemu_deviceadd
($vmid, $devicefull);
4267 qemu_deviceaddverify
($vmid, $deviceid);
4269 die "can't hotplug device '$deviceid'\n";
4275 # fixme: this should raise exceptions on error!
4276 sub vm_deviceunplug
{
4277 my ($vmid, $conf, $deviceid) = @_;
4279 my $devices_list = vm_devices_list
($vmid);
4280 return 1 if !defined($devices_list->{$deviceid});
4282 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4283 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4285 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4286 qemu_devicedel
($vmid, $deviceid);
4287 } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4288 qemu_devicedel
($vmid, $deviceid);
4289 qemu_devicedelverify
($vmid, $deviceid);
4290 } elsif ($deviceid =~ m/^usb\d+$/) {
4291 qemu_devicedel
($vmid, $deviceid);
4292 qemu_devicedelverify
($vmid, $deviceid);
4293 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4294 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4296 qemu_devicedel
($vmid, $deviceid);
4297 qemu_devicedelverify
($vmid, $deviceid);
4298 qemu_drivedel
($vmid, $deviceid);
4299 qemu_iothread_del
($vmid, $deviceid, $device);
4300 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4301 qemu_devicedel
($vmid, $deviceid);
4302 qemu_devicedelverify
($vmid, $deviceid);
4303 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4304 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4306 qemu_devicedel
($vmid, $deviceid);
4307 qemu_devicedelverify
($vmid, $deviceid);
4308 qemu_drivedel
($vmid, $deviceid);
4309 qemu_deletescsihw
($conf, $vmid, $deviceid);
4311 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4312 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4313 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4314 qemu_devicedel
($vmid, $deviceid);
4315 qemu_devicedelverify
($vmid, $deviceid);
4316 qemu_netdevdel
($vmid, $deviceid);
4318 die "can't unplug device '$deviceid'\n";
4324 sub qemu_spice_usbredir_chardev_add
{
4325 my ($vmid, $id) = @_;
4327 mon_cmd
($vmid, "chardev-add" , (
4338 sub qemu_iothread_add
{
4339 my ($vmid, $deviceid, $device) = @_;
4341 if ($device->{iothread
}) {
4342 my $iothreads = vm_iothreads_list
($vmid);
4343 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4347 sub qemu_iothread_del
{
4348 my ($vmid, $deviceid, $device) = @_;
4350 if ($device->{iothread
}) {
4351 my $iothreads = vm_iothreads_list
($vmid);
4352 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4357 my ($storecfg, $vmid, $device) = @_;
4359 my $kvmver = get_running_qemu_version
($vmid);
4360 my $io_uring = min_version
($kvmver, 6, 0);
4361 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4362 $drive =~ s/\\/\\\\/g;
4363 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4365 # If the command succeeds qemu prints: "OK
"
4366 return 1 if $ret =~ m/OK/s;
4368 die "adding drive failed
: $ret\n";
4372 my ($vmid, $deviceid) = @_;
4374 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4377 return 1 if $ret eq "";
4379 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4380 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4382 die "deleting drive
$deviceid failed
: $ret\n";
4385 sub qemu_deviceaddverify {
4386 my ($vmid, $deviceid) = @_;
4388 for (my $i = 0; $i <= 5; $i++) {
4389 my $devices_list = vm_devices_list($vmid);
4390 return 1 if defined($devices_list->{$deviceid});
4394 die "error on hotplug device
'$deviceid'\n";
4398 sub qemu_devicedelverify {
4399 my ($vmid, $deviceid) = @_;
4401 # need to verify that the device is correctly removed as device_del
4402 # is async and empty return is not reliable
4404 for (my $i = 0; $i <= 5; $i++) {
4405 my $devices_list = vm_devices_list($vmid);
4406 return 1 if !defined($devices_list->{$deviceid});
4410 die "error on hot-unplugging device
'$deviceid'\n";
4413 sub qemu_findorcreatescsihw {
4414 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4416 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4418 my $scsihwid="$controller_prefix$controller";
4419 my $devices_list = vm_devices_list($vmid);
4421 if (!defined($devices_list->{$scsihwid})) {
4422 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4428 sub qemu_deletescsihw {
4429 my ($conf, $vmid, $opt) = @_;
4431 my $device = parse_drive($opt, $conf->{$opt});
4433 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4434 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4438 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4440 my $devices_list = vm_devices_list($vmid);
4441 foreach my $opt (keys %{$devices_list}) {
4442 if (is_valid_drivename($opt)) {
4443 my $drive = parse_drive($opt, $conf->{$opt});
4444 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4450 my $scsihwid="scsihw
$controller";
4452 vm_deviceunplug($vmid, $conf, $scsihwid);
4457 sub qemu_add_pci_bridge {
4458 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4464 print_pci_addr($device, $bridges, $arch, $machine_type);
4466 while (my ($k, $v) = each %$bridges) {
4469 return 1 if !defined($bridgeid) || $bridgeid < 1;
4471 my $bridge = "pci
.$bridgeid";
4472 my $devices_list = vm_devices_list($vmid);
4474 if (!defined($devices_list->{$bridge})) {
4475 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4481 sub qemu_set_link_status {
4482 my ($vmid, $device, $up) = @_;
4484 mon_cmd($vmid, "set_link
", name => $device,
4485 up => $up ? JSON::true : JSON::false);
4488 sub qemu_netdevadd {
4489 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4491 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4492 my %options = split(/[=,]/, $netdev);
4494 if (defined(my $vhost = $options{vhost})) {
4495 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4498 if (defined(my $queues = $options{queues})) {
4499 $options{queues} = $queues + 0;
4502 mon_cmd($vmid, "netdev_add
", %options);
4506 sub qemu_netdevdel {
4507 my ($vmid, $deviceid) = @_;
4509 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4512 sub qemu_usb_hotplug {
4513 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4517 # remove the old one first
4518 vm_deviceunplug($vmid, $conf, $deviceid);
4520 # check if xhci controller is necessary and available
4521 my $devicelist = vm_devices_list($vmid);
4523 if (!$devicelist->{xhci}) {
4524 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4525 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4529 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4532 sub qemu_cpu_hotplug {
4533 my ($vmid, $conf, $vcpus) = @_;
4535 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4538 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4539 $sockets = $conf->{sockets} if $conf->{sockets};
4540 my $cores = $conf->{cores} || 1;
4541 my $maxcpus = $sockets * $cores;
4543 $vcpus = $maxcpus if !$vcpus;
4545 die "you can
't add more vcpus than maxcpus\n"
4546 if $vcpus > $maxcpus;
4548 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4550 if ($vcpus < $currentvcpus) {
4552 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4554 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4555 qemu_devicedel($vmid, "cpu$i");
4557 my $currentrunningvcpus = undef;
4559 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4560 last if scalar(@{$currentrunningvcpus}) == $i-1;
4561 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4565 #update conf after each succesfull cpu unplug
4566 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4567 PVE::QemuConfig->write_config($vmid, $conf);
4570 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4576 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4577 die "vcpus in running vm does not match its configuration\n"
4578 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4580 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4582 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4583 my $cpustr = print_cpu_device($conf, $i);
4584 qemu_deviceadd($vmid, $cpustr);
4587 my $currentrunningvcpus = undef;
4589 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4590 last if scalar(@{$currentrunningvcpus}) == $i;
4591 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4595 #update conf after each succesfull cpu hotplug
4596 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4597 PVE::QemuConfig->write_config($vmid, $conf);
4601 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4602 mon_cmd($vmid, "cpu-add", id => int($i));
4607 sub qemu_block_set_io_throttle {
4608 my ($vmid, $deviceid,
4609 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4610 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4611 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4612 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4614 return if !check_running($vmid) ;
4616 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4618 bps_rd => int($bps_rd),
4619 bps_wr => int($bps_wr),
4621 iops_rd => int($iops_rd),
4622 iops_wr => int($iops_wr),
4623 bps_max => int($bps_max),
4624 bps_rd_max => int($bps_rd_max),
4625 bps_wr_max => int($bps_wr_max),
4626 iops_max => int($iops_max),
4627 iops_rd_max => int($iops_rd_max),
4628 iops_wr_max => int($iops_wr_max),
4629 bps_max_length => int($bps_max_length),
4630 bps_rd_max_length => int($bps_rd_max_length),
4631 bps_wr_max_length => int($bps_wr_max_length),
4632 iops_max_length => int($iops_max_length),
4633 iops_rd_max_length => int($iops_rd_max_length),
4634 iops_wr_max_length => int($iops_wr_max_length),
4639 sub qemu_block_resize {
4640 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4642 my $running = check_running($vmid);
4644 PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4646 return if !$running;
4648 my $padding = (1024 - $size % 1024) % 1024;
4649 $size = $size + $padding;
4654 device => $deviceid,
4660 sub qemu_volume_snapshot {
4661 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4663 my $running = check_running($vmid);
4665 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4666 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4668 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4672 sub qemu_volume_snapshot_delete {
4673 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4675 my $running = check_running($vmid);
4680 my $conf = PVE::QemuConfig->load_config($vmid);
4681 PVE::QemuConfig->foreach_volume($conf, sub {
4682 my ($ds, $drive) = @_;
4683 $running = 1 if $drive->{file} eq $volid;
4687 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4688 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4690 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4694 sub set_migration_caps {
4695 my ($vmid, $savevm) = @_;
4697 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4699 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4700 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4705 "auto-converge" => 1,
4707 "x-rdma-pin-all" => 0,
4710 "dirty-bitmaps" => $dirty_bitmaps,
4713 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4715 for my $supported_capability (@$supported_capabilities) {
4717 capability => $supported_capability->{capability},
4718 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4722 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4726 my ($conf, $func, @param) = @_;
4730 my $test_volid = sub {
4731 my ($key, $drive, $snapname, $pending) = @_;
4733 my $volid = $drive->{file};
4736 $volhash->{$volid}->{cdrom} //= 1;
4737 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4739 my $replicate = $drive->{replicate} // 1;
4740 $volhash->{$volid}->{replicate} //= 0;
4741 $volhash->{$volid}->{replicate} = 1 if $replicate;
4743 $volhash->{$volid}->{shared} //= 0;
4744 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4746 $volhash->{$volid}->{is_unused} //= 0;
4747 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4749 $volhash->{$volid}->{is_attached} //= 0;
4750 $volhash->{$volid}->{is_attached} = 1
4751 if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4753 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4754 if defined($snapname);
4756 $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4758 my $size = $drive->{size};
4759 $volhash->{$volid}->{size} //= $size if $size;
4761 $volhash->{$volid}->{is_vmstate} //= 0;
4762 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4764 $volhash->{$volid}->{is_tpmstate} //= 0;
4765 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0
';
4767 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4770 my $include_opts = {
4771 extra_keys => ['vmstate
'],
4772 include_unused => 1,
4775 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4777 PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4778 if defined($conf->{pending}) && $conf->{pending}->%*;
4780 foreach my $snapname (keys %{$conf->{snapshots}}) {
4781 my $snap = $conf->{snapshots}->{$snapname};
4782 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4785 foreach my $volid (keys %$volhash) {
4786 &$func($volid, $volhash->{$volid}, @param);
4790 my $fast_plug_option = {
4794 'migrate_downtime
' => 1,
4795 'migrate_speed
' => 1,
4802 'vmstatestorage
' => 1,
4805 for my $opt (keys %$confdesc_cloudinit) {
4806 $fast_plug_option->{$opt} = 1;
4809 # hotplug changes in [PENDING]
4810 # $selection hash can be used to only apply specified options, for
4811 # example: { cores => 1 } (only apply changed 'cores
')
4812 # $errors ref is used to return error messages
4813 sub vmconfig_hotplug_pending {
4814 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4816 my $defaults = load_defaults();
4817 my $arch = get_vm_arch($conf);
4818 my $machine_type = get_vm_machine($conf, undef, $arch);
4820 # commit values which do not have any impact on running VM first
4821 # Note: those option cannot raise errors, we we do not care about
4822 # $selection and always apply them.
4824 my $add_error = sub {
4825 my ($opt, $msg) = @_;
4826 $errors->{$opt} = "hotplug problem - $msg";
4829 my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4831 my $cloudinit_record_changed = sub {
4832 my ($conf, $opt, $old, $new) = @_;
4833 return if !$cloudinit_pending_properties->{$opt};
4835 my $ci = ($conf->{cloudinit} //= {});
4837 my $recorded = $ci->{$opt};
4838 my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4840 if (defined($new)) {
4841 if (defined($old)) {
4842 # an existing value is being modified
4843 if (defined($recorded)) {
4844 # the value was already not in sync
4845 if ($new eq $recorded) {
4846 # a value is being reverted to the cloud-init state:
4848 delete $added{$opt};
4850 # the value was changed multiple times, do nothing
4852 } elsif ($added{$opt}) {
4853 # the value had been marked as added and is being changed, do nothing
4855 # the value is new, record it:
4859 # a new value is being added
4860 if (defined($recorded)) {
4861 # it was already not in sync
4862 if ($new eq $recorded) {
4863 # a value is being reverted to the cloud-init state:
4865 delete $added{$opt};
4867 # the value had temporarily been removed, do nothing
4869 } elsif ($added{$opt}) {
4870 # the value had been marked as added already, do nothing
4872 # the value is new, add it
4876 } elsif (!defined($old)) {
4877 # a non-existent value is being removed? ignore...
4879 # a value is being deleted
4880 if (defined($recorded)) {
4881 # a value was already recorded, just keep it
4882 } elsif ($added{$opt}) {
4883 # the value was marked as added, remove it
4884 delete $added{$opt};
4886 # a previously unrecorded value is being removed, record the old value:
4891 my $added = join(',', sort keys %added);
4892 $ci->{added} = $added if length($added);
4896 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4897 if ($fast_plug_option->{$opt}) {
4898 my $new = delete $conf->{pending}->{$opt};
4899 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4900 $conf->{$opt} = $new;
4906 PVE::QemuConfig->write_config($vmid, $conf);
4909 my $ostype = $conf->{ostype};
4910 my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4911 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4912 my $usb_hotplug = $hotplug_features->{usb}
4913 && min_version($version, 7, 1)
4914 && defined($ostype) && ($ostype eq 'l26
' || windows_version($ostype) > 7);
4916 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4917 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4919 foreach my $opt (sort keys %$pending_delete_hash) {
4920 next if $selection && !$selection->{$opt};
4921 my $force = $pending_delete_hash->{$opt}->{force};
4923 if ($opt eq 'hotplug
') {
4924 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4925 } elsif ($opt eq 'tablet
') {
4926 die "skip\n" if !$hotplug_features->{usb};
4927 if ($defaults->{tablet}) {
4928 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4929 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4930 if $arch eq 'aarch64
';
4932 vm_deviceunplug($vmid, $conf, 'tablet
');
4933 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4935 } elsif ($opt =~ m/^usb(\d+)$/) {
4937 die "skip\n" if !$usb_hotplug;
4938 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4939 vm_deviceunplug
($vmid, $conf, $opt);
4940 } elsif ($opt eq 'vcpus') {
4941 die "skip\n" if !$hotplug_features->{cpu
};
4942 qemu_cpu_hotplug
($vmid, $conf, undef);
4943 } elsif ($opt eq 'balloon') {
4944 # enable balloon device is not hotpluggable
4945 die "skip\n" if defined($conf->{balloon
}) && $conf->{balloon
} == 0;
4946 # here we reset the ballooning value to memory
4947 my $balloon = get_current_memory
($conf->{memory
});
4948 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
4949 } elsif ($fast_plug_option->{$opt}) {
4951 } elsif ($opt =~ m/^net(\d+)$/) {
4952 die "skip\n" if !$hotplug_features->{network
};
4953 vm_deviceunplug
($vmid, $conf, $opt);
4955 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
4956 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
});
4958 } elsif (is_valid_drivename
($opt)) {
4959 die "skip\n" if !$hotplug_features->{disk
} || $opt =~ m/(ide|sata)(\d+)/;
4960 vm_deviceunplug
($vmid, $conf, $opt);
4961 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
4962 } elsif ($opt =~ m/^memory$/) {
4963 die "skip\n" if !$hotplug_features->{memory
};
4964 PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf);
4965 } elsif ($opt eq 'cpuunits') {
4966 $cgroup->change_cpu_shares(undef);
4967 } elsif ($opt eq 'cpulimit') {
4968 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4974 &$add_error($opt, $err) if $err ne "skip\n";
4976 my $old = delete $conf->{$opt};
4977 $cloudinit_record_changed->($conf, $opt, $old, undef);
4978 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
4983 foreach my $opt (keys %{$conf->{pending
}}) {
4984 next if $selection && !$selection->{$opt};
4985 my $value = $conf->{pending
}->{$opt};
4987 if ($opt eq 'hotplug') {
4988 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug
} =~ /memory/);
4989 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug
} =~ /cpu/);
4990 } elsif ($opt eq 'tablet') {
4991 die "skip\n" if !$hotplug_features->{usb
};
4993 vm_deviceplug
($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4994 vm_deviceplug
($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4995 if $arch eq 'aarch64';
4996 } elsif ($value == 0) {
4997 vm_deviceunplug
($vmid, $conf, 'tablet');
4998 vm_deviceunplug
($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5000 } elsif ($opt =~ m/^usb(\d+)$/) {
5002 die "skip\n" if !$usb_hotplug;
5003 my $d = eval { parse_property_string
('pve-qm-usb', $value) };
5005 if ($d->{host
} =~ m/^spice$/i) {
5006 $id = "usbredirdev$index";
5008 qemu_usb_hotplug
($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5009 } elsif ($opt eq 'vcpus') {
5010 die "skip\n" if !$hotplug_features->{cpu
};
5011 qemu_cpu_hotplug
($vmid, $conf, $value);
5012 } elsif ($opt eq 'balloon') {
5013 # enable/disable balloning device is not hotpluggable
5014 my $old_balloon_enabled = !!(!defined($conf->{balloon
}) || $conf->{balloon
});
5015 my $new_balloon_enabled = !!(!defined($conf->{pending
}->{balloon
}) || $conf->{pending
}->{balloon
});
5016 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5018 # allow manual ballooning if shares is set to zero
5019 if ((defined($conf->{shares
}) && ($conf->{shares
} == 0))) {
5020 my $memory = get_current_memory
($conf->{memory
});
5021 my $balloon = $conf->{pending
}->{balloon
} || $memory;
5022 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5024 } elsif ($opt =~ m/^net(\d+)$/) {
5025 # some changes can be done without hotplug
5026 vmconfig_update_net
($storecfg, $conf, $hotplug_features->{network
},
5027 $vmid, $opt, $value, $arch, $machine_type);
5028 } elsif (is_valid_drivename
($opt)) {
5029 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5030 # some changes can be done without hotplug
5031 my $drive = parse_drive
($opt, $value);
5032 if (drive_is_cloudinit
($drive)) {
5033 $cloudinit_opt = [$opt, $drive];
5034 # apply all the other changes first, then generate the cloudinit disk
5037 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5038 $vmid, $opt, $value, $arch, $machine_type);
5039 } elsif ($opt =~ m/^memory$/) { #dimms
5040 die "skip\n" if !$hotplug_features->{memory
};
5041 $value = PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $value);
5042 } elsif ($opt eq 'cpuunits') {
5043 my $new_cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{pending
}->{$opt}); #clamp
5044 $cgroup->change_cpu_shares($new_cpuunits);
5045 } elsif ($opt eq 'cpulimit') {
5046 my $cpulimit = $conf->{pending
}->{$opt} == 0 ?
-1 : int($conf->{pending
}->{$opt} * 100000);
5047 $cgroup->change_cpu_quota($cpulimit, 100000);
5048 } elsif ($opt eq 'agent') {
5049 vmconfig_update_agent
($conf, $opt, $value);
5051 die "skip\n"; # skip non-hot-pluggable options
5055 &$add_error($opt, $err) if $err ne "skip\n";
5057 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5058 $conf->{$opt} = $value;
5059 delete $conf->{pending
}->{$opt};
5063 if (defined($cloudinit_opt)) {
5064 my ($opt, $drive) = @$cloudinit_opt;
5065 my $value = $conf->{pending
}->{$opt};
5067 my $temp = {%$conf, $opt => $value};
5068 PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($temp, $vmid);
5069 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5070 $vmid, $opt, $value, $arch, $machine_type);
5073 &$add_error($opt, $err) if $err ne "skip\n";
5075 $conf->{$opt} = $value;
5076 delete $conf->{pending
}->{$opt};
5080 # unplug xhci controller if no usb device is left
5083 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
5084 next if !defined($conf->{"usb$i"});
5089 vm_deviceunplug
($vmid, $conf, 'xhci');
5093 PVE
::QemuConfig-
>write_config($vmid, $conf);
5095 if ($hotplug_features->{cloudinit
} && PVE
::QemuServer
::Cloudinit
::has_changes
($conf)) {
5096 PVE
::QemuServer
::vmconfig_update_cloudinit_drive
($storecfg, $conf, $vmid);
5100 sub try_deallocate_drive
{
5101 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5103 if (($force || $key =~ /^unused/) && !drive_is_cdrom
($drive, 1)) {
5104 my $volid = $drive->{file
};
5105 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
5106 my $sid = PVE
::Storage
::parse_volume_id
($volid);
5107 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5109 # check if the disk is really unused
5110 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5111 if PVE
::QemuServer
::Drive
::is_volume_in_use
($storecfg, $conf, $key, $volid);
5112 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5115 # If vm is not owner of this disk remove from config
5123 sub vmconfig_delete_or_detach_drive
{
5124 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5126 my $drive = parse_drive
($opt, $conf->{$opt});
5128 my $rpcenv = PVE
::RPCEnvironment
::get
();
5129 my $authuser = $rpcenv->get_user();
5132 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5133 try_deallocate_drive
($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5135 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $drive);
5141 sub vmconfig_apply_pending
{
5142 my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5144 return if !scalar(keys %{$conf->{pending
}});
5146 my $add_apply_error = sub {
5147 my ($opt, $msg) = @_;
5148 my $err_msg = "unable to apply pending change $opt : $msg";
5149 $errors->{$opt} = $err_msg;
5155 my $pending_delete_hash = PVE
::QemuConfig-
>parse_pending_delete($conf->{pending
}->{delete});
5156 foreach my $opt (sort keys %$pending_delete_hash) {
5157 my $force = $pending_delete_hash->{$opt}->{force
};
5159 if ($opt =~ m/^unused/) {
5160 die "internal error";
5161 } elsif (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5162 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5163 } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5165 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
5166 eval { PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
}) };
5172 $add_apply_error->($opt, $err);
5174 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5175 delete $conf->{$opt};
5179 PVE
::QemuConfig-
>cleanup_pending($conf);
5181 my $generate_cloudinit = $skip_cloud_init ?
0 : undef;
5183 foreach my $opt (keys %{$conf->{pending
}}) { # add/change
5184 next if $opt eq 'delete'; # just to be sure
5186 if (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5187 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, parse_drive
($opt, $conf->{$opt}))
5188 } elsif (defined($conf->{pending
}->{$opt}) && $opt =~ m/^net\d+$/) {
5190 my $new_net = PVE
::QemuServer
::parse_net
($conf->{pending
}->{$opt});
5192 my $old_net = PVE
::QemuServer
::parse_net
($conf->{$opt});
5194 if ($old_net->{bridge
} ne $new_net->{bridge
} ||
5195 $old_net->{macaddr
} ne $new_net->{macaddr
}) {
5196 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($old_net->{bridge
}, $old_net->{macaddr
}, $conf->{name
});
5199 #fixme: reuse ip if mac change && same bridge
5200 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($new_net->{bridge
}, $conf->{name
}, $new_net->{macaddr
}, $vmid, undef, 1);
5205 $add_apply_error->($opt, $err);
5208 if (is_valid_drivename
($opt)) {
5209 my $drive = parse_drive
($opt, $conf->{pending
}->{$opt});
5210 $generate_cloudinit //= 1 if drive_is_cloudinit
($drive);
5213 $conf->{$opt} = delete $conf->{pending
}->{$opt};
5217 # write all changes at once to avoid unnecessary i/o
5218 PVE
::QemuConfig-
>write_config($vmid, $conf);
5219 if ($generate_cloudinit) {
5220 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5221 # After successful generation and if there were changes to be applied, update the
5222 # config to drop the {cloudinit} entry.
5223 PVE
::QemuConfig-
>write_config($vmid, $conf);
5228 sub vmconfig_update_net
{
5229 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5231 my $newnet = parse_net
($value);
5233 if ($conf->{$opt}) {
5234 my $oldnet = parse_net
($conf->{$opt});
5236 if (safe_string_ne
($oldnet->{model
}, $newnet->{model
}) ||
5237 safe_string_ne
($oldnet->{macaddr
}, $newnet->{macaddr
}) ||
5238 safe_num_ne
($oldnet->{queues
}, $newnet->{queues
}) ||
5239 safe_num_ne
($oldnet->{mtu
}, $newnet->{mtu
}) ||
5240 !($newnet->{bridge
} && $oldnet->{bridge
})) { # bridge/nat mode change
5242 # for non online change, we try to hot-unplug
5243 die "skip\n" if !$hotplug;
5244 vm_deviceunplug
($vmid, $conf, $opt);
5247 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($oldnet->{bridge
}, $oldnet->{macaddr
}, $conf->{name
});
5252 die "internal error" if $opt !~ m/net(\d+)/;
5253 my $iface = "tap${vmid}i$1";
5255 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5256 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
}) ||
5257 safe_string_ne
($oldnet->{trunks
}, $newnet->{trunks
}) ||
5258 safe_num_ne
($oldnet->{firewall
}, $newnet->{firewall
})) {
5259 PVE
::Network
::tap_unplug
($iface);
5261 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5262 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5263 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
})) {
5264 qemu_set_link_status
($vmid, $opt, 0);
5267 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
})) {
5269 PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($oldnet->{bridge
}, $oldnet->{macaddr
}, $conf->{name
});
5270 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($newnet->{bridge
}, $conf->{name
}, $newnet->{macaddr
}, $vmid, undef, 1);
5275 PVE
::Network
::SDN
::Zones
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5277 PVE
::Network
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5280 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5281 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5282 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
})) {
5283 qemu_set_link_status
($vmid, $opt, 1);
5286 } elsif (safe_num_ne
($oldnet->{rate
}, $newnet->{rate
})) {
5287 # Rate can be applied on its own but any change above needs to
5288 # include the rate in tap_plug since OVS resets everything.
5289 PVE
::Network
::tap_rate_limit
($iface, $newnet->{rate
});
5292 if (safe_string_ne
($oldnet->{link_down
}, $newnet->{link_down
})) {
5293 qemu_set_link_status
($vmid, $opt, !$newnet->{link_down
});
5302 PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($newnet->{bridge
}, $conf->{name
}, $newnet->{macaddr
}, $vmid, undef, 1);
5303 PVE
::Network
::SDN
::Vnets
::add_dhcp_mapping
($newnet->{bridge
}, $newnet->{macaddr
}, $vmid, $conf->{name
});
5305 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5311 sub vmconfig_update_agent
{
5312 my ($conf, $opt, $value) = @_;
5314 die "skip\n" if !$conf->{$opt};
5316 my $hotplug_options = { fstrim_cloned_disks
=> 1 };
5318 my $old_agent = parse_guest_agent
($conf);
5319 my $agent = parse_guest_agent
({$opt => $value});
5321 for my $option (keys %$agent) { # added/changed options
5322 next if defined($hotplug_options->{$option});
5323 die "skip\n" if safe_string_ne
($agent->{$option}, $old_agent->{$option});
5326 for my $option (keys %$old_agent) { # removed options
5327 next if defined($hotplug_options->{$option});
5328 die "skip\n" if safe_string_ne
($old_agent->{$option}, $agent->{$option});
5331 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5334 sub vmconfig_update_disk
{
5335 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5337 my $drive = parse_drive
($opt, $value);
5339 if ($conf->{$opt} && (my $old_drive = parse_drive
($opt, $conf->{$opt}))) {
5340 my $media = $drive->{media
} || 'disk';
5341 my $oldmedia = $old_drive->{media
} || 'disk';
5342 die "unable to change media type\n" if $media ne $oldmedia;
5344 if (!drive_is_cdrom
($old_drive)) {
5346 if ($drive->{file
} ne $old_drive->{file
}) {
5348 die "skip\n" if !$hotplug;
5350 # unplug and register as unused
5351 vm_deviceunplug
($vmid, $conf, $opt);
5352 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive)
5355 # update existing disk
5357 # skip non hotpluggable value
5358 if (safe_string_ne
($drive->{aio
}, $old_drive->{aio
}) ||
5359 safe_string_ne
($drive->{discard
}, $old_drive->{discard
}) ||
5360 safe_string_ne
($drive->{iothread
}, $old_drive->{iothread
}) ||
5361 safe_string_ne
($drive->{queues
}, $old_drive->{queues
}) ||
5362 safe_string_ne
($drive->{cache
}, $old_drive->{cache
}) ||
5363 safe_string_ne
($drive->{ssd
}, $old_drive->{ssd
}) ||
5364 safe_string_ne
($drive->{ro
}, $old_drive->{ro
})) {
5369 if (safe_num_ne
($drive->{mbps
}, $old_drive->{mbps
}) ||
5370 safe_num_ne
($drive->{mbps_rd
}, $old_drive->{mbps_rd
}) ||
5371 safe_num_ne
($drive->{mbps_wr
}, $old_drive->{mbps_wr
}) ||
5372 safe_num_ne
($drive->{iops
}, $old_drive->{iops
}) ||
5373 safe_num_ne
($drive->{iops_rd
}, $old_drive->{iops_rd
}) ||
5374 safe_num_ne
($drive->{iops_wr
}, $old_drive->{iops_wr
}) ||
5375 safe_num_ne
($drive->{mbps_max
}, $old_drive->{mbps_max
}) ||
5376 safe_num_ne
($drive->{mbps_rd_max
}, $old_drive->{mbps_rd_max
}) ||
5377 safe_num_ne
($drive->{mbps_wr_max
}, $old_drive->{mbps_wr_max
}) ||
5378 safe_num_ne
($drive->{iops_max
}, $old_drive->{iops_max
}) ||
5379 safe_num_ne
($drive->{iops_rd_max
}, $old_drive->{iops_rd_max
}) ||
5380 safe_num_ne
($drive->{iops_wr_max
}, $old_drive->{iops_wr_max
}) ||
5381 safe_num_ne
($drive->{bps_max_length
}, $old_drive->{bps_max_length
}) ||
5382 safe_num_ne
($drive->{bps_rd_max_length
}, $old_drive->{bps_rd_max_length
}) ||
5383 safe_num_ne
($drive->{bps_wr_max_length
}, $old_drive->{bps_wr_max_length
}) ||
5384 safe_num_ne
($drive->{iops_max_length
}, $old_drive->{iops_max_length
}) ||
5385 safe_num_ne
($drive->{iops_rd_max_length
}, $old_drive->{iops_rd_max_length
}) ||
5386 safe_num_ne
($drive->{iops_wr_max_length
}, $old_drive->{iops_wr_max_length
})) {
5388 qemu_block_set_io_throttle
(
5390 ($drive->{mbps
} || 0)*1024*1024,
5391 ($drive->{mbps_rd
} || 0)*1024*1024,
5392 ($drive->{mbps_wr
} || 0)*1024*1024,
5393 $drive->{iops
} || 0,
5394 $drive->{iops_rd
} || 0,
5395 $drive->{iops_wr
} || 0,
5396 ($drive->{mbps_max
} || 0)*1024*1024,
5397 ($drive->{mbps_rd_max
} || 0)*1024*1024,
5398 ($drive->{mbps_wr_max
} || 0)*1024*1024,
5399 $drive->{iops_max
} || 0,
5400 $drive->{iops_rd_max
} || 0,
5401 $drive->{iops_wr_max
} || 0,
5402 $drive->{bps_max_length
} || 1,
5403 $drive->{bps_rd_max_length
} || 1,
5404 $drive->{bps_wr_max_length
} || 1,
5405 $drive->{iops_max_length
} || 1,
5406 $drive->{iops_rd_max_length
} || 1,
5407 $drive->{iops_wr_max_length
} || 1,
5417 if ($drive->{file
} eq 'none') {
5418 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5419 if (drive_is_cloudinit
($old_drive)) {
5420 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive);
5423 my $path = get_iso_path
($storecfg, $vmid, $drive->{file
});
5425 # force eject if locked
5426 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5429 mon_cmd
($vmid, "blockdev-change-medium",
5430 id
=> "$opt", filename
=> "$path");
5438 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5440 PVE
::Storage
::activate_volumes
($storecfg, [$drive->{file
}]) if $drive->{file
} !~ m
|^/dev/.+|;
5441 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5444 sub vmconfig_update_cloudinit_drive
{
5445 my ($storecfg, $conf, $vmid) = @_;
5447 my $cloudinit_ds = undef;
5448 my $cloudinit_drive = undef;
5450 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5451 my ($ds, $drive) = @_;
5452 if (PVE
::QemuServer
::drive_is_cloudinit
($drive)) {
5453 $cloudinit_ds = $ds;
5454 $cloudinit_drive = $drive;
5458 return if !$cloudinit_drive;
5460 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5461 PVE
::QemuConfig-
>write_config($vmid, $conf);
5464 my $running = PVE
::QemuServer
::check_running
($vmid);
5467 my $path = PVE
::Storage
::path
($storecfg, $cloudinit_drive->{file
});
5469 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$cloudinit_ds");
5470 mon_cmd
($vmid, "blockdev-change-medium", id
=> "$cloudinit_ds", filename
=> "$path");
5475 # called in locked context by incoming migration
5476 sub vm_migrate_get_nbd_disks
{
5477 my ($storecfg, $conf, $replicated_volumes) = @_;
5479 my $local_volumes = {};
5480 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5481 my ($ds, $drive) = @_;
5483 return if drive_is_cdrom
($drive);
5484 return if $ds eq 'tpmstate0';
5486 my $volid = $drive->{file
};
5490 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid);
5492 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5493 return if $scfg->{shared
};
5495 my $format = qemu_img_format
($scfg, $volname);
5497 # replicated disks re-use existing state via bitmap
5498 my $use_existing = $replicated_volumes->{$volid} ?
1 : 0;
5499 $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5501 return $local_volumes;
5504 # called in locked context by incoming migration
5505 sub vm_migrate_alloc_nbd_disks
{
5506 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5509 foreach my $opt (sort keys %$source_volumes) {
5510 my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5512 if ($use_existing) {
5513 $nbd->{$opt}->{drivestr
} = print_drive
($drive);
5514 $nbd->{$opt}->{volid
} = $volid;
5515 $nbd->{$opt}->{replicated
} = 1;
5519 $storeid = PVE
::JSONSchema
::map_id
($storagemap, $storeid);
5521 # order of precedence, filtered by whether storage supports it:
5522 # 1. explicit requested format
5523 # 2. default format of storage
5524 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5525 $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5527 my $size = $drive->{size
} / 1024;
5528 my $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $format, undef, $size);
5529 my $newdrive = $drive;
5530 $newdrive->{format
} = $format;
5531 $newdrive->{file
} = $newvolid;
5532 my $drivestr = print_drive
($newdrive);
5533 $nbd->{$opt}->{drivestr
} = $drivestr;
5534 $nbd->{$opt}->{volid
} = $newvolid;
5540 # see vm_start_nolock for parameters, additionally:
5542 # storagemap = parsed storage map for allocating NBD disks
5544 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5546 return PVE
::QemuConfig-
>lock_config($vmid, sub {
5547 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migrate_opts->{migratedfrom
});
5549 die "you can't start a vm if it's a template\n"
5550 if !$params->{skiptemplate
} && PVE
::QemuConfig-
>is_template($conf);
5552 my $has_suspended_lock = PVE
::QemuConfig-
>has_lock($conf, 'suspended');
5553 my $has_backup_lock = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5555 my $running = check_running
($vmid, undef, $migrate_opts->{migratedfrom
});
5557 if ($has_backup_lock && $running) {
5558 # a backup is currently running, attempt to start the guest in the
5559 # existing QEMU instance
5560 return vm_resume
($vmid);
5563 PVE
::QemuConfig-
>check_lock($conf)
5564 if !($params->{skiplock
} || $has_suspended_lock);
5566 $params->{resume
} = $has_suspended_lock || defined($conf->{vmstate
});
5568 die "VM $vmid already running\n" if $running;
5570 if (my $storagemap = $migrate_opts->{storagemap
}) {
5571 my $replicated = $migrate_opts->{replicated_volumes
};
5572 my $disks = vm_migrate_get_nbd_disks
($storecfg, $conf, $replicated);
5573 $migrate_opts->{nbd
} = vm_migrate_alloc_nbd_disks
($storecfg, $vmid, $disks, $storagemap);
5575 foreach my $opt (keys %{$migrate_opts->{nbd
}}) {
5576 $conf->{$opt} = $migrate_opts->{nbd
}->{$opt}->{drivestr
};
5580 return vm_start_nolock
($storecfg, $vmid, $conf, $params, $migrate_opts);
5586 # statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5587 # skiplock => 0/1, skip checking for config lock
5588 # skiptemplate => 0/1, skip checking whether VM is template
5589 # forcemachine => to force QEMU machine (rollback/migration)
5590 # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5591 # timeout => in seconds
5592 # paused => start VM in paused state (backup)
5593 # resume => resume from hibernation
5604 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5605 # migratedfrom => source node
5606 # spice_ticket => used for spice migration, passed via tunnel/stdin
5607 # network => CIDR of migration network
5608 # type => secure/insecure - tunnel over encrypted connection or plain-text
5609 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5610 # replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5611 # offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5612 # contained in config
5613 sub vm_start_nolock
{
5614 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5616 my $statefile = $params->{statefile
};
5617 my $resume = $params->{resume
};
5619 my $migratedfrom = $migrate_opts->{migratedfrom
};
5620 my $migration_type = $migrate_opts->{type
};
5624 # clean up leftover reboot request files
5625 eval { clear_reboot_request
($vmid); };
5628 if (!$statefile && scalar(keys %{$conf->{pending
}})) {
5629 vmconfig_apply_pending
($vmid, $conf, $storecfg);
5630 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5633 # don't regenerate the ISO if the VM is started as part of a live migration
5634 # this way we can reuse the old ISO with the correct config
5635 if (!$migratedfrom) {
5636 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5637 # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5638 # $conf->{cloudinit}, so we could just not do this?
5639 # But we do it above, so for now let's be consistent.
5640 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5644 # override offline migrated volumes, conf is out of date still
5645 if (my $offline_volumes = $migrate_opts->{offline_volumes
}) {
5646 for my $key (sort keys $offline_volumes->%*) {
5647 my $parsed = parse_drive
($key, $conf->{$key});
5648 $parsed->{file
} = $offline_volumes->{$key};
5649 $conf->{$key} = print_drive
($parsed);
5653 my $defaults = load_defaults
();
5655 # set environment variable useful inside network script
5656 # for remote migration the config is available on the target node!
5657 if (!$migrate_opts->{remote_node
}) {
5658 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom;
5661 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5663 my $forcemachine = $params->{forcemachine
};
5664 my $forcecpu = $params->{forcecpu
};
5666 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5667 $forcemachine = $conf->{runningmachine
};
5668 $forcecpu = $conf->{runningcpu
};
5669 print "Resuming suspended VM\n";
5672 my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command
($storecfg, $vmid,
5673 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5676 my $get_migration_ip = sub {
5677 my ($nodename) = @_;
5679 return $migration_ip if defined($migration_ip);
5681 my $cidr = $migrate_opts->{network
};
5683 if (!defined($cidr)) {
5684 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5685 $cidr = $dc_conf->{migration
}->{network
};
5688 if (defined($cidr)) {
5689 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5691 die "could not get IP: no address configured on local " .
5692 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5694 die "could not get IP: multiple addresses configured on local " .
5695 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5697 $migration_ip = @$ips[0];
5700 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5701 if !defined($migration_ip);
5703 return $migration_ip;
5707 if ($statefile eq 'tcp') {
5708 my $migrate = $res->{migrate
} = { proto
=> 'tcp' };
5709 $migrate->{addr
} = "localhost";
5710 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5711 my $nodename = nodename
();
5713 if (!defined($migration_type)) {
5714 if (defined($datacenterconf->{migration
}->{type
})) {
5715 $migration_type = $datacenterconf->{migration
}->{type
};
5717 $migration_type = 'secure';
5721 if ($migration_type eq 'insecure') {
5722 $migrate->{addr
} = $get_migration_ip->($nodename);
5723 $migrate->{addr
} = "[$migrate->{addr}]" if Net
::IP
::ip_is_ipv6
($migrate->{addr
});
5726 # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5728 push @$cmd, '-incoming', 'defer';
5731 } elsif ($statefile eq 'unix') {
5732 # should be default for secure migrations as a ssh TCP forward
5733 # tunnel is not deterministic reliable ready and fails regurarly
5734 # to set up in time, so use UNIX socket forwards
5735 my $migrate = $res->{migrate
} = { proto
=> 'unix' };
5736 $migrate->{addr
} = "/run/qemu-server/$vmid.migrate";
5737 unlink $migrate->{addr
};
5739 $migrate->{uri
} = "unix:$migrate->{addr}";
5740 push @$cmd, '-incoming', $migrate->{uri
};
5743 } elsif (-e
$statefile) {
5744 push @$cmd, '-loadstate', $statefile;
5746 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5747 push @$vollist, $statefile;
5748 push @$cmd, '-loadstate', $statepath;
5750 } elsif ($params->{paused
}) {
5754 my $memory = get_current_memory
($conf->{memory
});
5755 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $memory, $resume);
5757 my $pci_reserve_list = [];
5758 for my $device (values $pci_devices->%*) {
5759 next if $device->{mdev
}; # we don't reserve for mdev devices
5760 push $pci_reserve_list->@*, map { $_->{id
} } $device->{ids
}->@*;
5763 # reserve all PCI IDs before actually doing anything with them
5764 PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, $start_timeout);
5768 for my $id (sort keys %$pci_devices) {
5769 my $d = $pci_devices->{$id};
5770 my ($index) = ($id =~ m/^hostpci(\d+)$/);
5773 for my $dev ($d->{ids
}->@*) {
5774 my $info = eval { PVE
::QemuServer
::PCI
::prepare_pci_device
($vmid, $dev->{id
}, $index, $d->{mdev
}) };
5777 $chosen_mdev = $info;
5778 last if $chosen_mdev; # if successful, we're done
5784 next if !$d->{mdev
};
5785 die "could not create mediated device\n" if !defined($chosen_mdev);
5787 # nvidia grid needs the uuid of the mdev as qemu parameter
5788 if (!defined($uuid) && $chosen_mdev->{vendor
} =~ m/^(0x)?10de$/) {
5789 if (defined($conf->{smbios1
})) {
5790 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
5791 $uuid = $smbios_conf->{uuid
} if defined($smbios_conf->{uuid
});
5793 $uuid = PVE
::QemuServer
::PCI
::generate_mdev_uuid
($vmid, $index) if !defined($uuid);
5796 push @$cmd, '-uuid', $uuid if defined($uuid);
5799 eval { cleanup_pci_devices
($vmid, $conf) };
5804 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5807 my %silence_std_outs = (outfunc
=> sub {}, errfunc
=> sub {});
5808 eval { run_command
(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5809 eval { run_command
(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5810 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5811 # timeout should be more than enough here...
5812 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 20);
5814 my $cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{cpuunits
});
5817 timeout
=> $statefile ?
undef : $start_timeout,
5822 # when migrating, prefix QEMU output so other side can pick up any
5823 # errors that might occur and show the user
5824 if ($migratedfrom) {
5825 $run_params{quiet
} = 1;
5826 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5829 my %systemd_properties = (
5830 Slice
=> 'qemu.slice',
5831 KillMode
=> 'process',
5833 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5836 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5837 $systemd_properties{CPUWeight
} = $cpuunits;
5839 $systemd_properties{CPUShares
} = $cpuunits;
5842 if (my $cpulimit = $conf->{cpulimit
}) {
5843 $systemd_properties{CPUQuota
} = int($cpulimit * 100);
5845 $systemd_properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5847 my $run_qemu = sub {
5848 PVE
::Tools
::run_fork
sub {
5849 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5852 if ((my $tpm = $conf->{tpmstate0
}) && !PVE
::QemuConfig-
>is_template($conf)) {
5853 # start the TPM emulator so QEMU can connect on start
5854 $tpmpid = start_swtpm
($storecfg, $vmid, $tpm, $migratedfrom);
5857 my $exitcode = run_command
($cmd, %run_params);
5860 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5861 kill 'TERM', $tpmpid;
5863 die "QEMU exited with code $exitcode\n";
5868 if ($conf->{hugepages
}) {
5871 my $hotplug_features =
5872 parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
5873 my $hugepages_topology =
5874 PVE
::QemuServer
::Memory
::hugepages_topology
($conf, $hotplug_features->{memory
});
5876 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5878 PVE
::QemuServer
::Memory
::hugepages_mount
();
5879 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5881 eval { $run_qemu->() };
5883 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5884 if !$conf->{keephugepages
};
5888 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5889 if !$conf->{keephugepages
};
5891 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5894 eval { $run_qemu->() };
5898 # deactivate volumes if start fails
5899 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5901 eval { cleanup_pci_devices
($vmid, $conf) };
5904 die "start failed: $err";
5907 # re-reserve all PCI IDs now that we can know the actual VM PID
5908 my $pid = PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
5909 eval { PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, undef, $pid) };
5912 if (defined(my $migrate = $res->{migrate
})) {
5913 if ($migrate->{proto
} eq 'tcp') {
5914 my $nodename = nodename
();
5915 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5916 $migrate->{port
} = PVE
::Tools
::next_migrate_port
($pfamily);
5917 $migrate->{uri
} = "tcp:$migrate->{addr}:$migrate->{port}";
5918 mon_cmd
($vmid, "migrate-incoming", uri
=> $migrate->{uri
});
5920 print "migration listens on $migrate->{uri}\n";
5921 } elsif ($statefile) {
5922 eval { mon_cmd
($vmid, "cont"); };
5926 #start nbd server for storage migration
5927 if (my $nbd = $migrate_opts->{nbd
}) {
5928 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5930 my $migrate_storage_uri;
5931 # nbd_protocol_version > 0 for unix socket support
5932 if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5933 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5934 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
5935 $migrate_storage_uri = "nbd:unix:$socket_path";
5936 $res->{migrate
}->{unix_sockets
} = [$socket_path];
5938 my $nodename = nodename
();
5939 my $localip = $get_migration_ip->($nodename);
5940 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5941 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5943 mon_cmd
($vmid, "nbd-server-start", addr
=> {
5946 host
=> "${localip}",
5947 port
=> "${storage_migrate_port}",
5950 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5951 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5954 my $block_info = mon_cmd
($vmid, "query-block");
5955 $block_info = { map { $_->{device
} => $_ } $block_info->@* };
5957 foreach my $opt (sort keys %$nbd) {
5958 my $drivestr = $nbd->{$opt}->{drivestr
};
5959 my $volid = $nbd->{$opt}->{volid
};
5961 my $block_node = $block_info->{"drive-$opt"}->{inserted
}->{'node-name'};
5967 'node-name' => $block_node,
5968 writable
=> JSON
::true
,
5970 name
=> "drive-$opt", # NBD export name
5973 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5974 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5975 print "re-using replicated volume: $opt - $volid\n"
5976 if $nbd->{$opt}->{replicated
};
5978 $res->{drives
}->{$opt} = $nbd->{$opt};
5979 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
5983 if ($migratedfrom) {
5985 set_migration_caps
($vmid);
5990 print "spice listens on port $spice_port\n";
5991 $res->{spice_port
} = $spice_port;
5992 if ($migrate_opts->{spice_ticket
}) {
5993 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
5994 $migrate_opts->{spice_ticket
});
5995 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
6000 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
6001 if !$statefile && $conf->{balloon
};
6003 foreach my $opt (keys %$conf) {
6004 next if $opt !~ m/^net\d+$/;
6005 my $nicconf = parse_net
($conf->{$opt});
6006 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
6008 add_nets_bridge_fdb
($conf, $vmid);
6011 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
6016 path
=> "machine/peripheral/balloon0",
6017 property
=> "guest-stats-polling-interval",
6021 log_warn
("could not set polling interval for ballooning - $@") if $@;
6025 print "Resumed VM, removing state\n";
6026 if (my $vmstate = $conf->{vmstate
}) {
6027 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6028 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6030 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6031 PVE
::QemuConfig-
>write_config($vmid, $conf);
6034 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
6036 my ($current_machine, $is_deprecated) =
6037 PVE
::QemuServer
::Machine
::get_current_qemu_machine
($vmid);
6038 if ($is_deprecated) {
6040 "current machine version '$current_machine' is deprecated - see the documentation and ".
6041 "change to a newer one",
6048 sub vm_commandline
{
6049 my ($storecfg, $vmid, $snapname) = @_;
6051 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6053 my ($forcemachine, $forcecpu);
6055 my $snapshot = $conf->{snapshots
}->{$snapname};
6056 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6058 # check for machine or CPU overrides in snapshot
6059 $forcemachine = $snapshot->{runningmachine
};
6060 $forcecpu = $snapshot->{runningcpu
};
6062 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
6067 my $defaults = load_defaults
();
6069 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6071 return PVE
::Tools
::cmd2string
($cmd);
6075 my ($vmid, $skiplock) = @_;
6077 PVE
::QemuConfig-
>lock_config($vmid, sub {
6079 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6081 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6083 mon_cmd
($vmid, "system_reset");
6087 sub get_vm_volumes
{
6091 foreach_volid
($conf, sub {
6092 my ($volid, $attr) = @_;
6094 return if $volid =~ m
|^/|;
6096 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6099 push @$vollist, $volid;
6105 sub cleanup_pci_devices
{
6106 my ($vmid, $conf) = @_;
6108 foreach my $key (keys %$conf) {
6109 next if $key !~ m/^hostpci(\d+)$/;
6110 my $hostpciindex = $1;
6111 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
6112 my $d = parse_hostpci
($conf->{$key});
6114 # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6115 # don't want to break ABI just for this two liner
6116 my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6118 # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6119 # out when we do it first. so wait for 10 seconds and then try it
6120 if ($d->{ids
}->[0]->[0]->{vendor
} =~ m/^(0x)?10de$/) {
6124 PVE
::SysFSTools
::file_write
("$dev_sysfs_dir/remove", "1") if -e
$dev_sysfs_dir;
6127 PVE
::QemuServer
::PCI
::remove_pci_reservation
($vmid);
6130 sub vm_stop_cleanup
{
6131 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6136 my $vollist = get_vm_volumes
($conf);
6137 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6139 if (my $tpmdrive = $conf->{tpmstate0
}) {
6140 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
6141 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
6143 PVE
::Storage
::unmap_volume
($storecfg, $tpm->{file
});
6148 foreach my $ext (qw(mon qmp pid vnc qga)) {
6149 unlink "/var/run/qemu-server/${vmid}.$ext";
6152 if ($conf->{ivshmem
}) {
6153 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
6154 # just delete it for now, VMs which have this already open do not
6155 # are affected, but new VMs will get a separated one. If this
6156 # becomes an issue we either add some sort of ref-counting or just
6157 # add a "don't delete on stop" flag to the ivshmem format.
6158 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
6161 cleanup_pci_devices
($vmid, $conf);
6163 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
6165 warn $@ if $@; # avoid errors - just warn
6168 # call only in locked context
6170 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6172 my $pid = check_running
($vmid, $nocheck);
6177 $conf = PVE
::QemuConfig-
>load_config($vmid);
6178 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6179 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
6180 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
6181 $timeout = $opts->{down
} if $opts->{down
};
6183 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
6188 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
6189 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
6191 mon_cmd
($vmid, "system_powerdown");
6194 mon_cmd
($vmid, "quit");
6200 $timeout = 60 if !defined($timeout);
6203 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6208 if ($count >= $timeout) {
6210 warn "VM still running - terminating now with SIGTERM\n";
6213 die "VM quit/powerdown failed - got timeout\n";
6216 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6220 if (!check_running
($vmid, $nocheck)) {
6221 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6225 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6228 die "VM quit/powerdown failed\n";
6236 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6241 if ($count >= $timeout) {
6242 warn "VM still running - terminating now with SIGKILL\n";
6247 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6250 # Note: use $nocheck to skip tests if VM configuration file exists.
6251 # We need that when migration VMs to other nodes (files already moved)
6252 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6254 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6256 $force = 1 if !defined($force) && !$shutdown;
6259 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
6260 kill 15, $pid if $pid;
6261 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
6262 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
6266 PVE
::QemuConfig-
>lock_config($vmid, sub {
6267 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6272 my ($vmid, $timeout) = @_;
6274 PVE
::QemuConfig-
>lock_config($vmid, sub {
6277 # only reboot if running, as qmeventd starts it again on a stop event
6278 return if !check_running
($vmid);
6280 create_reboot_request
($vmid);
6282 my $storecfg = PVE
::Storage
::config
();
6283 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
6287 # avoid that the next normal shutdown will be confused for a reboot
6288 clear_reboot_request
($vmid);
6294 # note: if using the statestorage parameter, the caller has to check privileges
6296 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6303 PVE
::QemuConfig-
>lock_config($vmid, sub {
6305 $conf = PVE
::QemuConfig-
>load_config($vmid);
6307 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
6308 PVE
::QemuConfig-
>check_lock($conf)
6309 if !($skiplock || $is_backing_up);
6311 die "cannot suspend to disk during backup\n"
6312 if $is_backing_up && $includestate;
6314 if ($includestate) {
6315 $conf->{lock} = 'suspending';
6316 my $date = strftime
("%Y-%m-%d", localtime(time()));
6317 $storecfg = PVE
::Storage
::config
();
6318 if (!$statestorage) {
6319 $statestorage = find_vmstate_storage
($conf, $storecfg);
6320 # check permissions for the storage
6321 my $rpcenv = PVE
::RPCEnvironment
::get
();
6322 if ($rpcenv->{type
} ne 'cli') {
6323 my $authuser = $rpcenv->get_user();
6324 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6329 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
6330 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6331 $path = PVE
::Storage
::path
($storecfg, $vmstate);
6332 PVE
::QemuConfig-
>write_config($vmid, $conf);
6334 mon_cmd
($vmid, "stop");
6338 if ($includestate) {
6340 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
6343 set_migration_caps
($vmid, 1);
6344 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
6346 my $state = mon_cmd
($vmid, "query-savevm");
6347 if (!$state->{status
}) {
6348 die "savevm not active\n";
6349 } elsif ($state->{status
} eq 'active') {
6352 } elsif ($state->{status
} eq 'completed') {
6353 print "State saved, quitting\n";
6355 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
6356 die "query-savevm failed with error '$state->{error}'\n"
6358 die "query-savevm returned status '$state->{status}'\n";
6364 PVE
::QemuConfig-
>lock_config($vmid, sub {
6365 $conf = PVE
::QemuConfig-
>load_config($vmid);
6367 # cleanup, but leave suspending lock, to indicate something went wrong
6369 mon_cmd
($vmid, "savevm-end");
6370 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6371 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6372 delete $conf->@{qw(vmstate runningmachine runningcpu)};
6373 PVE
::QemuConfig-
>write_config($vmid, $conf);
6379 die "lock changed unexpectedly\n"
6380 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
6382 mon_cmd
($vmid, "quit");
6383 $conf->{lock} = 'suspended';
6384 PVE
::QemuConfig-
>write_config($vmid, $conf);
6389 # $nocheck is set when called as part of a migration - in this context the
6390 # location of the config file (source or target node) is not deterministic,
6391 # since migration cannot wait for pmxcfs to process the rename
6393 my ($vmid, $skiplock, $nocheck) = @_;
6395 PVE
::QemuConfig-
>lock_config($vmid, sub {
6396 my $res = mon_cmd
($vmid, 'query-status');
6397 my $resume_cmd = 'cont';
6401 $conf = eval { PVE
::QemuConfig-
>load_config($vmid) }; # try on target node
6403 my $vmlist = PVE
::Cluster
::get_vmlist
();
6404 if (exists($vmlist->{ids
}->{$vmid})) {
6405 my $node = $vmlist->{ids
}->{$vmid}->{node
};
6406 $conf = eval { PVE
::QemuConfig-
>load_config($vmid, $node) }; # try on source node
6409 PVE
::Cluster
::cfs_update
(); # vmlist was wrong, invalidate cache
6410 $conf = PVE
::QemuConfig-
>load_config($vmid); # last try on target node again
6414 $conf = PVE
::QemuConfig-
>load_config($vmid);
6417 if ($res->{status
}) {
6418 return if $res->{status
} eq 'running'; # job done, go home
6419 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
6420 $reset = 1 if $res->{status
} eq 'shutdown';
6424 PVE
::QemuConfig-
>check_lock($conf)
6425 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
6429 # required if a VM shuts down during a backup and we get a resume
6430 # request before the backup finishes for example
6431 mon_cmd
($vmid, "system_reset");
6434 add_nets_bridge_fdb
($conf, $vmid) if $resume_cmd eq 'cont';
6436 mon_cmd
($vmid, $resume_cmd);
6441 my ($vmid, $skiplock, $key) = @_;
6443 PVE
::QemuConfig-
>lock_config($vmid, sub {
6445 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6447 # there is no qmp command, so we use the human monitor command
6448 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
6449 die $res if $res ne '';
6453 sub check_bridge_access
{
6454 my ($rpcenv, $authuser, $conf) = @_;
6456 return 1 if $authuser eq 'root@pam';
6458 for my $opt (sort keys $conf->%*) {
6459 next if $opt !~ m/^net\d+$/;
6460 my $net = parse_net
($conf->{$opt});
6461 my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6462 PVE
::GuestHelpers
::check_vnet_access
($rpcenv, $authuser, $bridge, $tag, $trunks);
6467 sub check_mapping_access
{
6468 my ($rpcenv, $user, $conf) = @_;
6470 for my $opt (keys $conf->%*) {
6471 if ($opt =~ m/^usb\d+$/) {
6472 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-usb', $conf->{$opt});
6473 if (my $host = $device->{host
}) {
6474 die "only root can set '$opt' config for real devices\n"
6475 if $host !~ m/^spice$/i && $user ne 'root@pam';
6476 } elsif ($device->{mapping
}) {
6477 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6479 die "either 'host' or 'mapping' must be set.\n";
6481 } elsif ($opt =~ m/^hostpci\d+$/) {
6482 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-hostpci', $conf->{$opt});
6483 if ($device->{host
}) {
6484 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6485 } elsif ($device->{mapping
}) {
6486 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6488 die "either 'host' or 'mapping' must be set.\n";
6494 sub check_restore_permissions
{
6495 my ($rpcenv, $user, $conf) = @_;
6497 check_bridge_access
($rpcenv, $user, $conf);
6498 check_mapping_access
($rpcenv, $user, $conf);
6500 # vzdump restore implementaion
6502 sub tar_archive_read_firstfile
{
6503 my $archive = shift;
6505 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
6507 # try to detect archive type first
6508 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6509 die "unable to open file '$archive'\n";
6510 my $firstfile = <$fh>;
6514 die "ERROR: archive contaions no data\n" if !$firstfile;
6520 sub tar_restore_cleanup
{
6521 my ($storecfg, $statfile) = @_;
6523 print STDERR
"starting cleanup\n";
6525 if (my $fd = IO
::File-
>new($statfile, "r")) {
6526 while (defined(my $line = <$fd>)) {
6527 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6530 if ($volid =~ m
|^/|) {
6531 unlink $volid || die 'unlink failed\n';
6533 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6535 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6537 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6539 print STDERR
"unable to parse line in statfile - $line";
6546 sub restore_file_archive
{
6547 my ($archive, $vmid, $user, $opts) = @_;
6549 return restore_vma_archive
($archive, $vmid, $user, $opts)
6552 my $info = PVE
::Storage
::archive_info
($archive);
6553 my $format = $opts->{format
} // $info->{format
};
6554 my $comp = $info->{compression
};
6556 # try to detect archive format
6557 if ($format eq 'tar') {
6558 return restore_tar_archive
($archive, $vmid, $user, $opts);
6560 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
6564 # hepler to remove disks that will not be used after restore
6565 my $restore_cleanup_oldconf = sub {
6566 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6568 my $kept_disks = {};
6570 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
6571 my ($ds, $drive) = @_;
6573 return if drive_is_cdrom
($drive, 1);
6575 my $volid = $drive->{file
};
6576 return if !$volid || $volid =~ m
|^/|;
6578 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
6579 return if !$path || !$owner || ($owner != $vmid);
6581 # Note: only delete disk we want to restore
6582 # other volumes will become unused
6583 if ($virtdev_hash->{$ds}) {
6584 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
6589 $kept_disks->{$volid} = 1;
6593 # after the restore we have no snapshots anymore
6594 for my $snapname (keys $oldconf->{snapshots
}->%*) {
6595 my $snap = $oldconf->{snapshots
}->{$snapname};
6596 if ($snap->{vmstate
}) {
6597 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
6603 for my $volid (keys $kept_disks->%*) {
6604 eval { PVE
::Storage
::volume_snapshot_delete
($storecfg, $volid, $snapname); };
6610 # Helper to parse vzdump backup device hints
6612 # $rpcenv: Environment, used to ckeck storage permissions
6613 # $user: User ID, to check storage permissions
6614 # $storecfg: Storage configuration
6615 # $fh: the file handle for reading the configuration
6616 # $devinfo: should contain device sizes for all backu-up'ed devices
6617 # $options: backup options (pool, default storage)
6619 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6620 my $parse_backup_hints = sub {
6621 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6623 my $check_storage = sub { # assert if an image can be allocate
6624 my ($storeid, $scfg) = @_;
6625 die "Content type 'images' is not available on storage '$storeid'\n"
6626 if !$scfg->{content
}->{images
};
6627 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6628 if $user ne 'root@pam';
6631 my $virtdev_hash = {};
6632 while (defined(my $line = <$fh>)) {
6633 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6634 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6635 die "archive does not contain data for drive '$virtdev'\n"
6636 if !$devinfo->{$devname};
6638 if (defined($options->{storage
})) {
6639 $storeid = $options->{storage
} || 'local';
6640 } elsif (!$storeid) {
6643 $format = 'raw' if !$format;
6644 $devinfo->{$devname}->{devname
} = $devname;
6645 $devinfo->{$devname}->{virtdev
} = $virtdev;
6646 $devinfo->{$devname}->{format
} = $format;
6647 $devinfo->{$devname}->{storeid
} = $storeid;
6649 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6650 $check_storage->($storeid, $scfg); # permission and content type check
6652 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6653 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6655 my $drive = parse_drive
($virtdev, $2);
6657 if (drive_is_cloudinit
($drive)) {
6658 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6659 $storeid = $options->{storage
} if defined ($options->{storage
});
6660 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6661 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6663 $check_storage->($storeid, $scfg); # permission and content type check
6665 $virtdev_hash->{$virtdev} = {
6667 storeid
=> $storeid,
6668 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6675 return $virtdev_hash;
6678 # Helper to allocate and activate all volumes required for a restore
6680 # $storecfg: Storage configuration
6681 # $virtdev_hash: as returned by parse_backup_hints()
6683 # Returns: { $virtdev => $volid }
6684 my $restore_allocate_devices = sub {
6685 my ($storecfg, $virtdev_hash, $vmid) = @_;
6688 foreach my $virtdev (sort keys %$virtdev_hash) {
6689 my $d = $virtdev_hash->{$virtdev};
6690 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6691 my $storeid = $d->{storeid
};
6692 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6694 # test if requested format is supported
6695 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6696 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6697 $d->{format
} = $defFormat if !$supported;
6700 if ($d->{is_cloudinit
}) {
6701 $name = "vm-$vmid-cloudinit";
6702 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6703 if ($scfg->{path
}) {
6704 $name .= ".$d->{format}";
6708 my $volid = PVE
::Storage
::vdisk_alloc
(
6709 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6711 print STDERR
"new volume ID is '$volid'\n";
6712 $d->{volid
} = $volid;
6714 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6716 $map->{$virtdev} = $volid;
6722 sub restore_update_config_line
{
6723 my ($cookie, $map, $line, $unique) = @_;
6725 return '' if $line =~ m/^\#qmdump\#/;
6726 return '' if $line =~ m/^\#vzdump\#/;
6727 return '' if $line =~ m/^lock:/;
6728 return '' if $line =~ m/^unused\d+:/;
6729 return '' if $line =~ m/^parent:/;
6733 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6734 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6735 # try to convert old 1.X settings
6736 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6737 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6738 my ($model, $macaddr) = split(/\=/, $devconfig);
6739 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6742 bridge
=> "vmbr$ind",
6743 macaddr
=> $macaddr,
6745 my $netstr = print_net
($net);
6747 $res .= "net$cookie->{netcount}: $netstr\n";
6748 $cookie->{netcount
}++;
6750 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6751 my ($id, $netstr) = ($1, $2);
6752 my $net = parse_net
($netstr);
6753 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6754 $netstr = print_net
($net);
6755 $res .= "$id: $netstr\n";
6756 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6759 my $di = parse_drive
($virtdev, $value);
6760 if (defined($di->{backup
}) && !$di->{backup
}) {
6762 } elsif ($map->{$virtdev}) {
6763 delete $di->{format
}; # format can change on restore
6764 $di->{file
} = $map->{$virtdev};
6765 $value = print_drive
($di);
6766 $res .= "$virtdev: $value\n";
6770 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6772 if ($vmgenid ne '0') {
6773 # always generate a new vmgenid if there was a valid one setup
6774 $vmgenid = generate_uuid
();
6776 $res .= "vmgenid: $vmgenid\n";
6777 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6778 my ($uuid, $uuid_str);
6779 UUID
::generate
($uuid);
6780 UUID
::unparse
($uuid, $uuid_str);
6781 my $smbios1 = parse_smbios1
($2);
6782 $smbios1->{uuid
} = $uuid_str;
6783 $res .= $1.print_smbios1
($smbios1)."\n";
6791 my $restore_deactivate_volumes = sub {
6792 my ($storecfg, $virtdev_hash) = @_;
6795 for my $dev (values $virtdev_hash->%*) {
6796 push $vollist->@*, $dev->{volid
} if $dev->{volid
};
6799 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
6800 print STDERR
$@ if $@;
6803 my $restore_destroy_volumes = sub {
6804 my ($storecfg, $virtdev_hash) = @_;
6806 for my $dev (values $virtdev_hash->%*) {
6807 my $volid = $dev->{volid
} or next;
6809 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6810 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6812 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6816 sub restore_merge_config
{
6817 my ($filename, $backup_conf_raw, $override_conf) = @_;
6819 my $backup_conf = parse_vm_config
($filename, $backup_conf_raw);
6820 for my $key (keys $override_conf->%*) {
6821 $backup_conf->{$key} = $override_conf->{$key};
6824 return $backup_conf;
6828 my ($cfg, $vmid) = @_;
6830 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6832 my $volid_hash = {};
6833 foreach my $storeid (keys %$info) {
6834 foreach my $item (@{$info->{$storeid}}) {
6835 next if !($item->{volid
} && $item->{size
});
6836 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6837 $volid_hash->{$item->{volid
}} = $item;
6844 sub update_disk_config
{
6845 my ($vmid, $conf, $volid_hash) = @_;
6848 my $prefix = "VM $vmid";
6850 # used and unused disks
6851 my $referenced = {};
6853 # Note: it is allowed to define multiple storages with same path (alias), so
6854 # we need to check both 'volid' and real 'path' (two different volid can point
6855 # to the same path).
6857 my $referencedpath = {};
6860 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6861 my ($opt, $drive) = @_;
6863 my $volid = $drive->{file
};
6865 my $volume = $volid_hash->{$volid};
6867 # mark volid as "in-use" for next step
6868 $referenced->{$volid} = 1;
6869 if ($volume && (my $path = $volume->{path
})) {
6870 $referencedpath->{$path} = 1;
6873 return if drive_is_cdrom
($drive);
6876 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6877 if (defined($updated)) {
6879 $conf->{$opt} = print_drive
($updated);
6880 print "$prefix ($opt): $msg\n";
6884 # remove 'unusedX' entry if volume is used
6885 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6886 my ($opt, $drive) = @_;
6888 my $volid = $drive->{file
};
6892 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6893 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6894 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6896 delete $conf->{$opt};
6899 $referenced->{$volid} = 1;
6900 $referencedpath->{$path} = 1 if $path;
6903 foreach my $volid (sort keys %$volid_hash) {
6904 next if $volid =~ m/vm-$vmid-state-/;
6905 next if $referenced->{$volid};
6906 my $path = $volid_hash->{$volid}->{path
};
6907 next if !$path; # just to be sure
6908 next if $referencedpath->{$path};
6910 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6911 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6912 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6919 my ($vmid, $nolock, $dryrun) = @_;
6921 my $cfg = PVE
::Storage
::config
();
6923 print "rescan volumes...\n";
6924 my $volid_hash = scan_volids
($cfg, $vmid);
6926 my $updatefn = sub {
6929 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6931 PVE
::QemuConfig-
>check_lock($conf);
6934 foreach my $volid (keys %$volid_hash) {
6935 my $info = $volid_hash->{$volid};
6936 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6939 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6941 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6944 if (defined($vmid)) {
6948 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6951 my $vmlist = config_list
();
6952 foreach my $vmid (keys %$vmlist) {
6956 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6962 sub restore_proxmox_backup_archive
{
6963 my ($archive, $vmid, $user, $options) = @_;
6965 my $storecfg = PVE
::Storage
::config
();
6967 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6968 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6970 my $fingerprint = $scfg->{fingerprint
};
6971 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6973 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6974 my $namespace = $scfg->{namespace
};
6976 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6977 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6978 local $ENV{PBS_PASSWORD
} = $password;
6979 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6981 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6982 PVE
::Storage
::parse_volname
($storecfg, $archive);
6984 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6986 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6988 my $tmpdir = "/var/tmp/vzdumptmp$$";
6992 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6993 # disable interrupts (always do cleanups)
6997 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6999 # Note: $oldconf is undef if VM does not exists
7000 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7001 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7002 my $new_conf_raw = '';
7004 my $rpcenv = PVE
::RPCEnvironment
::get
();
7005 my $devinfo = {}; # info about drives included in backup
7006 my $virtdev_hash = {}; # info about allocated drives
7014 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7016 my $cfgfn = "$tmpdir/qemu-server.conf";
7017 my $firewall_config_fn = "$tmpdir/fw.conf";
7018 my $index_fn = "$tmpdir/index.json";
7020 my $cmd = "restore";
7022 my $param = [$pbs_backup_name, "index.json", $index_fn];
7023 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7024 my $index = PVE
::Tools
::file_get_contents
($index_fn);
7025 $index = decode_json
($index);
7027 foreach my $info (@{$index->{files
}}) {
7028 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
7030 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
7031 $devinfo->{$devname}->{size
} = $1;
7033 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7038 my $is_qemu_server_backup = scalar(
7039 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
7041 if (!$is_qemu_server_backup) {
7042 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7044 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
7046 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7047 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7049 if ($has_firewall_config) {
7050 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7051 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7053 my $pve_firewall_dir = '/etc/pve/firewall';
7054 mkdir $pve_firewall_dir; # make sure the dir exists
7055 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7058 my $fh = IO
::File-
>new($cfgfn, "r") ||
7059 die "unable to read qemu-server.conf - $!\n";
7061 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7063 # fixme: rate limit?
7065 # create empty/temp config
7066 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
7068 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7071 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7073 foreach my $virtdev (sort keys %$virtdev_hash) {
7074 my $d = $virtdev_hash->{$virtdev};
7075 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7077 # this fails if storage is unavailable
7078 my $volid = $d->{volid
};
7079 my $path = PVE
::Storage
::path
($storecfg, $volid);
7081 # for live-restore we only want to preload the efidisk and TPM state
7082 next if $options->{live
} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7085 if (defined(my $ns = $scfg->{namespace
})) {
7086 @ns_arg = ('--ns', $ns);
7089 my $pbs_restore_cmd = [
7090 '/usr/bin/pbs-restore',
7091 '--repository', $repo,
7094 "$d->{devname}.img.fidx",
7099 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
7100 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
7102 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
7103 push @$pbs_restore_cmd, '--skip-zero';
7106 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
7107 print "restore proxmox backup image: $dbg_cmdstring\n";
7108 run_command
($pbs_restore_cmd);
7111 $fh->seek(0, 0) || die "seek failed - $!\n";
7113 my $cookie = { netcount
=> 0 };
7114 while (defined(my $line = <$fh>)) {
7115 $new_conf_raw .= restore_update_config_line
(
7127 if ($err || !$options->{live
}) {
7128 $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7134 $restore_destroy_volumes->($storecfg, $virtdev_hash);
7138 if ($options->{live
}) {
7139 # keep lock during live-restore
7140 $new_conf_raw .= "\nlock: create";
7143 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $options->{override_conf
});
7144 check_restore_permissions
($rpcenv, $user, $new_conf);
7145 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7147 eval { rescan
($vmid, 1); };
7150 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
7152 if ($options->{live
}) {
7158 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
7160 my $conf = PVE
::QemuConfig-
>load_config($vmid);
7161 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
7163 # these special drives are already restored before start
7164 delete $devinfo->{'drive-efidisk0'};
7165 delete $devinfo->{'drive-tpmstate0-backup'};
7169 keyfile
=> $keyfile,
7170 snapshot
=> $pbs_backup_name,
7171 namespace
=> $namespace,
7173 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7175 PVE
::QemuConfig-
>remove_lock($vmid, "create");
7179 sub pbs_live_restore
{
7180 my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7182 print "starting VM for live-restore\n";
7183 print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7185 my $pbs_backing = {};
7186 for my $ds (keys %$restored_disks) {
7187 $ds =~ m/^drive-(.*)$/;
7189 $pbs_backing->{$confname} = {
7190 repository
=> $opts->{repo
},
7191 snapshot
=> $opts->{snapshot
},
7192 archive
=> "$ds.img.fidx",
7194 $pbs_backing->{$confname}->{keyfile
} = $opts->{keyfile
} if -e
$opts->{keyfile
};
7195 $pbs_backing->{$confname}->{namespace
} = $opts->{namespace
} if defined($opts->{namespace
});
7197 my $drive = parse_drive
($confname, $conf->{$confname});
7198 print "restoring '$ds' to '$drive->{file}'\n";
7201 my $drives_streamed = 0;
7203 # make sure HA doesn't interrupt our restore by stopping the VM
7204 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
7205 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7208 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7209 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7210 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'pbs-backing' => $pbs_backing}, {});
7212 my $qmeventd_fd = register_qmeventd_handle
($vmid);
7214 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7215 # this will effectively collapse the backing image chain consisting of
7216 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7217 # removes itself once all backing images vanish with 'auto-remove=on')
7219 for my $ds (sort keys %$restored_disks) {
7220 my $job_id = "restore-$ds";
7221 mon_cmd
($vmid, 'block-stream',
7222 'job-id' => $job_id,
7225 $jobs->{$job_id} = {};
7228 mon_cmd
($vmid, 'cont');
7229 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
7231 print "restore-drive jobs finished successfully, removing all tracking block devices"
7232 ." to disconnect from Proxmox Backup Server\n";
7234 for my $ds (sort keys %$restored_disks) {
7235 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7238 close($qmeventd_fd);
7244 warn "An error occurred during live-restore: $err\n";
7245 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
7246 die "live-restore failed\n";
7250 sub restore_vma_archive
{
7251 my ($archive, $vmid, $user, $opts, $comp) = @_;
7253 my $readfrom = $archive;
7255 my $cfg = PVE
::Storage
::config
();
7257 my $bwlimit = $opts->{bwlimit
};
7259 my $dbg_cmdstring = '';
7260 my $add_pipe = sub {
7262 push @$commands, $cmd;
7263 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7264 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
7269 if ($archive eq '-') {
7272 # If we use a backup from a PVE defined storage we also consider that
7273 # storage's rate limit:
7274 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
7275 if (defined($volid)) {
7276 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
7277 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
7279 print STDERR
"applying read rate limit: $readlimit\n";
7280 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7281 $add_pipe->($cstream);
7287 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
7288 my $cmd = $info->{decompressor
};
7289 push @$cmd, $readfrom;
7293 my $tmpdir = "/var/tmp/vzdumptmp$$";
7296 # disable interrupts (always do cleanups)
7300 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
7302 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7303 POSIX
::mkfifo
($mapfifo, 0600);
7305 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7307 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7309 my $devinfo = {}; # info about drives included in backup
7310 my $virtdev_hash = {}; # info about allocated drives
7312 my $rpcenv = PVE
::RPCEnvironment
::get
();
7314 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7316 # Note: $oldconf is undef if VM does not exist
7317 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7318 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7319 my $new_conf_raw = '';
7323 my $print_devmap = sub {
7324 my $cfgfn = "$tmpdir/qemu-server.conf";
7326 # we can read the config - that is already extracted
7327 my $fh = IO
::File-
>new($cfgfn, "r") ||
7328 die "unable to read qemu-server.conf - $!\n";
7330 my $fwcfgfn = "$tmpdir/qemu-server.fw";
7332 my $pve_firewall_dir = '/etc/pve/firewall';
7333 mkdir $pve_firewall_dir; # make sure the dir exists
7334 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7337 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7339 foreach my $info (values %{$virtdev_hash}) {
7340 my $storeid = $info->{storeid
};
7341 next if defined($storage_limits{$storeid});
7343 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
7344 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7345 $storage_limits{$storeid} = $limit * 1024;
7348 foreach my $devname (keys %$devinfo) {
7349 die "found no device mapping information for device '$devname'\n"
7350 if !$devinfo->{$devname}->{virtdev
};
7353 # create empty/temp config
7355 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
7356 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7360 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7362 # print restore information to $fifofh
7363 foreach my $virtdev (sort keys %$virtdev_hash) {
7364 my $d = $virtdev_hash->{$virtdev};
7365 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7367 my $storeid = $d->{storeid
};
7368 my $volid = $d->{volid
};
7371 if (my $limit = $storage_limits{$storeid}) {
7372 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7375 my $write_zeros = 1;
7376 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
7380 my $path = PVE
::Storage
::path
($cfg, $volid);
7382 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7384 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7387 $fh->seek(0, 0) || die "seek failed - $!\n";
7389 my $cookie = { netcount
=> 0 };
7390 while (defined(my $line = <$fh>)) {
7391 $new_conf_raw .= restore_update_config_line
(
7410 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7411 local $SIG{ALRM
} = sub { die "got timeout\n"; };
7413 $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7420 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7421 my ($dev_id, $size, $devname) = ($1, $2, $3);
7422 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
7423 } elsif ($line =~ m/^CTIME: /) {
7424 # we correctly received the vma config, so we can disable
7425 # the timeout now for disk allocation
7426 alarm($oldtimeout || 0);
7427 $oldtimeout = undef;
7429 print $fifofh "done\n";
7435 print "restore vma archive: $dbg_cmdstring\n";
7436 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
7440 alarm($oldtimeout) if $oldtimeout;
7442 $restore_deactivate_volumes->($cfg, $virtdev_hash);
7444 close($fifofh) if $fifofh;
7449 $restore_destroy_volumes->($cfg, $virtdev_hash);
7453 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $opts->{override_conf
});
7454 check_restore_permissions
($rpcenv, $user, $new_conf);
7455 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7457 eval { rescan
($vmid, 1); };
7460 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
7463 sub restore_tar_archive
{
7464 my ($archive, $vmid, $user, $opts) = @_;
7466 if (scalar(keys $opts->{override_conf
}->%*) > 0) {
7467 my $keystring = join(' ', keys $opts->{override_conf
}->%*);
7468 die "cannot pass along options ($keystring) when restoring from tar archive\n";
7471 if ($archive ne '-') {
7472 my $firstfile = tar_archive_read_firstfile
($archive);
7473 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7474 if $firstfile ne 'qemu-server.conf';
7477 my $storecfg = PVE
::Storage
::config
();
7479 # avoid zombie disks when restoring over an existing VM -> cleanup first
7480 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7481 # skiplock=1 because qmrestore has set the 'create' lock itself already
7482 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
7483 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
7485 my $tocmd = "/usr/lib/qemu-server/qmextract";
7487 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
7488 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
7489 $tocmd .= ' --prealloc' if $opts->{prealloc
};
7490 $tocmd .= ' --info' if $opts->{info
};
7492 # tar option "xf" does not autodetect compression when read from STDIN,
7493 # so we pipe to zcat
7494 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
7495 PVE
::Tools
::shellquote
("--to-command=$tocmd");
7497 my $tmpdir = "/var/tmp/vzdumptmp$$";
7500 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
7501 local $ENV{VZDUMP_VMID
} = $vmid;
7502 local $ENV{VZDUMP_USER
} = $user;
7504 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7505 my $new_conf_raw = '';
7507 # disable interrupts (always do cleanups)
7511 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7519 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7521 if ($archive eq '-') {
7522 print "extracting archive from STDIN\n";
7523 run_command
($cmd, input
=> "<&STDIN");
7525 print "extracting archive '$archive'\n";
7529 return if $opts->{info
};
7533 my $statfile = "$tmpdir/qmrestore.stat";
7534 if (my $fd = IO
::File-
>new($statfile, "r")) {
7535 while (defined (my $line = <$fd>)) {
7536 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7537 $map->{$1} = $2 if $1;
7539 print STDERR
"unable to parse line in statfile - $line\n";
7545 my $confsrc = "$tmpdir/qemu-server.conf";
7547 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7549 my $cookie = { netcount
=> 0 };
7550 while (defined (my $line = <$srcfd>)) {
7551 $new_conf_raw .= restore_update_config_line
(
7562 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
7568 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7570 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7572 eval { rescan
($vmid, 1); };
7576 sub foreach_storage_used_by_vm
{
7577 my ($conf, $func) = @_;
7581 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7582 my ($ds, $drive) = @_;
7583 return if drive_is_cdrom
($drive);
7585 my $volid = $drive->{file
};
7587 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
7588 $sidhash->{$sid} = $sid if $sid;
7591 foreach my $sid (sort keys %$sidhash) {
7596 my $qemu_snap_storage = {
7599 sub do_snapshots_with_qemu
{
7600 my ($storecfg, $volid, $deviceid) = @_;
7602 return if $deviceid =~ m/tpmstate0/;
7604 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
7605 my $scfg = $storecfg->{ids
}->{$storage_name};
7606 die "could not find storage '$storage_name'\n" if !defined($scfg);
7608 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
7612 if ($volid =~ m/\.(qcow2|qed)$/){
7619 sub qga_check_running
{
7620 my ($vmid, $nowarn) = @_;
7622 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
7624 warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7630 sub template_create
{
7631 my ($vmid, $conf, $disk) = @_;
7633 my $storecfg = PVE
::Storage
::config
();
7635 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7636 my ($ds, $drive) = @_;
7638 return if drive_is_cdrom
($drive);
7639 return if $disk && $ds ne $disk;
7641 my $volid = $drive->{file
};
7642 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
7644 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
7645 $drive->{file
} = $voliddst;
7646 $conf->{$ds} = print_drive
($drive);
7647 PVE
::QemuConfig-
>write_config($vmid, $conf);
7651 sub convert_iscsi_path
{
7654 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
7659 my $initiator_name = get_initiator_name
();
7661 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7662 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7665 die "cannot convert iscsi path '$path', unkown format\n";
7668 sub qemu_img_convert
{
7669 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7671 my $storecfg = PVE
::Storage
::config
();
7672 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
7673 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
7675 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7679 my $src_is_iscsi = 0;
7683 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
7684 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7685 $src_format = qemu_img_format
($src_scfg, $src_volname);
7686 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
7687 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
7688 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
7689 } elsif (-f
$src_volid || -b
$src_volid) {
7690 $src_path = $src_volid;
7691 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7696 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7698 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7699 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7700 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7701 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7704 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7705 push @$cmd, '-l', "snapshot.name=$snapname"
7706 if $snapname && $src_format && $src_format eq "qcow2";
7707 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7708 push @$cmd, '-T', $cachemode if defined($cachemode);
7709 push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7711 if ($src_is_iscsi) {
7712 push @$cmd, '--image-opts';
7713 $src_path = convert_iscsi_path
($src_path);
7714 } elsif ($src_format) {
7715 push @$cmd, '-f', $src_format;
7718 if ($dst_is_iscsi) {
7719 push @$cmd, '--target-image-opts';
7720 $dst_path = convert_iscsi_path
($dst_path);
7722 push @$cmd, '-O', $dst_format;
7725 push @$cmd, $src_path;
7727 if (!$dst_is_iscsi && $is_zero_initialized) {
7728 push @$cmd, "zeroinit:$dst_path";
7730 push @$cmd, $dst_path;
7735 if($line =~ m/\((\S+)\/100\
%\)/){
7737 my $transferred = int($size * $percent / 100);
7738 my $total_h = render_bytes
($size, 1);
7739 my $transferred_h = render_bytes
($transferred, 1);
7741 print "transferred $transferred_h of $total_h ($percent%)\n";
7746 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7748 die "copy failed: $err" if $err;
7751 sub qemu_img_format
{
7752 my ($scfg, $volname) = @_;
7754 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7761 sub qemu_drive_mirror
{
7762 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7764 $jobs = {} if !$jobs;
7768 $jobs->{"drive-$drive"} = {};
7770 if ($dst_volid =~ /^nbd:/) {
7771 $qemu_target = $dst_volid;
7774 my $storecfg = PVE
::Storage
::config
();
7775 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7777 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7779 $format = qemu_img_format
($dst_scfg, $dst_volname);
7781 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7783 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7786 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7787 $opts->{format
} = $format if $format;
7789 if (defined($src_bitmap)) {
7790 $opts->{sync
} = 'incremental';
7791 $opts->{bitmap
} = $src_bitmap;
7792 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7795 if (defined($bwlimit)) {
7796 $opts->{speed
} = $bwlimit * 1024;
7797 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7799 print "drive mirror is starting for drive-$drive\n";
7802 # if a job already runs for this device we get an error, catch it for cleanup
7803 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7805 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7807 die "mirroring error: $err\n";
7810 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7813 # $completion can be either
7814 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7815 # 'cancel': wait until all jobs are ready, block-job-cancel them
7816 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7817 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7818 sub qemu_drive_mirror_monitor
{
7819 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7821 $completion //= 'complete';
7825 my $err_complete = 0;
7827 my $starttime = time ();
7829 die "block job ('$op') timed out\n" if $err_complete > 300;
7831 my $stats = mon_cmd
($vmid, "query-block-jobs");
7834 my $running_jobs = {};
7835 for my $stat (@$stats) {
7836 next if $stat->{type
} ne $op;
7837 $running_jobs->{$stat->{device
}} = $stat;
7840 my $readycounter = 0;
7842 for my $job_id (sort keys %$jobs) {
7843 my $job = $running_jobs->{$job_id};
7845 my $vanished = !defined($job);
7846 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7847 if($complete || ($vanished && $completion eq 'auto')) {
7848 print "$job_id: $op-job finished\n";
7849 delete $jobs->{$job_id};
7853 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7855 my $busy = $job->{busy
};
7856 my $ready = $job->{ready
};
7857 if (my $total = $job->{len
}) {
7858 my $transferred = $job->{offset
} || 0;
7859 my $remaining = $total - $transferred;
7860 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7862 my $duration = $ctime - $starttime;
7863 my $total_h = render_bytes
($total, 1);
7864 my $transferred_h = render_bytes
($transferred, 1);
7866 my $status = sprintf(
7867 "transferred $transferred_h of $total_h ($percent%%) in %s",
7868 render_duration
($duration),
7873 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7875 $status .= ", ready";
7878 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
7879 $jobs->{$job_id}->{ready
} = $ready;
7882 $readycounter++ if $job->{ready
};
7885 last if scalar(keys %$jobs) == 0;
7887 if ($readycounter == scalar(keys %$jobs)) {
7888 print "all '$op' jobs are ready\n";
7890 # do the complete later (or has already been done)
7891 last if $completion eq 'skip' || $completion eq 'auto';
7893 if ($vmiddst && $vmiddst != $vmid) {
7894 my $agent_running = $qga && qga_check_running
($vmid);
7895 if ($agent_running) {
7896 print "freeze filesystem\n";
7897 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
7900 print "suspend vm\n";
7901 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
7905 # if we clone a disk for a new target vm, we don't switch the disk
7906 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
7908 if ($agent_running) {
7909 print "unfreeze filesystem\n";
7910 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
7913 print "resume vm\n";
7914 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
7921 for my $job_id (sort keys %$jobs) {
7922 # try to switch the disk if source and destination are on the same guest
7923 print "$job_id: Completing block job_id...\n";
7926 if ($completion eq 'complete') {
7927 $op = 'block-job-complete';
7928 } elsif ($completion eq 'cancel') {
7929 $op = 'block-job-cancel';
7931 die "invalid completion value: $completion\n";
7933 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
7934 if ($@ =~ m/cannot be completed/) {
7935 print "$job_id: block job cannot be completed, trying again.\n";
7938 print "$job_id: Completed successfully.\n";
7939 $jobs->{$job_id}->{complete
} = 1;
7950 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7951 die "block job ($op) error: $err";
7955 sub qemu_blockjobs_cancel
{
7956 my ($vmid, $jobs) = @_;
7958 foreach my $job (keys %$jobs) {
7959 print "$job: Cancelling block job\n";
7960 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
7961 $jobs->{$job}->{cancel
} = 1;
7965 my $stats = mon_cmd
($vmid, "query-block-jobs");
7967 my $running_jobs = {};
7968 foreach my $stat (@$stats) {
7969 $running_jobs->{$stat->{device
}} = $stat;
7972 foreach my $job (keys %$jobs) {
7974 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
7975 print "$job: Done.\n";
7976 delete $jobs->{$job};
7980 last if scalar(keys %$jobs) == 0;
7986 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
7987 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
7988 my sub clone_disk_check_io_uring
{
7989 my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
7991 return if !$use_drive_mirror;
7993 # Don't complain when not changing storage.
7994 # Assume if it works for the source, it'll work for the target too.
7995 return if $src_storeid eq $dst_storeid;
7997 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7998 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
8000 my $cache_direct = drive_uses_cache_direct
($src_drive);
8002 my $src_uses_io_uring;
8003 if ($src_drive->{aio
}) {
8004 $src_uses_io_uring = $src_drive->{aio
} eq 'io_uring';
8006 $src_uses_io_uring = storage_allows_io_uring_default
($src_scfg, $cache_direct);
8009 die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8010 if $src_uses_io_uring && !storage_allows_io_uring_default
($dst_scfg, $cache_direct);
8014 my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8016 my ($vmid, $running) = $source->@{qw(vmid running)};
8017 my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8019 my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8020 my ($storage, $format) = $dest->@{qw(storage format)};
8022 my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8024 if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8025 die "cloning from/to EFI disk requires EFI disk\n"
8026 if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8027 die "cloning from/to TPM state requires TPM state\n"
8028 if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8030 # This would lead to two device nodes in QEMU pointing to the same backing image!
8031 die "cannot change drive name when cloning disk from/to the same VM\n"
8032 if $use_drive_mirror && $vmid == $newvmid;
8035 die "cannot move TPM state while VM is running\n"
8036 if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8040 print "create " . ($full ?
'full' : 'linked') . " clone of drive ";
8041 print "$src_drivename " if $src_drivename;
8042 print "($drive->{file})\n";
8045 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
8046 push @$newvollist, $newvolid;
8048 my ($src_storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
8049 my $storeid = $storage || $src_storeid;
8051 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
8055 if (drive_is_cloudinit
($drive)) {
8056 $name = "vm-$newvmid-cloudinit";
8057 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8058 if ($scfg->{path
}) {
8059 $name .= ".$dst_format";
8062 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
8063 } elsif ($dst_drivename eq 'efidisk0') {
8064 $size = $efisize or die "internal error - need to specify EFI disk size\n";
8065 } elsif ($dst_drivename eq 'tpmstate0') {
8066 $dst_format = 'raw';
8067 $size = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8069 clone_disk_check_io_uring
($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8071 $size = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
8073 $newvolid = PVE
::Storage
::vdisk_alloc
(
8074 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8076 push @$newvollist, $newvolid;
8078 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
8080 if (drive_is_cloudinit
($drive)) {
8081 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8082 # if this is the case, we have to complete any block-jobs still there from
8083 # previous drive-mirrors
8084 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8085 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
8090 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
8091 if ($use_drive_mirror) {
8092 qemu_drive_mirror
($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8093 $completion, $qga, $bwlimit);
8095 if ($dst_drivename eq 'efidisk0') {
8096 # the relevant data on the efidisk may be smaller than the source
8097 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8098 # that is given by the OVMF_VARS.fd
8099 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
8100 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
8102 my $src_format = (PVE
::Storage
::parse_volname
($storecfg, $drive->{file
}))[6];
8104 # better for Ceph if block size is not too small, see bug #3324
8107 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8109 if ($src_format eq 'qcow2' && $snapname) {
8110 die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8111 if !min_version
(kvm_user_version
(), 6, 2);
8112 push $cmd->@*, '-l', $snapname;
8114 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8117 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8123 my $size = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
8125 my $disk = dclone
($drive);
8126 delete $disk->{format
};
8127 $disk->{file
} = $newvolid;
8128 $disk->{size
} = $size if defined($size);
8133 sub get_running_qemu_version
{
8135 my $res = mon_cmd
($vmid, "query-version");
8136 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8139 sub qemu_use_old_bios_files
{
8140 my ($machine_type) = @_;
8142 return if !$machine_type;
8144 my $use_old_bios_files = undef;
8146 if ($machine_type =~ m/^(\S+)\.pxe$/) {
8148 $use_old_bios_files = 1;
8150 my $version = extract_version
($machine_type, kvm_user_version
());
8151 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8152 # load new efi bios files on migration. So this hack is required to allow
8153 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8154 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8155 $use_old_bios_files = !min_version
($version, 2, 4);
8158 return ($use_old_bios_files, $machine_type);
8161 sub get_efivars_size
{
8162 my ($conf, $efidisk) = @_;
8164 my $arch = get_vm_arch
($conf);
8165 $efidisk //= $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
8166 my $smm = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
8167 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8168 return -s
$ovmf_vars;
8171 sub update_efidisk_size
{
8174 return if !defined($conf->{efidisk0
});
8176 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
8177 $disk->{size
} = get_efivars_size
($conf);
8178 $conf->{efidisk0
} = print_drive
($disk);
8183 sub update_tpmstate_size
{
8186 my $disk = PVE
::QemuServer
::parse_drive
('tpmstate0', $conf->{tpmstate0
});
8187 $disk->{size
} = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8188 $conf->{tpmstate0
} = print_drive
($disk);
8191 sub create_efidisk
($$$$$$$) {
8192 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8194 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8196 my $vars_size_b = -s
$ovmf_vars;
8197 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
8198 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8199 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
8201 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
8202 my $size = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
8204 return ($volid, $size/1024);
8207 sub vm_iothreads_list
{
8210 my $res = mon_cmd
($vmid, 'query-iothreads');
8213 foreach my $iothread (@$res) {
8214 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
8221 my ($conf, $drive) = @_;
8225 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
8227 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
8233 my $controller = int($drive->{index} / $maxdev);
8234 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
8238 return ($maxdev, $controller, $controller_prefix);
8241 sub resolve_dst_disk_format
{
8242 my ($storecfg, $storeid, $src_volname, $format) = @_;
8243 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
8246 # if no target format is specified, use the source disk format as hint
8248 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8249 $format = qemu_img_format
($scfg, $src_volname);
8255 # test if requested format is supported - else use default
8256 my $supported = grep { $_ eq $format } @$validFormats;
8257 $format = $defFormat if !$supported;
8261 # NOTE: if this logic changes, please update docs & possibly gui logic
8262 sub find_vmstate_storage
{
8263 my ($conf, $storecfg) = @_;
8265 # first, return storage from conf if set
8266 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
8268 my ($target, $shared, $local);
8270 foreach_storage_used_by_vm
($conf, sub {
8272 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
8273 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
8274 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
8277 # second, use shared storage where VM has at least one disk
8278 # third, use local storage where VM has at least one disk
8279 # fall back to local storage
8280 $target = $shared // $local // 'local';
8286 my ($uuid, $uuid_str);
8287 UUID
::generate
($uuid);
8288 UUID
::unparse
($uuid, $uuid_str);
8292 sub generate_smbios1_uuid
{
8293 return "uuid=".generate_uuid
();
8299 mon_cmd
($vmid, 'nbd-server-stop', timeout
=> 25);
8302 sub create_reboot_request
{
8304 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8305 or die "failed to create reboot trigger file: $!\n";
8309 sub clear_reboot_request
{
8311 my $path = "/run/qemu-server/$vmid.reboot";
8314 $res = unlink($path);
8315 die "could not remove reboot request for $vmid: $!"
8316 if !$res && $! != POSIX
::ENOENT
;
8321 sub bootorder_from_legacy
{
8322 my ($conf, $bootcfg) = @_;
8324 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
8325 my $bootindex_hash = {};
8327 foreach my $o (split(//, $boot)) {
8328 $bootindex_hash->{$o} = $i*100;
8334 PVE
::QemuConfig-
>foreach_volume($conf, sub {
8335 my ($ds, $drive) = @_;
8337 if (drive_is_cdrom
($drive, 1)) {
8338 if ($bootindex_hash->{d
}) {
8339 $bootorder->{$ds} = $bootindex_hash->{d
};
8340 $bootindex_hash->{d
} += 1;
8342 } elsif ($bootindex_hash->{c
}) {
8343 $bootorder->{$ds} = $bootindex_hash->{c
}
8344 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
8345 $bootindex_hash->{c
} += 1;
8349 if ($bootindex_hash->{n
}) {
8350 for (my $i = 0; $i < $MAX_NETS; $i++) {
8351 my $netname = "net$i";
8352 next if !$conf->{$netname};
8353 $bootorder->{$netname} = $bootindex_hash->{n
};
8354 $bootindex_hash->{n
} += 1;
8361 # Generate default device list for 'boot: order=' property. Matches legacy
8362 # default boot order, but with explicit device names. This is important, since
8363 # the fallback for when neither 'order' nor the old format is specified relies
8364 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8365 sub get_default_bootdevices
{
8371 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
8372 push @ret, $first if $first;
8375 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
8376 push @ret, $first if $first;
8379 for (my $i = 0; $i < $MAX_NETS; $i++) {
8380 my $netname = "net$i";
8381 next if !$conf->{$netname};
8382 push @ret, $netname;
8389 sub device_bootorder
{
8392 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
8394 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
8397 if (!defined($boot) || $boot->{legacy
}) {
8398 $bootorder = bootorder_from_legacy
($conf, $boot);
8399 } elsif ($boot->{order
}) {
8400 my $i = 100; # start at 100 to allow user to insert devices before us with -args
8401 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
8402 $bootorder->{$dev} = $i++;
8409 sub register_qmeventd_handle
{
8413 my $peer = "/var/run/qmeventd.sock";
8418 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
8420 if ($! != EINTR
&& $! != EAGAIN
) {
8421 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8424 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8425 . "after $count retries\n";
8430 # send handshake to mark VM as backing up
8431 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
8433 # return handle to be closed later when inhibit is no longer required
8437 # bash completion helper
8439 sub complete_backup_archives
{
8440 my ($cmdname, $pname, $cvalue) = @_;
8442 my $cfg = PVE
::Storage
::config
();
8446 if ($cvalue =~ m/^([^:]+):/) {
8450 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
8453 foreach my $id (keys %$data) {
8454 foreach my $item (@{$data->{$id}}) {
8455 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8456 push @$res, $item->{volid
} if defined($item->{volid
});
8463 my $complete_vmid_full = sub {
8466 my $idlist = vmstatus
();
8470 foreach my $id (keys %$idlist) {
8471 my $d = $idlist->{$id};
8472 if (defined($running)) {
8473 next if $d->{template
};
8474 next if $running && $d->{status
} ne 'running';
8475 next if !$running && $d->{status
} eq 'running';
8484 return &$complete_vmid_full();
8487 sub complete_vmid_stopped
{
8488 return &$complete_vmid_full(0);
8491 sub complete_vmid_running
{
8492 return &$complete_vmid_full(1);
8495 sub complete_storage
{
8497 my $cfg = PVE
::Storage
::config
();
8498 my $ids = $cfg->{ids
};
8501 foreach my $sid (keys %$ids) {
8502 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
8503 next if !$ids->{$sid}->{content
}->{images
};
8510 sub complete_migration_storage
{
8511 my ($cmd, $param, $current_value, $all_args) = @_;
8513 my $targetnode = @$all_args[1];
8515 my $cfg = PVE
::Storage
::config
();
8516 my $ids = $cfg->{ids
};
8519 foreach my $sid (keys %$ids) {
8520 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
8521 next if !$ids->{$sid}->{content
}->{images
};
8529 my ($vmid, $include_suspended) = @_;
8530 my $qmpstatus = eval {
8531 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
8532 mon_cmd
($vmid, "query-status");
8535 return $qmpstatus && (
8536 $qmpstatus->{status
} eq "paused" ||
8537 $qmpstatus->{status
} eq "prelaunch" ||
8538 ($include_suspended && $qmpstatus->{status
} eq "suspended")
8542 sub check_volume_storage_type
{
8543 my ($storecfg, $vol) = @_;
8545 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
8546 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8547 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
8549 die "storage '$storeid' does not support content-type '$vtype'\n"
8550 if !$scfg->{content
}->{$vtype};
8555 sub add_nets_bridge_fdb
{
8556 my ($conf, $vmid) = @_;
8558 for my $opt (keys %$conf) {
8559 next if $opt !~ m/^net(\d+)$/;
8560 my $iface = "tap${vmid}i$1";
8561 # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8562 my $net = parse_net
($conf->{$opt}, 1) or next;
8564 my $mac = $net->{macaddr
};
8566 log_warn
("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8567 if !file_read_firstline
("/sys/class/net/$iface/brport/learning");
8571 my $bridge = $net->{bridge
};
8573 log_warn
("Interface '$iface' not attached to any bridge.");
8577 PVE
::Network
::SDN
::Zones
::add_bridge_fdb
($iface, $mac, $bridge);
8578 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8579 PVE
::Network
::add_bridge_fdb
($iface, $mac);
8584 sub del_nets_bridge_fdb
{
8585 my ($conf, $vmid) = @_;
8587 for my $opt (keys %$conf) {
8588 next if $opt !~ m/^net(\d+)$/;
8589 my $iface = "tap${vmid}i$1";
8591 my $net = parse_net
($conf->{$opt}) or next;
8592 my $mac = $net->{macaddr
} or next;
8594 my $bridge = $net->{bridge
};
8596 PVE
::Network
::SDN
::Zones
::del_bridge_fdb
($iface, $mac, $bridge);
8597 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8598 PVE
::Network
::del_bridge_fdb
($iface, $mac);
8603 sub create_ifaces_ipams_ips
{
8604 my ($conf, $vmid) = @_;
8606 return if !$have_sdn;
8608 foreach my $opt (keys %$conf) {
8609 if ($opt =~ m/^net(\d+)$/) {
8610 my $value = $conf->{$opt};
8611 my $net = PVE
::QemuServer
::parse_net
($value);
8612 eval { PVE
::Network
::SDN
::Vnets
::add_next_free_cidr
($net->{bridge
}, $conf->{name
}, $net->{macaddr
}, $vmid, undef, 1) };
8618 sub delete_ifaces_ipams_ips
{
8619 my ($conf, $vmid) = @_;
8621 return if !$have_sdn;
8623 foreach my $opt (keys %$conf) {
8624 if ($opt =~ m/^net(\d+)$/) {
8625 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
8626 eval { PVE
::Network
::SDN
::Vnets
::del_ips_from_mac
($net->{bridge
}, $net->{macaddr
}, $conf->{name
}) };