1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday usleep);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
32 use PVE
::DataCenterConfig
;
33 use PVE
::Exception
qw(raise raise_param_exc);
34 use PVE
::Format
qw(render_duration render_bytes);
35 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
37 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
40 use PVE
::RESTEnvironment
qw(log_warn);
41 use PVE
::RPCEnvironment
;
45 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
49 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout windows_version);
50 use PVE
::QemuServer
::Cloudinit
;
51 use PVE
::QemuServer
::CGroup
;
52 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
53 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
54 use PVE
::QemuServer
::Machine
;
55 use PVE
::QemuServer
::Memory
;
56 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
57 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
58 use PVE
::QemuServer
::USB
qw(parse_usb_device);
62 require PVE
::Network
::SDN
::Zones
;
66 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
70 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
71 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
74 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
75 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
78 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
79 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
82 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
83 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
86 "$EDK2_FW_BASE/OVMF_CODE.fd",
87 "$EDK2_FW_BASE/OVMF_VARS.fd",
92 "$EDK2_FW_BASE/AAVMF_CODE.fd",
93 "$EDK2_FW_BASE/AAVMF_VARS.fd",
98 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
100 # Note about locking: we use flock on the config file protect against concurent actions.
101 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
102 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
103 # But you can ignore this kind of lock with the --skiplock flag.
111 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
112 description
=> "Some command save/restore state from this location.",
118 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
119 description
=> "Specifies the QEMU machine type.",
121 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
126 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
129 $nodename_cache //= PVE
::INotify
::nodename
();
130 return $nodename_cache;
137 enum
=> [qw(i6300esb ib700)],
138 description
=> "Watchdog type to emulate.",
139 default => 'i6300esb',
144 enum
=> [qw(reset shutdown poweroff pause debug none)],
145 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
149 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
153 description
=> "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
158 fstrim_cloned_disks
=> {
159 description
=> "Run fstrim after moving a disk or migrating the VM.",
165 description
=> "Select the agent type",
169 enum
=> [qw(virtio isa)],
175 description
=> "Select the VGA type.",
180 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
183 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
195 description
=> "The size of the file in MB.",
199 pattern
=> '[a-zA-Z0-9\-]+',
201 format_description
=> 'string',
202 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
209 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
210 description
=> "Configure an audio device."
214 enum
=> ['spice', 'none'],
217 description
=> "Driver backend for the audio device."
221 my $spice_enhancements_fmt = {
226 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
230 enum
=> ['off', 'all', 'filter'],
233 description
=> "Enable video streaming. Uses compression for detected video streams."
240 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
242 description
=> "The file on the host to gather entropy from. In most cases '/dev/urandom'"
243 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
244 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
245 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
246 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
247 ." a hardware RNG from the host.",
251 description
=> "Maximum bytes of entropy allowed to get injected into the guest every"
252 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
253 ." `0` to disable limiting (potentially dangerous!).",
256 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
257 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
258 # reading from /dev/urandom
263 description
=> "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
264 ." the guest to retrieve another 'max_bytes' of entropy.",
270 my $meta_info_fmt = {
273 description
=> "The guest creation timestamp as UNIX epoch time",
279 description
=> "The QEMU (machine) version from the time this VM was created.",
280 pattern
=> '\d+(\.\d+)+',
289 description
=> "Specifies whether a VM will be started during system bootup.",
295 description
=> "Automatic restart after crash (currently ignored).",
300 type
=> 'string', format
=> 'pve-hotplug-features',
301 description
=> "Selectively enable hotplug features. This is a comma separated list of"
302 ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
303 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
304 ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
306 default => 'network,disk,usb',
311 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
317 description
=> "Lock/unlock the VM.",
318 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
323 description
=> "Limit of CPU usage.",
324 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
325 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
333 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
334 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
335 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
336 ." weights of all the other running VMs.",
339 default => 'cgroup v1: 1024, cgroup v2: 100',
344 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when"
345 ." you use the balloon device.",
352 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
358 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the"
359 ." more memory this VM gets. Number is relative to weights of all other running VMs."
360 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
368 description
=> "Keyboard layout for VNC server. This option is generally not required and"
369 ." is often better handled from within the guest OS.",
370 enum
=> PVE
::Tools
::kvmkeymaplist
(),
375 type
=> 'string', format
=> 'dns-name',
376 description
=> "Set a name for the VM. Only used on the configuration web interface.",
381 description
=> "SCSI controller model",
382 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
388 description
=> "Description for the VM. Shown in the web-interface VM's summary."
389 ." This is saved as comment inside the configuration file.",
390 maxLength
=> 1024 * 8,
395 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
396 description
=> "Specify guest operating system.",
397 verbose_description
=> <<EODESC,
398 Specify guest operating system. This is used to enable special
399 optimization/features for specific operating systems:
402 other;; unspecified OS
403 wxp;; Microsoft Windows XP
404 w2k;; Microsoft Windows 2000
405 w2k3;; Microsoft Windows 2003
406 w2k8;; Microsoft Windows 2008
407 wvista;; Microsoft Windows Vista
408 win7;; Microsoft Windows 7
409 win8;; Microsoft Windows 8/2012/2012r2
410 win10;; Microsoft Windows 10/2016/2019
411 win11;; Microsoft Windows 11/2022
412 l24;; Linux 2.4 Kernel
413 l26;; Linux 2.6 - 5.X Kernel
414 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
419 type
=> 'string', format
=> 'pve-qm-boot',
420 description
=> "Specify guest boot order. Use the 'order=' sub-property as usage with no"
421 ." key or 'legacy=' is deprecated.",
425 type
=> 'string', format
=> 'pve-qm-bootdisk',
426 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
427 pattern
=> '(ide|sata|scsi|virtio)\d+',
432 description
=> "The number of CPUs. Please use option -sockets instead.",
439 description
=> "The number of CPU sockets.",
446 description
=> "The number of cores per socket.",
453 description
=> "Enable/disable NUMA.",
459 description
=> "Enable/disable hugepages memory.",
460 enum
=> [qw(any 2 1024)],
466 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
467 ." after VM shutdown and can be used for subsequent starts.",
472 description
=> "Number of hotplugged vcpus.",
479 description
=> "Enable/disable ACPI.",
484 description
=> "Enable/disable communication with the QEMU Guest Agent and its properties.",
486 format
=> $agent_fmt,
491 description
=> "Enable/disable KVM hardware virtualization.",
497 description
=> "Enable/disable time drift fix.",
503 description
=> "Set the real time clock (RTC) to local time. This is enabled by default if"
504 ." the `ostype` indicates a Microsoft Windows OS.",
509 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
513 type
=> 'string', format
=> $vga_fmt,
514 description
=> "Configure the VGA hardware.",
515 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
516 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
517 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
518 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
519 ." display server. For win* OS you can select how many independent displays you want,"
520 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
521 ." using a serial device as terminal.",
525 type
=> 'string', format
=> 'pve-qm-watchdog',
526 description
=> "Create a virtual hardware watchdog device.",
527 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
528 ." action), the watchdog must be periodically polled by an agent inside the guest or"
529 ." else the watchdog will reset the guest (or execute the respective action specified)",
534 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
535 description
=> "Set the initial date of the real time clock. Valid format for date are:"
536 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
537 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
540 startup
=> get_standard_option
('pve-startup-order'),
544 description
=> "Enable/disable Template.",
550 description
=> "Arbitrary arguments passed to kvm.",
551 verbose_description
=> <<EODESCR,
552 Arbitrary arguments passed to kvm, for example:
554 args: -no-reboot -no-hpet
556 NOTE: this option is for experts only.
563 description
=> "Enable/disable the USB tablet device.",
564 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
565 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
566 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
567 ." may consider disabling this to save some context switches. This is turned off by"
568 ." default if you use spice (`qm set <vmid> --vga qxl`).",
573 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
577 migrate_downtime
=> {
580 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
586 type
=> 'string', format
=> 'pve-qm-ide',
587 typetext
=> '<volume>',
588 description
=> "This is an alias for option -ide2",
592 description
=> "Emulated CPU type.",
594 format
=> 'pve-vm-cpu-conf',
596 parent
=> get_standard_option
('pve-snapshot-name', {
598 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
602 description
=> "Timestamp for snapshots.",
608 type
=> 'string', format
=> 'pve-volume-id',
609 description
=> "Reference to a volume which stores the VM state. This is used internally"
612 vmstatestorage
=> get_standard_option
('pve-storage-id', {
613 description
=> "Default storage for VM state volumes/files.",
616 runningmachine
=> get_standard_option
('pve-qemu-machine', {
617 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
621 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
622 ." internally for snapshots.",
625 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
626 format_description
=> 'QEMU -cpu parameter'
628 machine
=> get_standard_option
('pve-qemu-machine'),
630 description
=> "Virtual processor architecture. Defaults to the host.",
633 enum
=> [qw(x86_64 aarch64)],
636 description
=> "Specify SMBIOS type 1 fields.",
637 type
=> 'string', format
=> 'pve-qm-smbios1',
644 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
645 ." remove disk operations.",
651 enum
=> [ qw(seabios ovmf) ],
652 description
=> "Select BIOS implementation.",
653 default => 'seabios',
657 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
658 format_description
=> 'UUID',
659 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
660 ." to disable explicitly.",
661 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
662 ." value identifier to the guest OS. This allows to notify the guest operating system"
663 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
664 ." execution or creation from a template). The guest operating system notices the"
665 ." change, and is then able to react as appropriate by marking its copies of"
666 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
667 ."Note that auto-creation only works when done through API/CLI create or update methods"
668 .", but not when manually editing the config file.",
669 default => "1 (autogenerated)",
674 format
=> 'pve-volume-id',
676 description
=> "Script that will be executed during various steps in the vms lifetime.",
680 format
=> $ivshmem_fmt,
681 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
687 format
=> $audio_fmt,
688 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
691 spice_enhancements
=> {
693 format
=> $spice_enhancements_fmt,
694 description
=> "Configure additional enhancements for SPICE.",
698 type
=> 'string', format
=> 'pve-tag-list',
699 description
=> 'Tags of the VM. This is only meta information.',
705 description
=> "Configure a VirtIO-based Random Number Generator.",
710 format
=> $meta_info_fmt,
711 description
=> "Some (read-only) meta-information about this guest.",
715 type
=> 'string', format
=> 'pve-cpuset',
716 description
=> "List of host cores used to execute guest processes, for example: 0,5,8-11",
725 description
=> 'Specify a custom file containing all meta data passed to the VM via"
726 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
727 format
=> 'pve-volume-id',
728 format_description
=> 'volume',
733 description
=> 'Specify a custom file containing all network data passed to the VM via'
735 format
=> 'pve-volume-id',
736 format_description
=> 'volume',
741 description
=> 'Specify a custom file containing all user data passed to the VM via'
743 format
=> 'pve-volume-id',
744 format_description
=> 'volume',
749 description
=> 'Specify a custom file containing all vendor data passed to the VM via'
751 format
=> 'pve-volume-id',
752 format_description
=> 'volume',
755 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
757 my $confdesc_cloudinit = {
761 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
762 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
763 .' and `configdrive2` for windows.',
764 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
769 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
770 ." image's configured default user.",
775 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
776 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
777 .' support hashed passwords.',
782 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
784 format
=> 'pve-qm-cicustom',
789 description
=> 'cloud-init: Sets DNS search domains for a container. Create will'
790 .' automatically use the setting from the host if neither searchdomain nor nameserver'
795 type
=> 'string', format
=> 'address-list',
796 description
=> 'cloud-init: Sets DNS server IP address for a container. Create will'
797 .' automatically use the setting from the host if neither searchdomain nor nameserver'
803 format
=> 'urlencoded',
804 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
808 # what about other qemu settings ?
810 #machine => 'string',
823 ##soundhw => 'string',
825 while (my ($k, $v) = each %$confdesc) {
826 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
829 my $MAX_USB_DEVICES = 14;
831 my $MAX_SERIAL_PORTS = 4;
832 my $MAX_PARALLEL_PORTS = 3;
838 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
839 description
=> "CPUs accessing this NUMA node.",
840 format_description
=> "id[-id];...",
844 description
=> "Amount of memory this NUMA node provides.",
849 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
850 description
=> "Host NUMA nodes to use.",
851 format_description
=> "id[-id];...",
856 enum
=> [qw(preferred bind interleave)],
857 description
=> "NUMA allocation policy.",
861 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
864 type
=> 'string', format
=> $numa_fmt,
865 description
=> "NUMA topology.",
867 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
869 for (my $i = 0; $i < $MAX_NUMA; $i++) {
870 $confdesc->{"numa$i"} = $numadesc;
873 my $nic_model_list = [
889 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
891 my $net_fmt_bridge_descr = <<__EOD__;
892 Bridge to attach the network device to. The Proxmox VE standard bridge
895 If you do not specify a bridge, we create a kvm user (NATed) network
896 device, which provides DHCP and DNS services. The following addresses
903 The DHCP server assign addresses to the guest starting from 10.0.2.15.
907 macaddr
=> get_standard_option
('mac-addr', {
908 description
=> "MAC address. That address must be unique withing your network. This is"
909 ." automatically generated if not specified.",
913 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
914 ." very low CPU overhead. If your guest does not support this driver, it is usually"
915 ." best to use 'e1000'.",
916 enum
=> $nic_model_list,
919 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
920 bridge
=> get_standard_option
('pve-bridge-id', {
921 description
=> $net_fmt_bridge_descr,
926 minimum
=> 0, maximum
=> 64,
927 description
=> 'Number of packet queues to be used on the device.',
933 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
938 minimum
=> 1, maximum
=> 4094,
939 description
=> 'VLAN tag to apply to packets on this interface.',
944 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
945 description
=> 'VLAN trunks to pass through this interface.',
946 format_description
=> 'vlanid[;vlanid...]',
951 description
=> 'Whether this interface should be protected by the firewall.',
956 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
961 minimum
=> 1, maximum
=> 65520,
962 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
969 type
=> 'string', format
=> $net_fmt,
970 description
=> "Specify network devices.",
973 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
978 format
=> 'pve-ipv4-config',
979 format_description
=> 'IPv4Format/CIDR',
980 description
=> 'IPv4 address in CIDR format.',
987 format_description
=> 'GatewayIPv4',
988 description
=> 'Default gateway for IPv4 traffic.',
994 format
=> 'pve-ipv6-config',
995 format_description
=> 'IPv6Format/CIDR',
996 description
=> 'IPv6 address in CIDR format.',
1003 format_description
=> 'GatewayIPv6',
1004 description
=> 'Default gateway for IPv6 traffic.',
1009 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
1010 my $ipconfigdesc = {
1012 type
=> 'string', format
=> 'pve-qm-ipconfig',
1013 description
=> <<'EODESCR',
1014 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1016 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1018 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1019 gateway should be provided.
1020 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1021 cloud-init 19.4 or newer.
1023 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1027 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
1029 for (my $i = 0; $i < $MAX_NETS; $i++) {
1030 $confdesc->{"net$i"} = $netdesc;
1031 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1034 foreach my $key (keys %$confdesc_cloudinit) {
1035 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1038 PVE
::JSONSchema
::register_format
('pve-cpuset', \
&pve_verify_cpuset
);
1039 sub pve_verify_cpuset
{
1040 my ($set_text, $noerr) = @_;
1042 my ($count, $members) = eval { PVE
::CpuSet
::parse_cpuset
($set_text) };
1046 die "unable to parse cpuset option\n";
1049 return PVE
::CpuSet-
>new($members)->short_string();
1052 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1053 sub verify_volume_id_or_qm_path
{
1054 my ($volid, $noerr) = @_;
1056 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1058 return verify_volume_id_or_absolute_path
($volid, $noerr);
1061 PVE
::JSONSchema
::register_format
('pve-volume-id-or-absolute-path', \
&verify_volume_id_or_absolute_path
);
1062 sub verify_volume_id_or_absolute_path
{
1063 my ($volid, $noerr) = @_;
1065 return $volid if $volid =~ m
|^/|;
1067 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1078 type
=> 'string', format
=> 'pve-qm-usb-device',
1079 format_description
=> 'HOSTUSBDEVICE|spice',
1080 description
=> <<EODESCR,
1081 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1083 'bus-port(.port)*' (decimal numbers) or
1084 'vendor_id:product_id' (hexadeciaml numbers) or
1087 You can use the 'lsusb -t' command to list existing usb devices.
1089 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1090 machines - use with special care.
1092 The value 'spice' can be used to add a usb redirection devices for spice.
1098 description
=> "Specifies whether if given host option is a USB3 device or port."
1099 ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1100 ." is irrelevant (all devices are plugged into a xhci controller).",
1107 type
=> 'string', format
=> $usb_fmt,
1108 description
=> "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1109 ." l26 or windows > 7, n can be up to 14).",
1111 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1116 pattern
=> '(/dev/.+|socket)',
1117 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1118 verbose_description
=> <<EODESCR,
1119 Create a serial device inside the VM (n is 0 to 3), and pass through a
1120 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1121 host side (use 'qm terminal' to open a terminal connection).
1123 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1124 use with special care.
1126 CAUTION: Experimental! User reported problems with this option.
1133 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1134 description
=> "Map host parallel devices (n is 0 to 2).",
1135 verbose_description
=> <<EODESCR,
1136 Map host parallel devices (n is 0 to 2).
1138 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1139 machines - use with special care.
1141 CAUTION: Experimental! User reported problems with this option.
1145 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1146 $confdesc->{"parallel$i"} = $paralleldesc;
1149 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1150 $confdesc->{"serial$i"} = $serialdesc;
1153 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1154 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1157 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1158 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1161 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1162 $confdesc->{"usb$i"} = $usbdesc;
1170 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1171 . " Deprecated, use 'order=' instead.",
1172 pattern
=> '[acdn]{1,4}',
1173 format_description
=> "[acdn]{1,4}",
1175 # note: this is also the fallback if boot: is not given at all
1181 format
=> 'pve-qm-bootdev-list',
1182 format_description
=> "device[;device...]",
1183 description
=> <<EODESC,
1184 The guest will attempt to boot from devices in the order they appear here.
1186 Disks, optical drives and passed-through storage USB devices will be directly
1187 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1188 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1190 Note that only devices in this list will be marked as bootable and thus loaded
1191 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1192 (e.g. software-raid), you need to specify all of them here.
1194 Overrides the deprecated 'legacy=[acdn]*' value when given.
1198 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1200 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1201 sub verify_bootdev
{
1202 my ($dev, $noerr) = @_;
1204 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1205 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && !$special;
1209 return 0 if $dev !~ m/^$base\d+$/;
1210 return 0 if !$confdesc->{$dev};
1214 return $dev if $check->("net");
1215 return $dev if $check->("usb");
1216 return $dev if $check->("hostpci");
1219 die "invalid boot device '$dev'\n";
1222 sub print_bootorder
{
1224 return "" if !@$devs;
1225 my $data = { order
=> join(';', @$devs) };
1226 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1229 my $kvm_api_version = 0;
1232 return $kvm_api_version if $kvm_api_version;
1234 open my $fh, '<', '/dev/kvm' or return;
1236 # 0xae00 => KVM_GET_API_VERSION
1237 $kvm_api_version = ioctl($fh, 0xae00, 0);
1240 return $kvm_api_version;
1243 my $kvm_user_version = {};
1246 sub kvm_user_version
{
1249 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1250 my $st = stat($binary);
1252 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1253 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1254 $cachedmtime == $st->mtime;
1256 $kvm_user_version->{$binary} = 'unknown';
1257 $kvm_mtime->{$binary} = $st->mtime;
1261 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1262 $kvm_user_version->{$binary} = $2;
1266 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1269 return $kvm_user_version->{$binary};
1272 my sub extract_version
{
1273 my ($machine_type, $version) = @_;
1274 $version = kvm_user_version
() if !defined($version);
1275 return PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1278 sub kernel_has_vhost_net
{
1279 return -c
'/dev/vhost-net';
1284 return defined($confdesc->{$key});
1288 sub get_cdrom_path
{
1290 return $cdrom_path if $cdrom_path;
1292 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1293 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1294 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1298 my ($storecfg, $vmid, $cdrom) = @_;
1300 if ($cdrom eq 'cdrom') {
1301 return get_cdrom_path
();
1302 } elsif ($cdrom eq 'none') {
1304 } elsif ($cdrom =~ m
|^/|) {
1307 return PVE
::Storage
::path
($storecfg, $cdrom);
1311 # try to convert old style file names to volume IDs
1312 sub filename_to_volume_id
{
1313 my ($vmid, $file, $media) = @_;
1315 if (!($file eq 'none' || $file eq 'cdrom' ||
1316 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1318 return if $file =~ m
|/|;
1320 if ($media && $media eq 'cdrom') {
1321 $file = "local:iso/$file";
1323 $file = "local:$vmid/$file";
1330 sub verify_media_type
{
1331 my ($opt, $vtype, $media) = @_;
1336 if ($media eq 'disk') {
1338 } elsif ($media eq 'cdrom') {
1341 die "internal error";
1344 return if ($vtype eq $etype);
1346 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1349 sub cleanup_drive_path
{
1350 my ($opt, $storecfg, $drive) = @_;
1352 # try to convert filesystem paths to volume IDs
1354 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1355 ($drive->{file
} !~ m
|^/dev/.+|) &&
1356 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1357 ($drive->{file
} !~ m/^\d+$/)) {
1358 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1359 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1361 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1362 verify_media_type
($opt, $vtype, $drive->{media
});
1363 $drive->{file
} = $volid;
1366 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1369 sub parse_hotplug_features
{
1374 return $res if $data eq '0';
1376 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1378 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1379 if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1382 die "invalid hotplug feature '$feature'\n";
1388 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1389 sub pve_verify_hotplug_features
{
1390 my ($value, $noerr) = @_;
1392 return $value if parse_hotplug_features
($value);
1396 die "unable to parse hotplug option\n";
1400 my($fh, $noerr) = @_;
1403 my $SG_GET_VERSION_NUM = 0x2282;
1405 my $versionbuf = "\x00" x
8;
1406 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1408 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1411 my $version = unpack("I", $versionbuf);
1412 if ($version < 30000) {
1413 die "scsi generic interface too old\n" if !$noerr;
1417 my $buf = "\x00" x
36;
1418 my $sensebuf = "\x00" x
8;
1419 my $cmd = pack("C x3 C x1", 0x12, 36);
1421 # see /usr/include/scsi/sg.h
1422 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1425 $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1428 $ret = ioctl($fh, $SG_IO, $packet);
1430 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1434 my @res = unpack($sg_io_hdr_t, $packet);
1435 if ($res[17] || $res[18]) {
1436 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1441 $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1443 $res->{removable
} = $res->{removable
} & 128 ?
1 : 0;
1444 $res->{type
} &= 0x1F;
1452 my $fh = IO
::File-
>new("+<$path") || return;
1453 my $res = scsi_inquiry
($fh, 1);
1459 sub print_tabletdevice_full
{
1460 my ($conf, $arch) = @_;
1462 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1464 # we use uhci for old VMs because tablet driver was buggy in older qemu
1466 if ($q35 || $arch eq 'aarch64') {
1472 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1475 sub print_keyboarddevice_full
{
1476 my ($conf, $arch) = @_;
1478 return if $arch ne 'aarch64';
1480 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1483 my sub get_drive_id
{
1485 return "$drive->{interface}$drive->{index}";
1488 sub print_drivedevice_full
{
1489 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1494 my $drive_id = get_drive_id
($drive);
1495 if ($drive->{interface
} eq 'virtio') {
1496 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1497 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1498 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1499 } elsif ($drive->{interface
} eq 'scsi') {
1501 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1502 my $unit = $drive->{index} % $maxdev;
1503 my $devicetype = 'hd';
1505 if (drive_is_cdrom
($drive)) {
1508 if ($drive->{file
} =~ m
|^/|) {
1509 $path = $drive->{file
};
1510 if (my $info = path_is_scsi
($path)) {
1511 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1512 $devicetype = 'block';
1513 } elsif ($info->{type
} == 1) { # tape
1514 $devicetype = 'generic';
1518 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1521 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1522 my $version = extract_version
($machine_type, kvm_user_version
());
1523 if ($path =~ m/^iscsi\:\/\
// &&
1524 !min_version
($version, 4, 1)) {
1525 $devicetype = 'generic';
1529 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1530 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1532 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1533 .",lun=$drive->{index}";
1535 $device .= ",drive=drive-$drive_id,id=$drive_id";
1537 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1538 $device .= ",rotation_rate=1";
1540 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1542 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1543 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1544 my $controller = int($drive->{index} / $maxdev);
1545 my $unit = $drive->{index} % $maxdev;
1546 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1548 $device = "ide-$devicetype";
1549 if ($drive->{interface
} eq 'ide') {
1550 $device .= ",bus=ide.$controller,unit=$unit";
1552 $device .= ",bus=ahci$controller.$unit";
1554 $device .= ",drive=drive-$drive_id,id=$drive_id";
1556 if ($devicetype eq 'hd') {
1557 if (my $model = $drive->{model
}) {
1558 $model = URI
::Escape
::uri_unescape
($model);
1559 $device .= ",model=$model";
1561 if ($drive->{ssd
}) {
1562 $device .= ",rotation_rate=1";
1565 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1566 } elsif ($drive->{interface
} eq 'usb') {
1568 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1570 die "unsupported interface type";
1573 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1575 if (my $serial = $drive->{serial
}) {
1576 $serial = URI
::Escape
::uri_unescape
($serial);
1577 $device .= ",serial=$serial";
1584 sub get_initiator_name
{
1587 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1588 while (defined(my $line = <$fh>)) {
1589 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1598 sub print_drive_commandline_full
{
1599 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1602 my $volid = $drive->{file
};
1603 my $format = $drive->{format
};
1604 my $drive_id = get_drive_id
($drive);
1606 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1607 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1609 if (drive_is_cdrom
($drive)) {
1610 $path = get_iso_path
($storecfg, $vmid, $volid);
1611 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1614 $path = PVE
::Storage
::path
($storecfg, $volid);
1615 $format //= qemu_img_format
($scfg, $volname);
1622 my $is_rbd = $path =~ m/^rbd:/;
1625 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1626 foreach my $o (@qemu_drive_options) {
1627 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1630 # snapshot only accepts on|off
1631 if (defined($drive->{snapshot
})) {
1632 my $v = $drive->{snapshot
} ?
'on' : 'off';
1633 $opts .= ",snapshot=$v";
1636 if (defined($drive->{ro
})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1637 $opts .= ",readonly=" . ($drive->{ro
} ?
'on' : 'off');
1640 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1641 my ($dir, $qmpname) = @$type;
1642 if (my $v = $drive->{"mbps$dir"}) {
1643 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1645 if (my $v = $drive->{"mbps${dir}_max"}) {
1646 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1648 if (my $v = $drive->{"bps${dir}_max_length"}) {
1649 $opts .= ",throttling.bps$qmpname-max-length=$v";
1651 if (my $v = $drive->{"iops${dir}"}) {
1652 $opts .= ",throttling.iops$qmpname=$v";
1654 if (my $v = $drive->{"iops${dir}_max"}) {
1655 $opts .= ",throttling.iops$qmpname-max=$v";
1657 if (my $v = $drive->{"iops${dir}_max_length"}) {
1658 $opts .= ",throttling.iops$qmpname-max-length=$v";
1663 $format = "rbd" if $is_rbd;
1664 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1666 $opts .= ",format=alloc-track,file.driver=$format";
1668 $opts .= ",format=$format";
1671 my $cache_direct = 0;
1673 if (my $cache = $drive->{cache
}) {
1674 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1675 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1676 $opts .= ",cache=none";
1680 # io_uring with cache mode writeback or writethrough on krbd will hang...
1681 my $rbd_no_io_uring = $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1683 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1684 # sometimes, just plain disable...
1685 my $lvm_no_io_uring = $scfg && $scfg->{type
} eq 'lvm';
1687 # io_uring causes problems when used with CIFS since kernel 5.15
1688 # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1689 my $cifs_no_io_uring = $scfg && $scfg->{type
} eq 'cifs';
1691 if (!$drive->{aio
}) {
1692 if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1693 # io_uring supports all cache modes
1694 $opts .= ",aio=io_uring";
1696 # aio native works only with O_DIRECT
1698 $opts .= ",aio=native";
1700 $opts .= ",aio=threads";
1705 if (!drive_is_cdrom
($drive)) {
1707 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1708 $detectzeroes = 'off';
1709 } elsif ($drive->{discard
}) {
1710 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1712 # This used to be our default with discard not being specified:
1713 $detectzeroes = 'on';
1716 # note: 'detect-zeroes' works per blockdev and we want it to persist
1717 # after the alloc-track is removed, so put it on 'file' directly
1718 my $dz_param = $pbs_name ?
"file.detect-zeroes" : "detect-zeroes";
1719 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1723 $opts .= ",backing=$pbs_name";
1724 $opts .= ",auto-remove=on";
1727 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1728 my $file_param = "file";
1730 # non-rbd drivers require the underlying file to be a seperate block
1731 # node, so add a second .file indirection
1732 $file_param .= ".file" if !$is_rbd;
1733 $file_param .= ".filename";
1735 my $pathinfo = $path ?
"$file_param=$path," : '';
1737 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1740 sub print_pbs_blockdev
{
1741 my ($pbs_conf, $pbs_name) = @_;
1742 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1743 $blockdev .= ",repository=$pbs_conf->{repository}";
1744 $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace
};
1745 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1746 $blockdev .= ",archive=$pbs_conf->{archive}";
1747 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1751 sub print_netdevice_full
{
1752 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1754 my $device = $net->{model
};
1755 if ($net->{model
} eq 'virtio') {
1756 $device = 'virtio-net-pci';
1759 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1760 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1761 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1762 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1763 # and out of each queue plus one config interrupt and control vector queue
1764 my $vectors = $net->{queues
} * 2 + 2;
1765 $tmpstr .= ",vectors=$vectors,mq=on";
1766 if (min_version
($machine_version, 7, 1)) {
1767 $tmpstr .= ",packed=on";
1771 if (min_version
($machine_version, 7, 1) && $net->{model
} eq 'virtio'){
1772 $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1775 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1777 if (my $mtu = $net->{mtu
}) {
1778 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1779 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1782 } elsif ($mtu < 576) {
1783 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1784 } elsif ($mtu > $bridge_mtu) {
1785 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1787 $tmpstr .= ",host_mtu=$mtu";
1789 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1793 if ($use_old_bios_files) {
1795 if ($device eq 'virtio-net-pci') {
1796 $romfile = 'pxe-virtio.rom';
1797 } elsif ($device eq 'e1000') {
1798 $romfile = 'pxe-e1000.rom';
1799 } elsif ($device eq 'e1000e') {
1800 $romfile = 'pxe-e1000e.rom';
1801 } elsif ($device eq 'ne2k') {
1802 $romfile = 'pxe-ne2k_pci.rom';
1803 } elsif ($device eq 'pcnet') {
1804 $romfile = 'pxe-pcnet.rom';
1805 } elsif ($device eq 'rtl8139') {
1806 $romfile = 'pxe-rtl8139.rom';
1808 $tmpstr .= ",romfile=$romfile" if $romfile;
1814 sub print_netdev_full
{
1815 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1818 if ($netid =~ m/^net(\d+)$/) {
1822 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1824 my $ifname = "tap${vmid}i$i";
1826 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1827 die "interface name '$ifname' is too long (max 15 character)\n"
1828 if length($ifname) >= 16;
1830 my $vhostparam = '';
1831 if (is_native
($arch)) {
1832 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1835 my $vmname = $conf->{name
} || "vm$vmid";
1838 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1840 if ($net->{bridge
}) {
1841 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1842 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1844 $netdev = "type=user,id=$netid,hostname=$vmname";
1847 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1853 'cirrus' => 'cirrus-vga',
1855 'vmware' => 'vmware-svga',
1856 'virtio' => 'virtio-vga',
1857 'virtio-gl' => 'virtio-vga-gl',
1860 sub print_vga_device
{
1861 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1863 my $type = $vga_map->{$vga->{type
}};
1864 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1865 $type = 'virtio-gpu';
1867 my $vgamem_mb = $vga->{memory
};
1869 my $max_outputs = '';
1871 $type = $id ?
'qxl' : 'qxl-vga';
1873 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1874 # set max outputs so linux can have up to 4 qxl displays with one device
1875 if (min_version
($machine_version, 4, 1)) {
1876 $max_outputs = ",max_outputs=4";
1881 die "no devicetype for $vga->{type}\n" if !$type;
1885 if ($vga->{type
} =~ /^virtio/) {
1886 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1887 $memory = ",max_hostmem=$bytes";
1889 # from https://www.spice-space.org/multiple-monitors.html
1890 $memory = ",vgamem_mb=$vga->{memory}";
1891 my $ram = $vgamem_mb * 4;
1892 my $vram = $vgamem_mb * 2;
1893 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1895 $memory = ",vgamem_mb=$vga->{memory}";
1897 } elsif ($qxlnum && $id) {
1898 $memory = ",ram_size=67108864,vram_size=33554432";
1902 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1903 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1906 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1907 my $vgaid = "vga" . ($id // '');
1909 if ($q35 && $vgaid eq 'vga') {
1910 # the first display uses pcie.0 bus on q35 machines
1911 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1913 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1916 if ($vga->{type
} eq 'virtio-gl') {
1917 my $base = '/usr/lib/x86_64-linux-gnu/lib';
1918 die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1919 if !-e
"${base}EGL.so.1" || !-e
"${base}GL.so.1";
1921 die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1922 if !PVE
::Tools
::dir_glob_regex
('/dev/dri/', "renderD.*");
1925 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1928 sub parse_number_sets
{
1931 foreach my $part (split(/;/, $set)) {
1932 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1933 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1934 push @$res, [ $1, $2 ];
1936 die "invalid range: $part\n";
1945 my $res = parse_property_string
($numa_fmt, $data);
1946 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1947 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1951 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1953 my ($data, $disable_mac_autogen) = @_;
1955 my $res = eval { parse_property_string
($net_fmt, $data) };
1960 if (!defined($res->{macaddr
}) && !$disable_mac_autogen) {
1961 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1962 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1967 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1968 sub parse_ipconfig
{
1971 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1977 if ($res->{gw
} && !$res->{ip
}) {
1978 warn 'gateway specified without specifying an IP address';
1981 if ($res->{gw6
} && !$res->{ip6
}) {
1982 warn 'IPv6 gateway specified without specifying an IPv6 address';
1985 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1986 warn 'gateway specified together with DHCP';
1989 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1991 warn "IPv6 gateway specified together with $res->{ip6} address";
1995 if (!$res->{ip
} && !$res->{ip6
}) {
1996 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
2005 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
2008 sub add_random_macs
{
2009 my ($settings) = @_;
2011 foreach my $opt (keys %$settings) {
2012 next if $opt !~ m/^net(\d+)$/;
2013 my $net = parse_net
($settings->{$opt});
2015 $settings->{$opt} = print_net
($net);
2019 sub vm_is_volid_owner
{
2020 my ($storecfg, $vmid, $volid) = @_;
2022 if ($volid !~ m
|^/|) {
2024 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
2025 if ($owner && ($owner == $vmid)) {
2033 sub vmconfig_register_unused_drive
{
2034 my ($storecfg, $vmid, $conf, $drive) = @_;
2036 if (drive_is_cloudinit
($drive)) {
2037 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2039 delete $conf->{cloudinit
};
2040 } elsif (!drive_is_cdrom
($drive)) {
2041 my $volid = $drive->{file
};
2042 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2043 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2048 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2052 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2053 format_description
=> 'UUID',
2054 description
=> "Set SMBIOS1 UUID.",
2059 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2060 format_description
=> 'Base64 encoded string',
2061 description
=> "Set SMBIOS1 version.",
2066 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2067 format_description
=> 'Base64 encoded string',
2068 description
=> "Set SMBIOS1 serial number.",
2073 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2074 format_description
=> 'Base64 encoded string',
2075 description
=> "Set SMBIOS1 manufacturer.",
2080 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2081 format_description
=> 'Base64 encoded string',
2082 description
=> "Set SMBIOS1 product ID.",
2087 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2088 format_description
=> 'Base64 encoded string',
2089 description
=> "Set SMBIOS1 SKU string.",
2094 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2095 format_description
=> 'Base64 encoded string',
2096 description
=> "Set SMBIOS1 family string.",
2101 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2109 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2116 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2119 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2121 sub parse_watchdog
{
2126 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2131 sub parse_guest_agent
{
2134 return {} if !defined($conf->{agent
});
2136 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2139 # if the agent is disabled ignore the other potentially set properties
2140 return {} if !$res->{enabled
};
2145 my ($conf, $key) = @_;
2146 return undef if !defined($conf->{agent
});
2148 my $agent = parse_guest_agent
($conf);
2149 return $agent->{$key};
2155 return {} if !$value;
2156 my $res = eval { parse_property_string
($vga_fmt, $value) };
2166 my $res = eval { parse_property_string
($rng_fmt, $value) };
2171 sub parse_meta_info
{
2176 my $res = eval { parse_property_string
($meta_info_fmt, $value) };
2181 sub new_meta_info_string
{
2182 my () = @_; # for now do not allow to override any value
2184 return PVE
::JSONSchema
::print_property_string
(
2186 'creation-qemu' => kvm_user_version
(),
2187 ctime
=> "". int(time()),
2193 sub qemu_created_version_fixups
{
2194 my ($conf, $forcemachine, $kvmver) = @_;
2196 my $meta = parse_meta_info
($conf->{meta
}) // {};
2197 my $forced_vers = PVE
::QemuServer
::Machine
::extract_version
($forcemachine);
2199 # check if we need to apply some handling for VMs that always use the latest machine version but
2200 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2201 # would be required (we do not want to pin machine version for non-windows OS type)
2203 (!defined($conf->{machine
}) || $conf->{machine
} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2204 && (!defined($meta->{'creation-qemu'}) || !min_version
($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2205 && (!$forced_vers || min_version
($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2206 && min_version
($kvmver, 6, 1) # only need to apply the change since 6.1
2208 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2209 if ($q35 && $conf->{ostype
} && $conf->{ostype
} eq 'l26') {
2210 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2211 # and thus with the predictable interface naming of systemd
2212 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2218 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2219 sub verify_usb_device
{
2220 my ($value, $noerr) = @_;
2222 return $value if parse_usb_device
($value);
2226 die "unable to parse usb device\n";
2229 # add JSON properties for create and set function
2230 sub json_config_properties
{
2231 my ($prop, $with_disk_alloc) = @_;
2233 my $skip_json_config_opts = {
2237 runningmachine
=> 1,
2242 foreach my $opt (keys %$confdesc) {
2243 next if $skip_json_config_opts->{$opt};
2245 if ($with_disk_alloc && is_valid_drivename
($opt)) {
2246 $prop->{$opt} = $PVE::QemuServer
::Drive
::drivedesc_hash_with_alloc-
>{$opt};
2248 $prop->{$opt} = $confdesc->{$opt};
2255 # Properties that we can read from an OVF file
2256 sub json_ovf_properties
{
2259 for my $device (PVE
::QemuServer
::Drive
::valid_drive_names
()) {
2260 $prop->{$device} = {
2262 format
=> 'pve-volume-id-or-absolute-path',
2263 description
=> "Disk image that gets imported to $device",
2270 description
=> "The number of CPU cores.",
2275 description
=> "Amount of RAM for the VM in MB.",
2280 description
=> "Name of the VM.",
2287 # return copy of $confdesc_cloudinit to generate documentation
2288 sub cloudinit_config_properties
{
2290 return dclone
($confdesc_cloudinit);
2293 sub cloudinit_pending_properties
{
2295 map { $_ => 1 } keys $confdesc_cloudinit->%*,
2298 $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2303 my ($key, $value) = @_;
2305 die "unknown setting '$key'\n" if !$confdesc->{$key};
2307 my $type = $confdesc->{$key}->{type
};
2309 if (!defined($value)) {
2310 die "got undefined value\n";
2313 if ($value =~ m/[\n\r]/) {
2314 die "property contains a line feed\n";
2317 if ($type eq 'boolean') {
2318 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2319 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2320 die "type check ('boolean') failed - got '$value'\n";
2321 } elsif ($type eq 'integer') {
2322 return int($1) if $value =~ m/^(\d+)$/;
2323 die "type check ('integer') failed - got '$value'\n";
2324 } elsif ($type eq 'number') {
2325 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2326 die "type check ('number') failed - got '$value'\n";
2327 } elsif ($type eq 'string') {
2328 if (my $fmt = $confdesc->{$key}->{format
}) {
2329 PVE
::JSONSchema
::check_format
($fmt, $value);
2332 $value =~ s/^\"(.*)\"$/$1/;
2335 die "internal error"
2340 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2342 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2344 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2346 if ($conf->{template
}) {
2347 # check if any base image is still used by a linked clone
2348 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2349 my ($ds, $drive) = @_;
2350 return if drive_is_cdrom
($drive);
2352 my $volid = $drive->{file
};
2353 return if !$volid || $volid =~ m
|^/|;
2355 die "base volume '$volid' is still in use by linked cloned\n"
2356 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2362 my $remove_owned_drive = sub {
2363 my ($ds, $drive) = @_;
2364 return if drive_is_cdrom
($drive, 1);
2366 my $volid = $drive->{file
};
2367 return if !$volid || $volid =~ m
|^/|;
2368 return if $volids->{$volid};
2370 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2371 return if !$path || !$owner || ($owner != $vmid);
2373 $volids->{$volid} = 1;
2374 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2375 warn "Could not remove disk '$volid', check manually: $@" if $@;
2378 # only remove disks owned by this VM (referenced in the config)
2379 my $include_opts = {
2380 include_unused
=> 1,
2381 extra_keys
=> ['vmstate'],
2383 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2385 for my $snap (values %{$conf->{snapshots
}}) {
2386 next if !defined($snap->{vmstate
});
2387 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2388 next if !defined($drive);
2389 $remove_owned_drive->('vmstate', $drive);
2392 PVE
::QemuConfig-
>foreach_volume_full($conf->{pending
}, $include_opts, $remove_owned_drive);
2394 if ($purge_unreferenced) { # also remove unreferenced disk
2395 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2396 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2397 my ($volid, $sid, $volname, $d) = @_;
2398 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2403 if (defined $replacement_conf) {
2404 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2406 PVE
::QemuConfig-
>destroy_config($vmid);
2410 sub parse_vm_config
{
2411 my ($filename, $raw, $strict) = @_;
2413 return if !defined($raw);
2416 digest
=> Digest
::SHA
::sha1_hex
($raw),
2422 my $handle_error = sub {
2432 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2433 || die "got strange filename '$filename'";
2439 my $finish_description = sub {
2440 if (defined($descr)) {
2442 $conf->{description
} = $descr;
2448 my @lines = split(/\n/, $raw);
2449 foreach my $line (@lines) {
2450 next if $line =~ m/^\s*$/;
2452 if ($line =~ m/^\[PENDING\]\s*$/i) {
2453 $section = 'pending';
2454 $finish_description->();
2455 $conf = $res->{$section} = {};
2457 } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2458 $section = 'cloudinit';
2459 $finish_description->();
2460 $conf = $res->{$section} = {};
2463 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2465 $finish_description->();
2466 $conf = $res->{snapshots
}->{$section} = {};
2470 if ($line =~ m/^\#(.*)$/) {
2471 $descr = '' if !defined($descr);
2472 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2476 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2477 $descr = '' if !defined($descr);
2478 $descr .= PVE
::Tools
::decode_text
($2);
2479 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2480 $conf->{snapstate
} = $1;
2481 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2484 $conf->{$key} = $value;
2485 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2487 if ($section eq 'pending') {
2488 $conf->{delete} = $value; # we parse this later
2490 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2492 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2495 if ($section eq 'cloudinit') {
2496 # ignore validation only used for informative purpose
2497 $conf->{$key} = $value;
2500 eval { $value = check_type
($key, $value); };
2502 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2504 $key = 'ide2' if $key eq 'cdrom';
2505 my $fmt = $confdesc->{$key}->{format
};
2506 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2507 my $v = parse_drive
($key, $value);
2508 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2509 $v->{file
} = $volid;
2510 $value = print_drive
($v);
2512 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2517 $conf->{$key} = $value;
2520 $handle_error->("vm $vmid - unable to parse config: $line\n");
2524 $finish_description->();
2525 delete $res->{snapstate
}; # just to be sure
2530 sub write_vm_config
{
2531 my ($filename, $conf) = @_;
2533 delete $conf->{snapstate
}; # just to be sure
2535 if ($conf->{cdrom
}) {
2536 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2537 $conf->{ide2
} = $conf->{cdrom
};
2538 delete $conf->{cdrom
};
2541 # we do not use 'smp' any longer
2542 if ($conf->{sockets
}) {
2543 delete $conf->{smp
};
2544 } elsif ($conf->{smp
}) {
2545 $conf->{sockets
} = $conf->{smp
};
2546 delete $conf->{cores
};
2547 delete $conf->{smp
};
2550 my $used_volids = {};
2552 my $cleanup_config = sub {
2553 my ($cref, $pending, $snapname) = @_;
2555 foreach my $key (keys %$cref) {
2556 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2557 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2558 my $value = $cref->{$key};
2559 if ($key eq 'delete') {
2560 die "propertry 'delete' is only allowed in [PENDING]\n"
2562 # fixme: check syntax?
2565 eval { $value = check_type
($key, $value); };
2566 die "unable to parse value of '$key' - $@" if $@;
2568 $cref->{$key} = $value;
2570 if (!$snapname && is_valid_drivename
($key)) {
2571 my $drive = parse_drive
($key, $value);
2572 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2577 &$cleanup_config($conf);
2579 &$cleanup_config($conf->{pending
}, 1);
2581 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2582 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2583 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2586 # remove 'unusedX' settings if we re-add a volume
2587 foreach my $key (keys %$conf) {
2588 my $value = $conf->{$key};
2589 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2590 delete $conf->{$key};
2594 my $generate_raw_config = sub {
2595 my ($conf, $pending) = @_;
2599 # add description as comment to top of file
2600 if (defined(my $descr = $conf->{description
})) {
2602 foreach my $cl (split(/\n/, $descr)) {
2603 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2606 $raw .= "#\n" if $pending;
2610 foreach my $key (sort keys %$conf) {
2611 next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2612 $raw .= "$key: $conf->{$key}\n";
2617 my $raw = &$generate_raw_config($conf);
2619 if (scalar(keys %{$conf->{pending
}})){
2620 $raw .= "\n[PENDING]\n";
2621 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2624 if (scalar(keys %{$conf->{cloudinit
}}) && PVE
::QemuConfig-
>has_cloudinit($conf)){
2625 $raw .= "\n[special:cloudinit]\n";
2626 $raw .= &$generate_raw_config($conf->{cloudinit
});
2629 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2630 $raw .= "\n[$snapname]\n";
2631 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2641 # we use static defaults from our JSON schema configuration
2642 foreach my $key (keys %$confdesc) {
2643 if (defined(my $default = $confdesc->{$key}->{default})) {
2644 $res->{$key} = $default;
2652 my $vmlist = PVE
::Cluster
::get_vmlist
();
2654 return $res if !$vmlist || !$vmlist->{ids
};
2655 my $ids = $vmlist->{ids
};
2656 my $nodename = nodename
();
2658 foreach my $vmid (keys %$ids) {
2659 my $d = $ids->{$vmid};
2660 next if !$d->{node
} || $d->{node
} ne $nodename;
2661 next if !$d->{type
} || $d->{type
} ne 'qemu';
2662 $res->{$vmid}->{exists} = 1;
2667 # test if VM uses local resources (to prevent migration)
2668 sub check_local_resources
{
2669 my ($conf, $noerr) = @_;
2673 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2674 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2676 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2678 foreach my $k (keys %$conf) {
2679 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2680 # sockets are safe: they will recreated be on the target side post-migrate
2681 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2682 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2685 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2690 # check if used storages are available on all nodes (use by migrate)
2691 sub check_storage_availability
{
2692 my ($storecfg, $conf, $node) = @_;
2694 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2695 my ($ds, $drive) = @_;
2697 my $volid = $drive->{file
};
2700 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2703 # check if storage is available on both nodes
2704 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2705 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2707 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2709 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2710 if !$scfg->{content
}->{$vtype};
2714 # list nodes where all VM images are available (used by has_feature API)
2716 my ($conf, $storecfg) = @_;
2718 my $nodelist = PVE
::Cluster
::get_nodelist
();
2719 my $nodehash = { map { $_ => 1 } @$nodelist };
2720 my $nodename = nodename
();
2722 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2723 my ($ds, $drive) = @_;
2725 my $volid = $drive->{file
};
2728 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2730 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2731 if ($scfg->{disable
}) {
2733 } elsif (my $avail = $scfg->{nodes
}) {
2734 foreach my $node (keys %$nodehash) {
2735 delete $nodehash->{$node} if !$avail->{$node};
2737 } elsif (!$scfg->{shared
}) {
2738 foreach my $node (keys %$nodehash) {
2739 delete $nodehash->{$node} if $node ne $nodename
2748 sub check_local_storage_availability
{
2749 my ($conf, $storecfg) = @_;
2751 my $nodelist = PVE
::Cluster
::get_nodelist
();
2752 my $nodehash = { map { $_ => {} } @$nodelist };
2754 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2755 my ($ds, $drive) = @_;
2757 my $volid = $drive->{file
};
2760 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2762 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2764 if ($scfg->{disable
}) {
2765 foreach my $node (keys %$nodehash) {
2766 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2768 } elsif (my $avail = $scfg->{nodes
}) {
2769 foreach my $node (keys %$nodehash) {
2770 if (!$avail->{$node}) {
2771 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2778 foreach my $node (values %$nodehash) {
2779 if (my $unavail = $node->{unavailable_storages
}) {
2780 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2787 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2789 my ($vmid, $nocheck, $node) = @_;
2791 # $nocheck is set when called during a migration, in which case the config
2792 # file might still or already reside on the *other* node
2793 # - because rename has already happened, and current node is source
2794 # - because rename hasn't happened yet, and current node is target
2795 # - because rename has happened, current node is target, but hasn't yet
2797 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2798 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2803 my $vzlist = config_list
();
2805 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2807 while (defined(my $de = $fd->read)) {
2808 next if $de !~ m/^(\d+)\.pid$/;
2810 next if !defined($vzlist->{$vmid});
2811 if (my $pid = check_running
($vmid)) {
2812 $vzlist->{$vmid}->{pid
} = $pid;
2819 our $vmstatus_return_properties = {
2820 vmid
=> get_standard_option
('pve-vmid'),
2822 description
=> "QEMU process status.",
2824 enum
=> ['stopped', 'running'],
2827 description
=> "Maximum memory in bytes.",
2830 renderer
=> 'bytes',
2833 description
=> "Root disk size in bytes.",
2836 renderer
=> 'bytes',
2839 description
=> "VM name.",
2844 description
=> "QEMU QMP agent status.",
2849 description
=> "PID of running qemu process.",
2854 description
=> "Uptime.",
2857 renderer
=> 'duration',
2860 description
=> "Maximum usable CPUs.",
2865 description
=> "The current config lock, if any.",
2870 description
=> "The current configured tags, if any",
2874 'running-machine' => {
2875 description
=> "The currently running machine type (if running).",
2880 description
=> "The currently running QEMU version (if running).",
2886 my $last_proc_pid_stat;
2888 # get VM status information
2889 # This must be fast and should not block ($full == false)
2890 # We only query KVM using QMP if $full == true (this can be slow)
2892 my ($opt_vmid, $full) = @_;
2896 my $storecfg = PVE
::Storage
::config
();
2898 my $list = vzlist
();
2899 my $defaults = load_defaults
();
2901 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2903 my $cpucount = $cpuinfo->{cpus
} || 1;
2905 foreach my $vmid (keys %$list) {
2906 next if $opt_vmid && ($vmid ne $opt_vmid);
2908 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2910 my $d = { vmid
=> int($vmid) };
2911 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2913 # fixme: better status?
2914 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2916 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2917 if (defined($size)) {
2918 $d->{disk
} = 0; # no info available
2919 $d->{maxdisk
} = $size;
2925 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2926 * ($conf->{cores
} || $defaults->{cores
});
2927 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2928 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2930 $d->{name
} = $conf->{name
} || "VM $vmid";
2931 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2932 : $defaults->{memory
}*(1024*1024);
2934 if ($conf->{balloon
}) {
2935 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2936 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2937 : $defaults->{shares
};
2948 $d->{diskwrite
} = 0;
2950 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2952 $d->{serial
} = 1 if conf_has_serial
($conf);
2953 $d->{lock} = $conf->{lock} if $conf->{lock};
2954 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2959 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2960 foreach my $dev (keys %$netdev) {
2961 next if $dev !~ m/^tap([1-9]\d*)i/;
2963 my $d = $res->{$vmid};
2966 $d->{netout
} += $netdev->{$dev}->{receive
};
2967 $d->{netin
} += $netdev->{$dev}->{transmit
};
2970 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2971 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2976 my $ctime = gettimeofday
;
2978 foreach my $vmid (keys %$list) {
2980 my $d = $res->{$vmid};
2981 my $pid = $d->{pid
};
2984 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2985 next if !$pstat; # not running
2987 my $used = $pstat->{utime} + $pstat->{stime
};
2989 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2991 if ($pstat->{vsize
}) {
2992 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2995 my $old = $last_proc_pid_stat->{$pid};
2997 $last_proc_pid_stat->{$pid} = {
3005 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
3007 if ($dtime > 1000) {
3008 my $dutime = $used - $old->{used
};
3010 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
3011 $last_proc_pid_stat->{$pid} = {
3017 $d->{cpu
} = $old->{cpu
};
3021 return $res if !$full;
3023 my $qmpclient = PVE
::QMPClient-
>new();
3025 my $ballooncb = sub {
3026 my ($vmid, $resp) = @_;
3028 my $info = $resp->{'return'};
3029 return if !$info->{max_mem
};
3031 my $d = $res->{$vmid};
3033 # use memory assigned to VM
3034 $d->{maxmem
} = $info->{max_mem
};
3035 $d->{balloon
} = $info->{actual
};
3037 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
3038 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
3039 $d->{freemem
} = $info->{free_mem
};
3042 $d->{ballooninfo
} = $info;
3045 my $blockstatscb = sub {
3046 my ($vmid, $resp) = @_;
3047 my $data = $resp->{'return'} || [];
3048 my $totalrdbytes = 0;
3049 my $totalwrbytes = 0;
3051 for my $blockstat (@$data) {
3052 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
3053 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
3055 $blockstat->{device
} =~ s/drive-//;
3056 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
3058 $res->{$vmid}->{diskread
} = $totalrdbytes;
3059 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
3062 my $machinecb = sub {
3063 my ($vmid, $resp) = @_;
3064 my $data = $resp->{'return'} || [];
3066 $res->{$vmid}->{'running-machine'} =
3067 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
3070 my $versioncb = sub {
3071 my ($vmid, $resp) = @_;
3072 my $data = $resp->{'return'} // {};
3073 my $version = 'unknown';
3075 if (my $v = $data->{qemu
}) {
3076 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
3079 $res->{$vmid}->{'running-qemu'} = $version;
3082 my $statuscb = sub {
3083 my ($vmid, $resp) = @_;
3085 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3086 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3087 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3088 # this fails if ballon driver is not loaded, so this must be
3089 # the last commnand (following command are aborted if this fails).
3090 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3092 my $status = 'unknown';
3093 if (!defined($status = $resp->{'return'}->{status
})) {
3094 warn "unable to get VM status\n";
3098 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3101 foreach my $vmid (keys %$list) {
3102 next if $opt_vmid && ($vmid ne $opt_vmid);
3103 next if !$res->{$vmid}->{pid
}; # not running
3104 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3107 $qmpclient->queue_execute(undef, 2);
3109 foreach my $vmid (keys %$list) {
3110 next if $opt_vmid && ($vmid ne $opt_vmid);
3111 next if !$res->{$vmid}->{pid
}; #not running
3113 # we can't use the $qmpclient since it might have already aborted on
3114 # 'query-balloon', but this might also fail for older versions...
3115 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
3116 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3119 foreach my $vmid (keys %$list) {
3120 next if $opt_vmid && ($vmid ne $opt_vmid);
3121 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3127 sub conf_has_serial
{
3130 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3131 if ($conf->{"serial$i"}) {
3139 sub conf_has_audio
{
3140 my ($conf, $id) = @_;
3143 my $audio = $conf->{"audio$id"};
3144 return if !defined($audio);
3146 my $audioproperties = parse_property_string
($audio_fmt, $audio);
3147 my $audiodriver = $audioproperties->{driver
} // 'spice';
3150 dev
=> $audioproperties->{device
},
3151 dev_id
=> "audiodev$id",
3152 backend
=> $audiodriver,
3153 backend_id
=> "$audiodriver-backend${id}",
3158 my ($audio, $audiopciaddr, $machine_version) = @_;
3162 my $id = $audio->{dev_id
};
3164 if (min_version
($machine_version, 4, 2)) {
3165 $audiodev = ",audiodev=$audio->{backend_id}";
3168 if ($audio->{dev
} eq 'AC97') {
3169 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3170 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3171 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3172 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3173 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3175 die "unkown audio device '$audio->{dev}', implement me!";
3178 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3186 socket => "/var/run/qemu-server/$vmid.swtpm",
3187 pid
=> "/var/run/qemu-server/$vmid.swtpm.pid",
3191 sub add_tpm_device
{
3192 my ($vmid, $devices, $conf) = @_;
3194 return if !$conf->{tpmstate0
};
3196 my $paths = get_tpm_paths
($vmid);
3198 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3199 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3200 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3204 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3206 return if !$tpmdrive;
3209 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
3210 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
3212 $state = PVE
::Storage
::map_volume
($storecfg, $tpm->{file
});
3214 $state = $tpm->{file
};
3217 my $paths = get_tpm_paths
($vmid);
3219 # during migration, we will get state from remote
3222 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3229 "--create-platform-cert",
3232 "/etc/swtpm_setup.conf", # do not use XDG configs
3234 "0", # force creation as root, error if not possible
3235 "--not-overwrite", # ignore existing state, do not modify
3238 push @$setup_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3239 # TPM 2.0 supports ECC crypto, use if possible
3240 push @$setup_cmd, "--ecc" if $tpm->{version
} eq 'v2.0';
3242 run_command
($setup_cmd, outfunc
=> sub {
3243 print "swtpm_setup: $1\n";
3247 my $emulator_cmd = [
3251 "backend-uri=file://$state,mode=0600",
3253 "type=unixio,path=$paths->{socket},mode=0600",
3255 "file=$paths->{pid}",
3256 "--terminate", # terminate on QEMU disconnect
3259 push @$emulator_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3260 run_command
($emulator_cmd, outfunc
=> sub { print $1; });
3262 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3263 while (! -e
$paths->{pid
}) {
3264 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3268 # return untainted PID of swtpm daemon so it can be killed on error
3269 file_read_firstline
($paths->{pid
}) =~ m/(\d+)/;
3273 sub vga_conf_has_spice
{
3276 my $vgaconf = parse_vga
($vga);
3277 my $vgatype = $vgaconf->{type
};
3278 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3285 return get_host_arch
() eq $arch;
3290 return $conf->{arch
} // get_host_arch
();
3293 my $default_machines = {
3298 sub get_installed_machine_version
{
3299 my ($kvmversion) = @_;
3300 $kvmversion = kvm_user_version
() if !defined($kvmversion);
3301 $kvmversion =~ m/^(\d+\.\d+)/;
3305 sub windows_get_pinned_machine_version
{
3306 my ($machine, $base_version, $kvmversion) = @_;
3308 my $pin_version = $base_version;
3309 if (!defined($base_version) ||
3310 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3312 $pin_version = get_installed_machine_version
($kvmversion);
3314 if (!$machine || $machine eq 'pc') {
3315 $machine = "pc-i440fx-$pin_version";
3316 } elsif ($machine eq 'q35') {
3317 $machine = "pc-q35-$pin_version";
3318 } elsif ($machine eq 'virt') {
3319 $machine = "virt-$pin_version";
3321 warn "unknown machine type '$machine', not touching that!\n";
3327 sub get_vm_machine
{
3328 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3330 my $machine = $forcemachine || $conf->{machine
};
3332 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3333 $kvmversion //= kvm_user_version
();
3334 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3335 # layout which confuses windows quite a bit and may result in various regressions..
3336 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3337 if (windows_version
($conf->{ostype
})) {
3338 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3341 $machine ||= $default_machines->{$arch};
3342 if ($add_pve_version) {
3343 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3344 $machine .= "+pve$pvever";
3348 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3349 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3350 $machine = $1 if $is_pxe;
3352 # for version-pinned machines that do not include a pve-version (e.g.
3353 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3354 $machine .= '+pve0';
3356 $machine .= '.pxe' if $is_pxe;
3362 sub get_ovmf_files
($$$) {
3363 my ($arch, $efidisk, $smm) = @_;
3365 my $types = $OVMF->{$arch}
3366 or die "no OVMF images known for architecture '$arch'\n";
3368 my $type = 'default';
3369 if (defined($efidisk->{efitype
}) && $efidisk->{efitype
} eq '4m') {
3370 $type = $smm ?
"4m" : "4m-no-smm";
3371 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3374 return $types->{$type}->@*;
3378 aarch64
=> '/usr/bin/qemu-system-aarch64',
3379 x86_64
=> '/usr/bin/qemu-system-x86_64',
3381 sub get_command_for_arch
($) {
3383 return '/usr/bin/kvm' if is_native
($arch);
3385 my $cmd = $Arch2Qemu->{$arch}
3386 or die "don't know how to emulate architecture '$arch'\n";
3390 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3391 # to use in a QEMU command line (-cpu element), first array_intersect the result
3392 # of query_supported_ with query_understood_. This is necessary because:
3394 # a) query_understood_ returns flags the host cannot use and
3395 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3396 # flags, but CPU settings - with most of them being flags. Those settings
3397 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3399 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3400 # expensive. If you need the value returned from this, you can get it much
3401 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3402 # $accel being 'kvm' or 'tcg'.
3404 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3405 # changes, automatically populating pmxcfs.
3407 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3408 # since kvm and tcg machines support different flags
3410 sub query_supported_cpu_flags
{
3413 $arch //= get_host_arch
();
3414 my $default_machine = $default_machines->{$arch};
3418 # FIXME: Once this is merged, the code below should work for ARM as well:
3419 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3420 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3423 my $kvm_supported = defined(kvm_version
());
3424 my $qemu_cmd = get_command_for_arch
($arch);
3426 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3428 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3429 my $query_supported_run_qemu = sub {
3435 '-machine', $default_machine,
3437 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3438 '-mon', 'chardev=qmp,mode=control',
3439 '-pidfile', $pidfile,
3444 push @$cmd, '-accel', 'tcg';
3447 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3448 die "QEMU flag querying VM exited with code " . $rc if $rc;
3451 my $cmd_result = mon_cmd
(
3453 'query-cpu-model-expansion',
3455 model
=> { name
=> 'host' }
3458 my $props = $cmd_result->{model
}->{props
};
3459 foreach my $prop (keys %$props) {
3460 next if $props->{$prop} ne '1';
3461 # QEMU returns some flags multiple times, with '_', '.' or '-'
3462 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3463 # We only keep those with underscores, to match /proc/cpuinfo
3464 $prop =~ s/\.|-/_/g;
3465 $flags->{$prop} = 1;
3470 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3471 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3475 return [ sort keys %$flags ];
3478 # We need to query QEMU twice, since KVM and TCG have different supported flags
3479 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3480 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3481 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3483 if ($kvm_supported) {
3484 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3485 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3492 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3493 my $understood_cpu_flag_dir = "/usr/share/kvm";
3494 sub query_understood_cpu_flags
{
3495 my $arch = get_host_arch
();
3496 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3498 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3501 my $raw = file_get_contents
($filepath);
3502 $raw =~ s/^\s+|\s+$//g;
3503 my @flags = split(/\s+/, $raw);
3508 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3509 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3510 my sub should_disable_smm
{
3511 my ($conf, $vga) = @_;
3513 return (!defined($conf->{bios
}) || $conf->{bios
} eq 'seabios') &&
3514 $vga->{type
} && $vga->{type
} =~ m/^(serial\d+|none)$/;
3517 my sub print_ovmf_drive_commandlines
{
3518 my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3520 my $d = $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
3522 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch, $d, $q35);
3523 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3525 my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3527 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3528 my ($path, $format) = $d->@{'file', 'format'};
3530 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3531 if (!defined($format)) {
3532 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3533 $format = qemu_img_format
($scfg, $volname);
3535 } elsif (!defined($format)) {
3536 die "efidisk format must be specified\n";
3538 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3539 if ($path =~ m/^rbd:/) {
3540 $var_drive_str .= ',cache=writeback';
3541 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3543 $var_drive_str .= ",format=$format,file=$path";
3545 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3546 $var_drive_str .= ',readonly=on' if drive_is_read_only
($conf, $d);
3548 log_warn
("no efidisk configured! Using temporary efivars disk.");
3549 my $path = "/tmp/$vmid-ovmf.fd";
3550 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3551 $var_drive_str .= ",format=raw,file=$path";
3552 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $version_guard->(4, 1, 2);
3555 return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3558 sub config_to_command
{
3559 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3562 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3565 my $ostype = $conf->{ostype
};
3566 my $winversion = windows_version
($ostype);
3567 my $kvm = $conf->{kvm
};
3568 my $nodename = nodename
();
3570 my $arch = get_vm_arch
($conf);
3571 my $kvm_binary = get_command_for_arch
($arch);
3572 my $kvmver = kvm_user_version
($kvm_binary);
3574 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3575 $kvmver //= "undefined";
3576 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3579 my $add_pve_version = min_version
($kvmver, 4, 1);
3581 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3582 my $machine_version = extract_version
($machine_type, $kvmver);
3583 $kvm //= 1 if is_native
($arch);
3585 $machine_version =~ m/(\d+)\.(\d+)/;
3586 my ($machine_major, $machine_minor) = ($1, $2);
3588 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3589 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3590 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3591 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3592 ." please upgrade node '$nodename'\n"
3593 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3594 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3595 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3596 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3597 ." node '$nodename'\n";
3600 # if a specific +pve version is required for a feature, use $version_guard
3601 # instead of min_version to allow machines to be run with the minimum
3603 my $required_pve_version = 0;
3604 my $version_guard = sub {
3605 my ($major, $minor, $pve) = @_;
3606 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3607 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3608 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3609 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3613 if ($kvm && !defined kvm_version
()) {
3614 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3615 ." or enable in BIOS.\n";
3618 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3619 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3620 my $use_old_bios_files = undef;
3621 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3624 if ($conf->{affinity
}) {
3625 push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity
};
3628 push @$cmd, $kvm_binary;
3630 push @$cmd, '-id', $vmid;
3632 my $vmname = $conf->{name
} || "vm$vmid";
3634 push @$cmd, '-name', "$vmname,debug-threads=on";
3636 push @$cmd, '-no-shutdown';
3640 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3641 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3642 push @$cmd, '-mon', "chardev=qmp,mode=control";
3644 if (min_version
($machine_version, 2, 12)) {
3645 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3646 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3649 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3651 push @$cmd, '-daemonize';
3653 if ($conf->{smbios1
}) {
3654 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3655 if ($smbios_conf->{base64
}) {
3656 # Do not pass base64 flag to qemu
3657 delete $smbios_conf->{base64
};
3658 my $smbios_string = "";
3659 foreach my $key (keys %$smbios_conf) {
3661 if ($key eq "uuid") {
3662 $value = $smbios_conf->{uuid
}
3664 $value = decode_base64
($smbios_conf->{$key});
3666 # qemu accepts any binary data, only commas need escaping by double comma
3668 $smbios_string .= "," . $key . "=" . $value if $value;
3670 push @$cmd, '-smbios', "type=1" . $smbios_string;
3672 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3676 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3677 my ($code_drive_str, $var_drive_str) =
3678 print_ovmf_drive_commandlines
($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3679 push $cmd->@*, '-drive', $code_drive_str;
3680 push $cmd->@*, '-drive', $var_drive_str;
3683 if ($q35) { # tell QEMU to load q35 config early
3684 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3685 if (min_version
($machine_version, 4, 0)) {
3686 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3688 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3692 if (defined(my $fixups = qemu_created_version_fixups
($conf, $forcemachine, $kvmver))) {
3693 push @$cmd, $fixups->@*;
3696 if ($conf->{vmgenid
}) {
3697 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3700 # add usb controllers
3701 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3702 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES, $machine_version);
3703 push @$devices, @usbcontrollers if @usbcontrollers;
3704 my $vga = parse_vga
($conf->{vga
});
3706 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3707 $vga->{type
} = 'qxl' if $qxlnum;
3709 if (!$vga->{type
}) {
3710 if ($arch eq 'aarch64') {
3711 $vga->{type
} = 'virtio';
3712 } elsif (min_version
($machine_version, 2, 9)) {
3713 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3715 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3719 # enable absolute mouse coordinates (needed by vnc)
3720 my $tablet = $conf->{tablet
};
3721 if (!defined($tablet)) {
3722 $tablet = $defaults->{tablet
};
3723 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3724 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3728 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3729 my $kbd = print_keyboarddevice_full
($conf, $arch);
3730 push @$devices, '-device', $kbd if defined($kbd);
3733 my $bootorder = device_bootorder
($conf);
3735 # host pci device passthrough
3736 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3737 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3740 my $usb_dev_features = {};
3741 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3743 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3744 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3745 push @$devices, @usbdevices if @usbdevices;
3748 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3749 my $path = $conf->{"serial$i"} or next;
3750 if ($path eq 'socket') {
3751 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3752 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3753 # On aarch64, serial0 is the UART device. QEMU only allows
3754 # connecting UART devices via the '-serial' command line, as
3755 # the device has a fixed slot on the hardware...
3756 if ($arch eq 'aarch64' && $i == 0) {
3757 push @$devices, '-serial', "chardev:serial$i";
3759 push @$devices, '-device', "isa-serial,chardev=serial$i";
3762 die "no such serial device\n" if ! -c
$path;
3763 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3764 push @$devices, '-device', "isa-serial,chardev=serial$i";
3769 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3770 if (my $path = $conf->{"parallel$i"}) {
3771 die "no such parallel device\n" if ! -c
$path;
3772 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3773 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3774 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3778 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3779 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3780 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3781 push @$devices, @$audio_devs;
3784 add_tpm_device
($vmid, $devices, $conf);
3787 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3788 $sockets = $conf->{sockets
} if $conf->{sockets
};
3790 my $cores = $conf->{cores
} || 1;
3792 my $maxcpus = $sockets * $cores;
3794 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3796 my $allowed_vcpus = $cpuinfo->{cpus
};
3798 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3800 if ($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3801 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3802 for (my $i = 2; $i <= $vcpus; $i++) {
3803 my $cpustr = print_cpu_device
($conf,$i);
3804 push @$cmd, '-device', $cpustr;
3809 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3811 push @$cmd, '-nodefaults';
3813 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3815 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3817 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3819 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3820 push @$devices, '-device', print_vga_device
(
3821 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3823 push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type
} eq 'virtio-gl'; # VIRGL
3825 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3826 push @$cmd, '-vnc', "unix:$socket,password=on";
3828 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3829 push @$cmd, '-nographic';
3833 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3834 my $useLocaltime = $conf->{localtime};
3836 if ($winversion >= 5) { # windows
3837 $useLocaltime = 1 if !defined($conf->{localtime});
3839 # use time drift fix when acpi is enabled
3840 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3841 $tdf = 1 if !defined($conf->{tdf
});
3845 if ($winversion >= 6) {
3846 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3847 push @$cmd, '-no-hpet';
3850 push @$rtcFlags, 'driftfix=slew' if $tdf;
3852 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3853 push @$rtcFlags, "base=$conf->{startdate}";
3854 } elsif ($useLocaltime) {
3855 push @$rtcFlags, 'base=localtime';
3859 push @$cmd, '-cpu', $forcecpu;
3861 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3864 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3866 push @$cmd, '-S' if $conf->{freeze
};
3868 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3870 my $guest_agent = parse_guest_agent
($conf);
3872 if ($guest_agent->{enabled
}) {
3873 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3874 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3876 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3877 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3878 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3879 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3880 } elsif ($guest_agent->{type
} eq 'isa') {
3881 push @$devices, '-device', "isa-serial,chardev=qga0";
3885 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3886 if ($rng && $version_guard->(4, 1, 2)) {
3887 check_rng_source
($rng->{source
});
3889 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3890 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3891 my $limiter_str = "";
3893 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3896 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3897 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3898 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3903 if ($qxlnum || $vga->{type
} =~ /^virtio/) {
3906 for (my $i = 1; $i < $qxlnum; $i++){
3907 push @$devices, '-device', print_vga_device
(
3908 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3911 # assume other OS works like Linux
3912 my ($ram, $vram) = ("134217728", "67108864");
3913 if ($vga->{memory
}) {
3914 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3915 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3917 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3918 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3922 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3924 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3925 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3926 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3928 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3929 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3930 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3932 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3933 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3935 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3936 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3937 if ($spice_enhancement->{foldersharing
}) {
3938 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3939 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3942 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3943 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3944 if $spice_enhancement->{videostreaming
};
3946 push @$devices, '-spice', "$spice_opts";
3949 # enable balloon by default, unless explicitly disabled
3950 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3951 my $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3952 my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3953 $ballooncmd .= ",free-page-reporting=on" if min_version
($machine_version, 6, 2);
3954 push @$devices, '-device', $ballooncmd;
3957 if ($conf->{watchdog
}) {
3958 my $wdopts = parse_watchdog
($conf->{watchdog
});
3959 my $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3960 my $watchdog = $wdopts->{model
} || 'i6300esb';
3961 push @$devices, '-device', "$watchdog$pciaddr";
3962 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3966 my $scsicontroller = {};
3967 my $ahcicontroller = {};
3968 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3970 # Add iscsi initiator name if available
3971 if (my $initiator = get_initiator_name
()) {
3972 push @$devices, '-iscsi', "initiator-name=$initiator";
3975 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3976 my ($ds, $drive) = @_;
3978 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3979 check_volume_storage_type
($storecfg, $drive->{file
});
3980 push @$vollist, $drive->{file
};
3983 # ignore efidisk here, already added in bios/fw handling code above
3984 return if $drive->{interface
} eq 'efidisk';
3986 return if $drive->{interface
} eq 'tpmstate';
3988 $use_virtio = 1 if $ds =~ m/^virtio/;
3990 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3992 if ($drive->{interface
} eq 'virtio'){
3993 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3996 if ($drive->{interface
} eq 'scsi') {
3998 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
4000 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4001 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4003 my $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
4004 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
4007 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
4008 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4009 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4010 } elsif ($drive->{iothread
}) {
4012 "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4017 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
4018 $queues = ",num_queues=$drive->{queues}";
4021 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4022 if !$scsicontroller->{$controller};
4023 $scsicontroller->{$controller}=1;
4026 if ($drive->{interface
} eq 'sata') {
4027 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
4028 my $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
4029 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4030 if !$ahcicontroller->{$controller};
4031 $ahcicontroller->{$controller}=1;
4034 my $pbs_conf = $pbs_backing->{$ds};
4035 my $pbs_name = undef;
4037 $pbs_name = "drive-$ds-pbs";
4038 push @$devices, '-blockdev', print_pbs_blockdev
($pbs_conf, $pbs_name);
4041 my $drive_cmd = print_drive_commandline_full
(
4042 $storecfg, $vmid, $drive, $pbs_name, min_version
($kvmver, 6, 0));
4044 # extra protection for templates, but SATA and IDE don't support it..
4045 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
4047 push @$devices, '-drive',$drive_cmd;
4048 push @$devices, '-device', print_drivedevice_full
(
4049 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4052 for (my $i = 0; $i < $MAX_NETS; $i++) {
4053 my $netname = "net$i";
4055 next if !$conf->{$netname};
4056 my $d = parse_net
($conf->{$netname});
4058 # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4060 $use_virtio = 1 if $d->{model
} eq 'virtio';
4062 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
4064 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
4065 push @$devices, '-netdev', $netdevfull;
4067 my $netdevicefull = print_netdevice_full
(
4068 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4070 push @$devices, '-device', $netdevicefull;
4073 if ($conf->{ivshmem
}) {
4074 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4078 $bus = print_pcie_addr
("ivshmem");
4080 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4083 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4084 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4086 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4087 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4088 .",size=$ivshmem->{size}M";
4091 # pci.4 is nested in pci.1
4092 $bridges->{1} = 1 if $bridges->{4};
4094 if (!$q35) { # add pci bridges
4095 if (min_version
($machine_version, 2, 3)) {
4099 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4102 for my $k (sort {$b cmp $a} keys %$bridges) {
4103 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4106 if ($k == 2 && $legacy_igd) {
4109 my $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
4110 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4112 if ($q35) { # add after -readconfig pve-q35.cfg
4113 splice @$devices, 2, 0, '-device', $devstr;
4115 unshift @$devices, '-device', $devstr if $k > 0;
4120 push @$machineFlags, 'accel=tcg';
4123 push @$machineFlags, 'smm=off' if should_disable_smm
($conf, $vga);
4125 my $machine_type_min = $machine_type;
4126 if ($add_pve_version) {
4127 $machine_type_min =~ s/\+pve\d+$//;
4128 $machine_type_min .= "+pve$required_pve_version";
4130 push @$machineFlags, "type=${machine_type_min}";
4132 push @$cmd, @$devices;
4133 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4134 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4135 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4137 if (my $vmstate = $conf->{vmstate
}) {
4138 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4139 push @$vollist, $vmstate;
4140 push @$cmd, '-loadstate', $statepath;
4141 print "activating and using '$vmstate' as vmstate\n";
4144 if (PVE
::QemuConfig-
>is_template($conf)) {
4145 # needed to workaround base volumes being read-only
4146 push @$cmd, '-snapshot';
4150 if ($conf->{args
}) {
4151 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4155 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
4158 sub check_rng_source
{
4161 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4162 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4165 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4166 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
4167 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4168 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4169 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4170 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4178 my $res = mon_cmd
($vmid, 'query-spice');
4180 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4183 sub vm_devices_list
{
4186 my $res = mon_cmd
($vmid, 'query-pci');
4187 my $devices_to_check = [];
4189 foreach my $pcibus (@$res) {
4190 push @$devices_to_check, @{$pcibus->{devices
}},
4193 while (@$devices_to_check) {
4195 for my $d (@$devices_to_check) {
4196 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4197 next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices
};
4199 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4200 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4202 $devices_to_check = $to_check;
4205 my $resblock = mon_cmd
($vmid, 'query-block');
4206 foreach my $block (@$resblock) {
4207 if($block->{device
} =~ m/^drive-(\S+)/){
4212 my $resmice = mon_cmd
($vmid, 'query-mice');
4213 foreach my $mice (@$resmice) {
4214 if ($mice->{name
} eq 'QEMU HID Tablet') {
4215 $devices->{tablet
} = 1;
4220 # for usb devices there is no query-usb
4221 # but we can iterate over the entries in
4222 # qom-list path=/machine/peripheral
4223 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4224 foreach my $per (@$resperipheral) {
4225 if ($per->{name
} =~ m/^usb(?:redirdev)?\d+$/) {
4226 $devices->{$per->{name
}} = 1;
4234 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4236 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4238 my $devices_list = vm_devices_list
($vmid);
4239 return 1 if defined($devices_list->{$deviceid});
4241 # add PCI bridge if we need it for the device
4242 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4244 if ($deviceid eq 'tablet') {
4245 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4246 } elsif ($deviceid eq 'keyboard') {
4247 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4248 } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4250 qemu_spice_usbredir_chardev_add
($vmid, "usbredirchardev$id");
4251 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_spice_usbdevice
($id, "xhci", $id + 1));
4252 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4253 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device, {}, $1 + 1));
4254 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4255 qemu_iothread_add
($vmid, $deviceid, $device);
4257 qemu_driveadd
($storecfg, $vmid, $device);
4258 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4260 qemu_deviceadd
($vmid, $devicefull);
4261 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4263 eval { qemu_drivedel
($vmid, $deviceid); };
4267 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4268 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4269 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4270 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4272 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4274 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4275 qemu_iothread_add
($vmid, $deviceid, $device);
4276 $devicefull .= ",iothread=iothread-$deviceid";
4279 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4280 $devicefull .= ",num_queues=$device->{queues}";
4283 qemu_deviceadd
($vmid, $devicefull);
4284 qemu_deviceaddverify
($vmid, $deviceid);
4285 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4286 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4287 qemu_driveadd
($storecfg, $vmid, $device);
4289 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4290 eval { qemu_deviceadd
($vmid, $devicefull); };
4292 eval { qemu_drivedel
($vmid, $deviceid); };
4296 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4297 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4299 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4300 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type);
4301 my $use_old_bios_files = undef;
4302 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4304 my $netdevicefull = print_netdevice_full
(
4305 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4306 qemu_deviceadd
($vmid, $netdevicefull);
4308 qemu_deviceaddverify
($vmid, $deviceid);
4309 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4312 eval { qemu_netdevdel
($vmid, $deviceid); };
4316 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4318 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4319 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4321 qemu_deviceadd
($vmid, $devicefull);
4322 qemu_deviceaddverify
($vmid, $deviceid);
4324 die "can't hotplug device '$deviceid'\n";
4330 # fixme: this should raise exceptions on error!
4331 sub vm_deviceunplug
{
4332 my ($vmid, $conf, $deviceid) = @_;
4334 my $devices_list = vm_devices_list
($vmid);
4335 return 1 if !defined($devices_list->{$deviceid});
4337 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4338 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4340 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4341 qemu_devicedel
($vmid, $deviceid);
4342 } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4343 qemu_devicedel
($vmid, $deviceid);
4344 qemu_devicedelverify
($vmid, $deviceid);
4345 } elsif ($deviceid =~ m/^usb\d+$/) {
4346 qemu_devicedel
($vmid, $deviceid);
4347 qemu_devicedelverify
($vmid, $deviceid);
4348 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4349 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4351 qemu_devicedel
($vmid, $deviceid);
4352 qemu_devicedelverify
($vmid, $deviceid);
4353 qemu_drivedel
($vmid, $deviceid);
4354 qemu_iothread_del
($vmid, $deviceid, $device);
4355 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4356 qemu_devicedel
($vmid, $deviceid);
4357 qemu_devicedelverify
($vmid, $deviceid);
4358 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4359 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4361 qemu_devicedel
($vmid, $deviceid);
4362 qemu_devicedelverify
($vmid, $deviceid);
4363 qemu_drivedel
($vmid, $deviceid);
4364 qemu_deletescsihw
($conf, $vmid, $deviceid);
4366 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4367 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4368 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4369 qemu_devicedel
($vmid, $deviceid);
4370 qemu_devicedelverify
($vmid, $deviceid);
4371 qemu_netdevdel
($vmid, $deviceid);
4373 die "can't unplug device '$deviceid'\n";
4379 sub qemu_spice_usbredir_chardev_add
{
4380 my ($vmid, $id) = @_;
4382 mon_cmd
($vmid, "chardev-add" , (
4393 sub qemu_deviceadd
{
4394 my ($vmid, $devicefull) = @_;
4396 $devicefull = "driver=".$devicefull;
4397 my %options = split(/[=,]/, $devicefull);
4399 mon_cmd
($vmid, "device_add" , %options);
4402 sub qemu_devicedel
{
4403 my ($vmid, $deviceid) = @_;
4405 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4408 sub qemu_iothread_add
{
4409 my ($vmid, $deviceid, $device) = @_;
4411 if ($device->{iothread
}) {
4412 my $iothreads = vm_iothreads_list
($vmid);
4413 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4417 sub qemu_iothread_del
{
4418 my ($vmid, $deviceid, $device) = @_;
4420 if ($device->{iothread
}) {
4421 my $iothreads = vm_iothreads_list
($vmid);
4422 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4426 sub qemu_objectadd
{
4427 my ($vmid, $objectid, $qomtype) = @_;
4429 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4434 sub qemu_objectdel
{
4435 my ($vmid, $objectid) = @_;
4437 mon_cmd
($vmid, "object-del", id
=> $objectid);
4443 my ($storecfg, $vmid, $device) = @_;
4445 my $kvmver = get_running_qemu_version
($vmid);
4446 my $io_uring = min_version
($kvmver, 6, 0);
4447 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4448 $drive =~ s/\\/\\\\/g;
4449 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4451 # If the command succeeds qemu prints: "OK
"
4452 return 1 if $ret =~ m/OK/s;
4454 die "adding drive failed
: $ret\n";
4458 my ($vmid, $deviceid) = @_;
4460 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4463 return 1 if $ret eq "";
4465 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4466 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4468 die "deleting drive
$deviceid failed
: $ret\n";
4471 sub qemu_deviceaddverify {
4472 my ($vmid, $deviceid) = @_;
4474 for (my $i = 0; $i <= 5; $i++) {
4475 my $devices_list = vm_devices_list($vmid);
4476 return 1 if defined($devices_list->{$deviceid});
4480 die "error on hotplug device
'$deviceid'\n";
4484 sub qemu_devicedelverify {
4485 my ($vmid, $deviceid) = @_;
4487 # need to verify that the device is correctly removed as device_del
4488 # is async and empty return is not reliable
4490 for (my $i = 0; $i <= 5; $i++) {
4491 my $devices_list = vm_devices_list($vmid);
4492 return 1 if !defined($devices_list->{$deviceid});
4496 die "error on hot-unplugging device
'$deviceid'\n";
4499 sub qemu_findorcreatescsihw {
4500 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4502 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4504 my $scsihwid="$controller_prefix$controller";
4505 my $devices_list = vm_devices_list($vmid);
4507 if (!defined($devices_list->{$scsihwid})) {
4508 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4514 sub qemu_deletescsihw {
4515 my ($conf, $vmid, $opt) = @_;
4517 my $device = parse_drive($opt, $conf->{$opt});
4519 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4520 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4524 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4526 my $devices_list = vm_devices_list($vmid);
4527 foreach my $opt (keys %{$devices_list}) {
4528 if (is_valid_drivename($opt)) {
4529 my $drive = parse_drive($opt, $conf->{$opt});
4530 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4536 my $scsihwid="scsihw
$controller";
4538 vm_deviceunplug($vmid, $conf, $scsihwid);
4543 sub qemu_add_pci_bridge {
4544 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4550 print_pci_addr($device, $bridges, $arch, $machine_type);
4552 while (my ($k, $v) = each %$bridges) {
4555 return 1 if !defined($bridgeid) || $bridgeid < 1;
4557 my $bridge = "pci
.$bridgeid";
4558 my $devices_list = vm_devices_list($vmid);
4560 if (!defined($devices_list->{$bridge})) {
4561 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4567 sub qemu_set_link_status {
4568 my ($vmid, $device, $up) = @_;
4570 mon_cmd($vmid, "set_link
", name => $device,
4571 up => $up ? JSON::true : JSON::false);
4574 sub qemu_netdevadd {
4575 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4577 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4578 my %options = split(/[=,]/, $netdev);
4580 if (defined(my $vhost = $options{vhost})) {
4581 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4584 if (defined(my $queues = $options{queues})) {
4585 $options{queues} = $queues + 0;
4588 mon_cmd($vmid, "netdev_add
", %options);
4592 sub qemu_netdevdel {
4593 my ($vmid, $deviceid) = @_;
4595 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4598 sub qemu_usb_hotplug {
4599 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4603 # remove the old one first
4604 vm_deviceunplug($vmid, $conf, $deviceid);
4606 # check if xhci controller is necessary and available
4607 my $devicelist = vm_devices_list($vmid);
4609 if (!$devicelist->{xhci}) {
4610 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4611 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4614 # print_usbdevice_full expects the parsed device
4615 my $d = parse_usb_device($device->{host});
4616 $d->{usb3} = $device->{usb3};
4619 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4622 sub qemu_cpu_hotplug {
4623 my ($vmid, $conf, $vcpus) = @_;
4625 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4628 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4629 $sockets = $conf->{sockets} if $conf->{sockets};
4630 my $cores = $conf->{cores} || 1;
4631 my $maxcpus = $sockets * $cores;
4633 $vcpus = $maxcpus if !$vcpus;
4635 die "you can
't add more vcpus than maxcpus\n"
4636 if $vcpus > $maxcpus;
4638 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4640 if ($vcpus < $currentvcpus) {
4642 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4644 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4645 qemu_devicedel($vmid, "cpu$i");
4647 my $currentrunningvcpus = undef;
4649 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4650 last if scalar(@{$currentrunningvcpus}) == $i-1;
4651 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4655 #update conf after each succesfull cpu unplug
4656 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4657 PVE::QemuConfig->write_config($vmid, $conf);
4660 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4666 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4667 die "vcpus in running vm does not match its configuration\n"
4668 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4670 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4672 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4673 my $cpustr = print_cpu_device($conf, $i);
4674 qemu_deviceadd($vmid, $cpustr);
4677 my $currentrunningvcpus = undef;
4679 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4680 last if scalar(@{$currentrunningvcpus}) == $i;
4681 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4685 #update conf after each succesfull cpu hotplug
4686 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4687 PVE::QemuConfig->write_config($vmid, $conf);
4691 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4692 mon_cmd($vmid, "cpu-add", id => int($i));
4697 sub qemu_block_set_io_throttle {
4698 my ($vmid, $deviceid,
4699 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4700 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4701 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4702 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4704 return if !check_running($vmid) ;
4706 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4708 bps_rd => int($bps_rd),
4709 bps_wr => int($bps_wr),
4711 iops_rd => int($iops_rd),
4712 iops_wr => int($iops_wr),
4713 bps_max => int($bps_max),
4714 bps_rd_max => int($bps_rd_max),
4715 bps_wr_max => int($bps_wr_max),
4716 iops_max => int($iops_max),
4717 iops_rd_max => int($iops_rd_max),
4718 iops_wr_max => int($iops_wr_max),
4719 bps_max_length => int($bps_max_length),
4720 bps_rd_max_length => int($bps_rd_max_length),
4721 bps_wr_max_length => int($bps_wr_max_length),
4722 iops_max_length => int($iops_max_length),
4723 iops_rd_max_length => int($iops_rd_max_length),
4724 iops_wr_max_length => int($iops_wr_max_length),
4729 sub qemu_block_resize {
4730 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4732 my $running = check_running($vmid);
4734 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4736 return if !$running;
4738 my $padding = (1024 - $size % 1024) % 1024;
4739 $size = $size + $padding;
4744 device => $deviceid,
4750 sub qemu_volume_snapshot {
4751 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4753 my $running = check_running($vmid);
4755 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4756 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4758 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4762 sub qemu_volume_snapshot_delete {
4763 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4765 my $running = check_running($vmid);
4770 my $conf = PVE::QemuConfig->load_config($vmid);
4771 PVE::QemuConfig->foreach_volume($conf, sub {
4772 my ($ds, $drive) = @_;
4773 $running = 1 if $drive->{file} eq $volid;
4777 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4778 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4780 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4784 sub set_migration_caps {
4785 my ($vmid, $savevm) = @_;
4787 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4789 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4790 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4795 "auto-converge" => 1,
4797 "x-rdma-pin-all" => 0,
4800 "dirty-bitmaps" => $dirty_bitmaps,
4803 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4805 for my $supported_capability (@$supported_capabilities) {
4807 capability => $supported_capability->{capability},
4808 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4812 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4816 my ($conf, $func, @param) = @_;
4820 my $test_volid = sub {
4821 my ($key, $drive, $snapname) = @_;
4823 my $volid = $drive->{file};
4826 $volhash->{$volid}->{cdrom} //= 1;
4827 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4829 my $replicate = $drive->{replicate} // 1;
4830 $volhash->{$volid}->{replicate} //= 0;
4831 $volhash->{$volid}->{replicate} = 1 if $replicate;
4833 $volhash->{$volid}->{shared} //= 0;
4834 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4836 $volhash->{$volid}->{referenced_in_config} //= 0;
4837 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4839 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4840 if defined($snapname);
4842 my $size = $drive->{size};
4843 $volhash->{$volid}->{size} //= $size if $size;
4845 $volhash->{$volid}->{is_vmstate} //= 0;
4846 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4848 $volhash->{$volid}->{is_tpmstate} //= 0;
4849 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0
';
4851 $volhash->{$volid}->{is_unused} //= 0;
4852 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4854 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4857 my $include_opts = {
4858 extra_keys => ['vmstate
'],
4859 include_unused => 1,
4862 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4863 foreach my $snapname (keys %{$conf->{snapshots}}) {
4864 my $snap = $conf->{snapshots}->{$snapname};
4865 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4868 foreach my $volid (keys %$volhash) {
4869 &$func($volid, $volhash->{$volid}, @param);
4873 my $fast_plug_option = {
4881 'vmstatestorage
' => 1,
4886 for my $opt (keys %$confdesc_cloudinit) {
4887 $fast_plug_option->{$opt} = 1;
4890 # hotplug changes in [PENDING]
4891 # $selection hash can be used to only apply specified options, for
4892 # example: { cores => 1 } (only apply changed 'cores
')
4893 # $errors ref is used to return error messages
4894 sub vmconfig_hotplug_pending {
4895 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4897 my $defaults = load_defaults();
4898 my $arch = get_vm_arch($conf);
4899 my $machine_type = get_vm_machine($conf, undef, $arch);
4901 # commit values which do not have any impact on running VM first
4902 # Note: those option cannot raise errors, we we do not care about
4903 # $selection and always apply them.
4905 my $add_error = sub {
4906 my ($opt, $msg) = @_;
4907 $errors->{$opt} = "hotplug problem - $msg";
4910 my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4912 my $cloudinit_record_changed = sub {
4913 my ($conf, $opt, $old, $new) = @_;
4914 return if !$cloudinit_pending_properties->{$opt};
4916 my $ci = ($conf->{cloudinit} //= {});
4918 my $recorded = $ci->{$opt};
4919 my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4921 if (defined($new)) {
4922 if (defined($old)) {
4923 # an existing value is being modified
4924 if (defined($recorded)) {
4925 # the value was already not in sync
4926 if ($new eq $recorded) {
4927 # a value is being reverted to the cloud-init state:
4929 delete $added{$opt};
4931 # the value was changed multiple times, do nothing
4933 } elsif ($added{$opt}) {
4934 # the value had been marked as added and is being changed, do nothing
4936 # the value is new, record it:
4940 # a new value is being added
4941 if (defined($recorded)) {
4942 # it was already not in sync
4943 if ($new eq $recorded) {
4944 # a value is being reverted to the cloud-init state:
4946 delete $added{$opt};
4948 # the value had temporarily been removed, do nothing
4950 } elsif ($added{$opt}) {
4951 # the value had been marked as added already, do nothing
4953 # the value is new, add it
4957 } elsif (!defined($old)) {
4958 # a non-existent value is being removed? ignore...
4960 # a value is being deleted
4961 if (defined($recorded)) {
4962 # a value was already recorded, just keep it
4963 } elsif ($added{$opt}) {
4964 # the value was marked as added, remove it
4965 delete $added{$opt};
4967 # a previously unrecorded value is being removed, record the old value:
4972 my $added = join(',', sort keys %added);
4973 $ci->{added} = $added if length($added);
4977 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4978 if ($fast_plug_option->{$opt}) {
4979 my $new = delete $conf->{pending}->{$opt};
4980 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4981 $conf->{$opt} = $new;
4987 PVE::QemuConfig->write_config($vmid, $conf);
4990 my $ostype = $conf->{ostype};
4991 my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4992 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4993 my $usb_hotplug = $hotplug_features->{usb}
4994 && min_version($version, 7, 1)
4995 && defined($ostype) && ($ostype eq 'l26
' || windows_version($ostype) > 7);
4997 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4998 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5000 foreach my $opt (sort keys %$pending_delete_hash) {
5001 next if $selection && !$selection->{$opt};
5002 my $force = $pending_delete_hash->{$opt}->{force};
5004 if ($opt eq 'hotplug
') {
5005 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5006 } elsif ($opt eq 'tablet
') {
5007 die "skip\n" if !$hotplug_features->{usb};
5008 if ($defaults->{tablet}) {
5009 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
5010 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
5011 if $arch eq 'aarch64
';
5013 vm_deviceunplug($vmid, $conf, 'tablet
');
5014 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
5016 } elsif ($opt =~ m/^usb(\d+)$/) {
5018 die "skip\n" if !$usb_hotplug;
5019 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5020 vm_deviceunplug
($vmid, $conf, $opt);
5021 } elsif ($opt eq 'vcpus') {
5022 die "skip\n" if !$hotplug_features->{cpu
};
5023 qemu_cpu_hotplug
($vmid, $conf, undef);
5024 } elsif ($opt eq 'balloon') {
5025 # enable balloon device is not hotpluggable
5026 die "skip\n" if defined($conf->{balloon
}) && $conf->{balloon
} == 0;
5027 # here we reset the ballooning value to memory
5028 my $balloon = $conf->{memory
} || $defaults->{memory
};
5029 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5030 } elsif ($fast_plug_option->{$opt}) {
5032 } elsif ($opt =~ m/^net(\d+)$/) {
5033 die "skip\n" if !$hotplug_features->{network
};
5034 vm_deviceunplug
($vmid, $conf, $opt);
5035 } elsif (is_valid_drivename
($opt)) {
5036 die "skip\n" if !$hotplug_features->{disk
} || $opt =~ m/(ide|sata)(\d+)/;
5037 vm_deviceunplug
($vmid, $conf, $opt);
5038 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5039 } elsif ($opt =~ m/^memory$/) {
5040 die "skip\n" if !$hotplug_features->{memory
};
5041 PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $defaults, $opt);
5042 } elsif ($opt eq 'cpuunits') {
5043 $cgroup->change_cpu_shares(undef);
5044 } elsif ($opt eq 'cpulimit') {
5045 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5051 &$add_error($opt, $err) if $err ne "skip\n";
5053 my $old = delete $conf->{$opt};
5054 $cloudinit_record_changed->($conf, $opt, $old, undef);
5055 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5060 foreach my $opt (keys %{$conf->{pending
}}) {
5061 next if $selection && !$selection->{$opt};
5062 my $value = $conf->{pending
}->{$opt};
5064 if ($opt eq 'hotplug') {
5065 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug
} =~ /memory/);
5066 } elsif ($opt eq 'tablet') {
5067 die "skip\n" if !$hotplug_features->{usb
};
5069 vm_deviceplug
($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5070 vm_deviceplug
($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5071 if $arch eq 'aarch64';
5072 } elsif ($value == 0) {
5073 vm_deviceunplug
($vmid, $conf, 'tablet');
5074 vm_deviceunplug
($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5076 } elsif ($opt =~ m/^usb(\d+)$/) {
5078 die "skip\n" if !$usb_hotplug;
5079 my $d = eval { parse_property_string
($usbdesc->{format
}, $value) };
5081 if ($d->{host
} eq 'spice') {
5082 $id = "usbredirdev$index";
5084 qemu_usb_hotplug
($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5085 } elsif ($opt eq 'vcpus') {
5086 die "skip\n" if !$hotplug_features->{cpu
};
5087 qemu_cpu_hotplug
($vmid, $conf, $value);
5088 } elsif ($opt eq 'balloon') {
5089 # enable/disable balloning device is not hotpluggable
5090 my $old_balloon_enabled = !!(!defined($conf->{balloon
}) || $conf->{balloon
});
5091 my $new_balloon_enabled = !!(!defined($conf->{pending
}->{balloon
}) || $conf->{pending
}->{balloon
});
5092 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5094 # allow manual ballooning if shares is set to zero
5095 if ((defined($conf->{shares
}) && ($conf->{shares
} == 0))) {
5096 my $balloon = $conf->{pending
}->{balloon
} || $conf->{memory
} || $defaults->{memory
};
5097 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5099 } elsif ($opt =~ m/^net(\d+)$/) {
5100 # some changes can be done without hotplug
5101 vmconfig_update_net
($storecfg, $conf, $hotplug_features->{network
},
5102 $vmid, $opt, $value, $arch, $machine_type);
5103 } elsif (is_valid_drivename
($opt)) {
5104 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5105 # some changes can be done without hotplug
5106 my $drive = parse_drive
($opt, $value);
5107 if (drive_is_cloudinit
($drive)) {
5108 $cloudinit_opt = [$opt, $drive];
5109 # apply all the other changes first, then generate the cloudinit disk
5112 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5113 $vmid, $opt, $value, $arch, $machine_type);
5114 } elsif ($opt =~ m/^memory$/) { #dimms
5115 die "skip\n" if !$hotplug_features->{memory
};
5116 $value = PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $defaults, $opt, $value);
5117 } elsif ($opt eq 'cpuunits') {
5118 my $new_cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{pending
}->{$opt}); #clamp
5119 $cgroup->change_cpu_shares($new_cpuunits);
5120 } elsif ($opt eq 'cpulimit') {
5121 my $cpulimit = $conf->{pending
}->{$opt} == 0 ?
-1 : int($conf->{pending
}->{$opt} * 100000);
5122 $cgroup->change_cpu_quota($cpulimit, 100000);
5123 } elsif ($opt eq 'agent') {
5124 vmconfig_update_agent
($conf, $opt, $value);
5126 die "skip\n"; # skip non-hot-pluggable options
5130 &$add_error($opt, $err) if $err ne "skip\n";
5132 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5133 $conf->{$opt} = $value;
5134 delete $conf->{pending
}->{$opt};
5138 if (defined($cloudinit_opt)) {
5139 my ($opt, $drive) = @$cloudinit_opt;
5140 my $value = $conf->{pending
}->{$opt};
5142 my $temp = {%$conf, $opt => $value};
5143 PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($temp, $vmid);
5144 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5145 $vmid, $opt, $value, $arch, $machine_type);
5148 &$add_error($opt, $err) if $err ne "skip\n";
5150 $conf->{$opt} = $value;
5151 delete $conf->{pending
}->{$opt};
5155 # unplug xhci controller if no usb device is left
5158 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5159 next if !defined($conf->{"usb$i"});
5164 vm_deviceunplug
($vmid, $conf, 'xhci');
5168 PVE
::QemuConfig-
>write_config($vmid, $conf);
5170 if ($hotplug_features->{cloudinit
} && PVE
::QemuServer
::Cloudinit
::has_changes
($conf)) {
5171 PVE
::QemuServer
::vmconfig_update_cloudinit_drive
($storecfg, $conf, $vmid);
5175 sub try_deallocate_drive
{
5176 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5178 if (($force || $key =~ /^unused/) && !drive_is_cdrom
($drive, 1)) {
5179 my $volid = $drive->{file
};
5180 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
5181 my $sid = PVE
::Storage
::parse_volume_id
($volid);
5182 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5184 # check if the disk is really unused
5185 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5186 if PVE
::QemuServer
::Drive
::is_volume_in_use
($storecfg, $conf, $key, $volid);
5187 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5190 # If vm is not owner of this disk remove from config
5198 sub vmconfig_delete_or_detach_drive
{
5199 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5201 my $drive = parse_drive
($opt, $conf->{$opt});
5203 my $rpcenv = PVE
::RPCEnvironment
::get
();
5204 my $authuser = $rpcenv->get_user();
5207 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5208 try_deallocate_drive
($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5210 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $drive);
5216 sub vmconfig_apply_pending
{
5217 my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5219 return if !scalar(keys %{$conf->{pending
}});
5221 my $add_apply_error = sub {
5222 my ($opt, $msg) = @_;
5223 my $err_msg = "unable to apply pending change $opt : $msg";
5224 $errors->{$opt} = $err_msg;
5230 my $pending_delete_hash = PVE
::QemuConfig-
>parse_pending_delete($conf->{pending
}->{delete});
5231 foreach my $opt (sort keys %$pending_delete_hash) {
5232 my $force = $pending_delete_hash->{$opt}->{force
};
5234 if ($opt =~ m/^unused/) {
5235 die "internal error";
5236 } elsif (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5237 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5241 $add_apply_error->($opt, $err);
5243 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5244 delete $conf->{$opt};
5248 PVE
::QemuConfig-
>cleanup_pending($conf);
5250 my $generate_cloudinit = $skip_cloud_init ?
0 : undef;
5252 foreach my $opt (keys %{$conf->{pending
}}) { # add/change
5253 next if $opt eq 'delete'; # just to be sure
5255 if (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5256 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, parse_drive
($opt, $conf->{$opt}))
5260 $add_apply_error->($opt, $err);
5263 if (is_valid_drivename
($opt)) {
5264 my $drive = parse_drive
($opt, $conf->{pending
}->{$opt});
5265 $generate_cloudinit //= 1 if drive_is_cloudinit
($drive);
5268 $conf->{$opt} = delete $conf->{pending
}->{$opt};
5272 # write all changes at once to avoid unnecessary i/o
5273 PVE
::QemuConfig-
>write_config($vmid, $conf);
5274 if ($generate_cloudinit) {
5275 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5276 # After successful generation and if there were changes to be applied, update the
5277 # config to drop the {cloudinit} entry.
5278 PVE
::QemuConfig-
>write_config($vmid, $conf);
5283 sub vmconfig_update_net
{
5284 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5286 my $newnet = parse_net
($value);
5288 if ($conf->{$opt}) {
5289 my $oldnet = parse_net
($conf->{$opt});
5291 if (safe_string_ne
($oldnet->{model
}, $newnet->{model
}) ||
5292 safe_string_ne
($oldnet->{macaddr
}, $newnet->{macaddr
}) ||
5293 safe_num_ne
($oldnet->{queues
}, $newnet->{queues
}) ||
5294 !($newnet->{bridge
} && $oldnet->{bridge
})) { # bridge/nat mode change
5296 # for non online change, we try to hot-unplug
5297 die "skip\n" if !$hotplug;
5298 vm_deviceunplug
($vmid, $conf, $opt);
5301 die "internal error" if $opt !~ m/net(\d+)/;
5302 my $iface = "tap${vmid}i$1";
5304 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5305 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
}) ||
5306 safe_string_ne
($oldnet->{trunks
}, $newnet->{trunks
}) ||
5307 safe_num_ne
($oldnet->{firewall
}, $newnet->{firewall
})) {
5308 PVE
::Network
::tap_unplug
($iface);
5311 PVE
::Network
::SDN
::Zones
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5313 PVE
::Network
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5315 } elsif (safe_num_ne
($oldnet->{rate
}, $newnet->{rate
})) {
5316 # Rate can be applied on its own but any change above needs to
5317 # include the rate in tap_plug since OVS resets everything.
5318 PVE
::Network
::tap_rate_limit
($iface, $newnet->{rate
});
5321 if (safe_string_ne
($oldnet->{link_down
}, $newnet->{link_down
})) {
5322 qemu_set_link_status
($vmid, $opt, !$newnet->{link_down
});
5330 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5336 sub vmconfig_update_agent
{
5337 my ($conf, $opt, $value) = @_;
5339 die "skip\n" if !$conf->{$opt};
5341 my $hotplug_options = { fstrim_cloned_disks
=> 1 };
5343 my $old_agent = parse_guest_agent
($conf);
5344 my $agent = parse_guest_agent
({$opt => $value});
5346 for my $option (keys %$agent) { # added/changed options
5347 next if defined($hotplug_options->{$option});
5348 die "skip\n" if safe_string_ne
($agent->{$option}, $old_agent->{$option});
5351 for my $option (keys %$old_agent) { # removed options
5352 next if defined($hotplug_options->{$option});
5353 die "skip\n" if safe_string_ne
($old_agent->{$option}, $agent->{$option});
5356 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5359 sub vmconfig_update_disk
{
5360 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5362 my $drive = parse_drive
($opt, $value);
5364 if ($conf->{$opt} && (my $old_drive = parse_drive
($opt, $conf->{$opt}))) {
5365 my $media = $drive->{media
} || 'disk';
5366 my $oldmedia = $old_drive->{media
} || 'disk';
5367 die "unable to change media type\n" if $media ne $oldmedia;
5369 if (!drive_is_cdrom
($old_drive)) {
5371 if ($drive->{file
} ne $old_drive->{file
}) {
5373 die "skip\n" if !$hotplug;
5375 # unplug and register as unused
5376 vm_deviceunplug
($vmid, $conf, $opt);
5377 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive)
5380 # update existing disk
5382 # skip non hotpluggable value
5383 if (safe_string_ne
($drive->{discard
}, $old_drive->{discard
}) ||
5384 safe_string_ne
($drive->{iothread
}, $old_drive->{iothread
}) ||
5385 safe_string_ne
($drive->{queues
}, $old_drive->{queues
}) ||
5386 safe_string_ne
($drive->{cache
}, $old_drive->{cache
}) ||
5387 safe_string_ne
($drive->{ssd
}, $old_drive->{ssd
}) ||
5388 safe_string_ne
($drive->{ro
}, $old_drive->{ro
})) {
5393 if (safe_num_ne
($drive->{mbps
}, $old_drive->{mbps
}) ||
5394 safe_num_ne
($drive->{mbps_rd
}, $old_drive->{mbps_rd
}) ||
5395 safe_num_ne
($drive->{mbps_wr
}, $old_drive->{mbps_wr
}) ||
5396 safe_num_ne
($drive->{iops
}, $old_drive->{iops
}) ||
5397 safe_num_ne
($drive->{iops_rd
}, $old_drive->{iops_rd
}) ||
5398 safe_num_ne
($drive->{iops_wr
}, $old_drive->{iops_wr
}) ||
5399 safe_num_ne
($drive->{mbps_max
}, $old_drive->{mbps_max
}) ||
5400 safe_num_ne
($drive->{mbps_rd_max
}, $old_drive->{mbps_rd_max
}) ||
5401 safe_num_ne
($drive->{mbps_wr_max
}, $old_drive->{mbps_wr_max
}) ||
5402 safe_num_ne
($drive->{iops_max
}, $old_drive->{iops_max
}) ||
5403 safe_num_ne
($drive->{iops_rd_max
}, $old_drive->{iops_rd_max
}) ||
5404 safe_num_ne
($drive->{iops_wr_max
}, $old_drive->{iops_wr_max
}) ||
5405 safe_num_ne
($drive->{bps_max_length
}, $old_drive->{bps_max_length
}) ||
5406 safe_num_ne
($drive->{bps_rd_max_length
}, $old_drive->{bps_rd_max_length
}) ||
5407 safe_num_ne
($drive->{bps_wr_max_length
}, $old_drive->{bps_wr_max_length
}) ||
5408 safe_num_ne
($drive->{iops_max_length
}, $old_drive->{iops_max_length
}) ||
5409 safe_num_ne
($drive->{iops_rd_max_length
}, $old_drive->{iops_rd_max_length
}) ||
5410 safe_num_ne
($drive->{iops_wr_max_length
}, $old_drive->{iops_wr_max_length
})) {
5412 qemu_block_set_io_throttle
(
5414 ($drive->{mbps
} || 0)*1024*1024,
5415 ($drive->{mbps_rd
} || 0)*1024*1024,
5416 ($drive->{mbps_wr
} || 0)*1024*1024,
5417 $drive->{iops
} || 0,
5418 $drive->{iops_rd
} || 0,
5419 $drive->{iops_wr
} || 0,
5420 ($drive->{mbps_max
} || 0)*1024*1024,
5421 ($drive->{mbps_rd_max
} || 0)*1024*1024,
5422 ($drive->{mbps_wr_max
} || 0)*1024*1024,
5423 $drive->{iops_max
} || 0,
5424 $drive->{iops_rd_max
} || 0,
5425 $drive->{iops_wr_max
} || 0,
5426 $drive->{bps_max_length
} || 1,
5427 $drive->{bps_rd_max_length
} || 1,
5428 $drive->{bps_wr_max_length
} || 1,
5429 $drive->{iops_max_length
} || 1,
5430 $drive->{iops_rd_max_length
} || 1,
5431 $drive->{iops_wr_max_length
} || 1,
5441 if ($drive->{file
} eq 'none') {
5442 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5443 if (drive_is_cloudinit
($old_drive)) {
5444 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive);
5447 my $path = get_iso_path
($storecfg, $vmid, $drive->{file
});
5449 # force eject if locked
5450 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5453 mon_cmd
($vmid, "blockdev-change-medium",
5454 id
=> "$opt", filename
=> "$path");
5462 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5464 PVE
::Storage
::activate_volumes
($storecfg, [$drive->{file
}]) if $drive->{file
} !~ m
|^/dev/.+|;
5465 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5468 sub vmconfig_update_cloudinit_drive
{
5469 my ($storecfg, $conf, $vmid) = @_;
5471 my $cloudinit_ds = undef;
5472 my $cloudinit_drive = undef;
5474 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5475 my ($ds, $drive) = @_;
5476 if (PVE
::QemuServer
::drive_is_cloudinit
($drive)) {
5477 $cloudinit_ds = $ds;
5478 $cloudinit_drive = $drive;
5482 return if !$cloudinit_drive;
5484 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5485 PVE
::QemuConfig-
>write_config($vmid, $conf);
5488 my $running = PVE
::QemuServer
::check_running
($vmid);
5491 my $path = PVE
::Storage
::path
($storecfg, $cloudinit_drive->{file
});
5493 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$cloudinit_ds");
5494 mon_cmd
($vmid, "blockdev-change-medium", id
=> "$cloudinit_ds", filename
=> "$path");
5499 # called in locked context by incoming migration
5500 sub vm_migrate_get_nbd_disks
{
5501 my ($storecfg, $conf, $replicated_volumes) = @_;
5503 my $local_volumes = {};
5504 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5505 my ($ds, $drive) = @_;
5507 return if drive_is_cdrom
($drive);
5508 return if $ds eq 'tpmstate0';
5510 my $volid = $drive->{file
};
5514 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid);
5516 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5517 return if $scfg->{shared
};
5519 # replicated disks re-use existing state via bitmap
5520 my $use_existing = $replicated_volumes->{$volid} ?
1 : 0;
5521 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5523 return $local_volumes;
5526 # called in locked context by incoming migration
5527 sub vm_migrate_alloc_nbd_disks
{
5528 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5531 foreach my $opt (sort keys %$source_volumes) {
5532 my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5534 if ($use_existing) {
5535 $nbd->{$opt}->{drivestr
} = print_drive
($drive);
5536 $nbd->{$opt}->{volid
} = $volid;
5537 $nbd->{$opt}->{replicated
} = 1;
5541 # storage mapping + volname = regular migration
5542 # storage mapping + format = remote migration
5543 # order of precedence, filtered by whether storage supports it:
5544 # 1. explicit requested format
5545 # 2. format of current volume
5546 # 3. default format of storage
5547 if (!$storagemap->{identity
}) {
5548 $storeid = PVE
::JSONSchema
::map_id
($storagemap, $storeid);
5549 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5550 if (!$format || !grep { $format eq $_ } @$validFormats) {
5552 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5553 my $fileFormat = qemu_img_format
($scfg, $volname);
5554 $format = $fileFormat
5555 if grep { $fileFormat eq $_ } @$validFormats;
5557 $format //= $defFormat;
5560 # can't happen for remote migration, so $volname is always defined
5561 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5562 $format = qemu_img_format
($scfg, $volname);
5565 my $size = $drive->{size
} / 1024;
5566 my $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $format, undef, $size);
5567 my $newdrive = $drive;
5568 $newdrive->{format
} = $format;
5569 $newdrive->{file
} = $newvolid;
5570 my $drivestr = print_drive
($newdrive);
5571 $nbd->{$opt}->{drivestr
} = $drivestr;
5572 $nbd->{$opt}->{volid
} = $newvolid;
5578 # see vm_start_nolock for parameters, additionally:
5580 # storagemap = parsed storage map for allocating NBD disks
5582 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5584 return PVE
::QemuConfig-
>lock_config($vmid, sub {
5585 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migrate_opts->{migratedfrom
});
5587 die "you can't start a vm if it's a template\n"
5588 if !$params->{skiptemplate
} && PVE
::QemuConfig-
>is_template($conf);
5590 my $has_suspended_lock = PVE
::QemuConfig-
>has_lock($conf, 'suspended');
5591 my $has_backup_lock = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5593 my $running = check_running
($vmid, undef, $migrate_opts->{migratedfrom
});
5595 if ($has_backup_lock && $running) {
5596 # a backup is currently running, attempt to start the guest in the
5597 # existing QEMU instance
5598 return vm_resume
($vmid);
5601 PVE
::QemuConfig-
>check_lock($conf)
5602 if !($params->{skiplock
} || $has_suspended_lock);
5604 $params->{resume
} = $has_suspended_lock || defined($conf->{vmstate
});
5606 die "VM $vmid already running\n" if $running;
5608 if (my $storagemap = $migrate_opts->{storagemap
}) {
5609 my $replicated = $migrate_opts->{replicated_volumes
};
5610 my $disks = vm_migrate_get_nbd_disks
($storecfg, $conf, $replicated);
5611 $migrate_opts->{nbd
} = vm_migrate_alloc_nbd_disks
($storecfg, $vmid, $disks, $storagemap);
5613 foreach my $opt (keys %{$migrate_opts->{nbd
}}) {
5614 $conf->{$opt} = $migrate_opts->{nbd
}->{$opt}->{drivestr
};
5618 return vm_start_nolock
($storecfg, $vmid, $conf, $params, $migrate_opts);
5624 # statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5625 # skiplock => 0/1, skip checking for config lock
5626 # skiptemplate => 0/1, skip checking whether VM is template
5627 # forcemachine => to force QEMU machine (rollback/migration)
5628 # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5629 # timeout => in seconds
5630 # paused => start VM in paused state (backup)
5631 # resume => resume from hibernation
5642 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5643 # migratedfrom => source node
5644 # spice_ticket => used for spice migration, passed via tunnel/stdin
5645 # network => CIDR of migration network
5646 # type => secure/insecure - tunnel over encrypted connection or plain-text
5647 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5648 # replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5649 # offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5650 # contained in config
5651 sub vm_start_nolock
{
5652 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5654 my $statefile = $params->{statefile
};
5655 my $resume = $params->{resume
};
5657 my $migratedfrom = $migrate_opts->{migratedfrom
};
5658 my $migration_type = $migrate_opts->{type
};
5662 # clean up leftover reboot request files
5663 eval { clear_reboot_request
($vmid); };
5666 if (!$statefile && scalar(keys %{$conf->{pending
}})) {
5667 vmconfig_apply_pending
($vmid, $conf, $storecfg);
5668 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5671 # don't regenerate the ISO if the VM is started as part of a live migration
5672 # this way we can reuse the old ISO with the correct config
5673 if (!$migratedfrom) {
5674 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5675 # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5676 # $conf->{cloudinit}, so we could just not do this?
5677 # But we do it above, so for now let's be consistent.
5678 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5682 # override offline migrated volumes, conf is out of date still
5683 if (my $offline_volumes = $migrate_opts->{offline_volumes
}) {
5684 for my $key (sort keys $offline_volumes->%*) {
5685 my $parsed = parse_drive
($key, $conf->{$key});
5686 $parsed->{file
} = $offline_volumes->{$key};
5687 $conf->{$key} = print_drive
($parsed);
5691 my $defaults = load_defaults
();
5693 # set environment variable useful inside network script
5694 # for remote migration the config is available on the target node!
5695 if (!$migrate_opts->{remote_node
}) {
5696 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom;
5699 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5701 my $forcemachine = $params->{forcemachine
};
5702 my $forcecpu = $params->{forcecpu
};
5704 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5705 $forcemachine = $conf->{runningmachine
};
5706 $forcecpu = $conf->{runningcpu
};
5707 print "Resuming suspended VM\n";
5710 my ($cmd, $vollist, $spice_port) = config_to_command
($storecfg, $vmid,
5711 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5714 my $get_migration_ip = sub {
5715 my ($nodename) = @_;
5717 return $migration_ip if defined($migration_ip);
5719 my $cidr = $migrate_opts->{network
};
5721 if (!defined($cidr)) {
5722 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5723 $cidr = $dc_conf->{migration
}->{network
};
5726 if (defined($cidr)) {
5727 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5729 die "could not get IP: no address configured on local " .
5730 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5732 die "could not get IP: multiple addresses configured on local " .
5733 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5735 $migration_ip = @$ips[0];
5738 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5739 if !defined($migration_ip);
5741 return $migration_ip;
5745 if ($statefile eq 'tcp') {
5746 my $migrate = $res->{migrate
} = { proto
=> 'tcp' };
5747 $migrate->{addr
} = "localhost";
5748 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5749 my $nodename = nodename
();
5751 if (!defined($migration_type)) {
5752 if (defined($datacenterconf->{migration
}->{type
})) {
5753 $migration_type = $datacenterconf->{migration
}->{type
};
5755 $migration_type = 'secure';
5759 if ($migration_type eq 'insecure') {
5760 $migrate->{addr
} = $get_migration_ip->($nodename);
5761 $migrate->{addr
} = "[$migrate->{addr}]" if Net
::IP
::ip_is_ipv6
($migrate->{addr
});
5764 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5765 $migrate->{port
} = PVE
::Tools
::next_migrate_port
($pfamily);
5766 $migrate->{uri
} = "tcp:$migrate->{addr}:$migrate->{port}";
5767 push @$cmd, '-incoming', $migrate->{uri
};
5770 } elsif ($statefile eq 'unix') {
5771 # should be default for secure migrations as a ssh TCP forward
5772 # tunnel is not deterministic reliable ready and fails regurarly
5773 # to set up in time, so use UNIX socket forwards
5774 my $migrate = $res->{migrate
} = { proto
=> 'unix' };
5775 $migrate->{addr
} = "/run/qemu-server/$vmid.migrate";
5776 unlink $migrate->{addr
};
5778 $migrate->{uri
} = "unix:$migrate->{addr}";
5779 push @$cmd, '-incoming', $migrate->{uri
};
5782 } elsif (-e
$statefile) {
5783 push @$cmd, '-loadstate', $statefile;
5785 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5786 push @$vollist, $statefile;
5787 push @$cmd, '-loadstate', $statepath;
5789 } elsif ($params->{paused
}) {
5793 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $resume);
5795 my $pci_devices = {}; # host pci devices
5796 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
5797 my $dev = $conf->{"hostpci$i"} or next;
5798 $pci_devices->{$i} = parse_hostpci
($dev);
5801 # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5802 my $real_pci_devices = [ grep { !(defined($_->{mdev
}) && scalar($_->{pciid
}->@*) == 1) } values $pci_devices->%* ];
5804 # map to a flat list of pci ids
5805 my $pci_id_list = [ map { $_->{id
} } map { $_->{pciid
}->@* } $real_pci_devices->@* ];
5807 # reserve all PCI IDs before actually doing anything with them
5808 PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_id_list, $vmid, $start_timeout);
5812 for my $id (sort keys %$pci_devices) {
5813 my $d = $pci_devices->{$id};
5814 for my $dev ($d->{pciid
}->@*) {
5815 my $info = PVE
::QemuServer
::PCI
::prepare_pci_device
($vmid, $dev->{id
}, $id, $d->{mdev
});
5817 # nvidia grid needs the uuid of the mdev as qemu parameter
5818 if ($d->{mdev
} && !defined($uuid) && $info->{vendor
} eq '10de') {
5819 $uuid = PVE
::QemuServer
::PCI
::generate_mdev_uuid
($vmid, $id);
5823 push @$cmd, '-uuid', $uuid if defined($uuid);
5826 eval { cleanup_pci_devices
($vmid, $conf) };
5831 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5834 run_command
(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc
=> sub{}, errfunc
=> sub{});
5836 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5837 # timeout should be more than enough here...
5838 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 20);
5840 my $cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{cpuunits
});
5843 timeout
=> $statefile ?
undef : $start_timeout,
5848 # when migrating, prefix QEMU output so other side can pick up any
5849 # errors that might occur and show the user
5850 if ($migratedfrom) {
5851 $run_params{quiet
} = 1;
5852 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5855 my %systemd_properties = (
5856 Slice
=> 'qemu.slice',
5857 KillMode
=> 'process',
5859 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5862 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5863 $systemd_properties{CPUWeight
} = $cpuunits;
5865 $systemd_properties{CPUShares
} = $cpuunits;
5868 if (my $cpulimit = $conf->{cpulimit
}) {
5869 $systemd_properties{CPUQuota
} = int($cpulimit * 100);
5871 $systemd_properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5873 my $run_qemu = sub {
5874 PVE
::Tools
::run_fork
sub {
5875 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5878 if (my $tpm = $conf->{tpmstate0
}) {
5879 # start the TPM emulator so QEMU can connect on start
5880 $tpmpid = start_swtpm
($storecfg, $vmid, $tpm, $migratedfrom);
5883 my $exitcode = run_command
($cmd, %run_params);
5886 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5887 kill 'TERM', $tpmpid;
5889 die "QEMU exited with code $exitcode\n";
5894 if ($conf->{hugepages
}) {
5897 my $hugepages_topology = PVE
::QemuServer
::Memory
::hugepages_topology
($conf);
5898 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5900 PVE
::QemuServer
::Memory
::hugepages_mount
();
5901 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5903 eval { $run_qemu->() };
5905 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5906 if !$conf->{keephugepages
};
5910 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5911 if !$conf->{keephugepages
};
5913 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5916 eval { $run_qemu->() };
5920 # deactivate volumes if start fails
5921 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5923 eval { cleanup_pci_devices
($vmid, $conf) };
5926 die "start failed: $err";
5929 # re-reserve all PCI IDs now that we can know the actual VM PID
5930 my $pid = PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
5931 eval { PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_id_list, $vmid, undef, $pid) };
5934 if (defined($res->{migrate
})) {
5935 print "migration listens on $res->{migrate}->{uri}\n";
5936 } elsif ($statefile) {
5937 eval { mon_cmd
($vmid, "cont"); };
5941 #start nbd server for storage migration
5942 if (my $nbd = $migrate_opts->{nbd
}) {
5943 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5945 my $migrate_storage_uri;
5946 # nbd_protocol_version > 0 for unix socket support
5947 if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5948 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5949 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
5950 $migrate_storage_uri = "nbd:unix:$socket_path";
5951 $res->{migrate
}->{unix_sockets
} = [$socket_path];
5953 my $nodename = nodename
();
5954 my $localip = $get_migration_ip->($nodename);
5955 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5956 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5958 mon_cmd
($vmid, "nbd-server-start", addr
=> {
5961 host
=> "${localip}",
5962 port
=> "${storage_migrate_port}",
5965 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5966 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5969 foreach my $opt (sort keys %$nbd) {
5970 my $drivestr = $nbd->{$opt}->{drivestr
};
5971 my $volid = $nbd->{$opt}->{volid
};
5972 mon_cmd
($vmid, "nbd-server-add", device
=> "drive-$opt", writable
=> JSON
::true
);
5973 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5974 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5975 print "re-using replicated volume: $opt - $volid\n"
5976 if $nbd->{$opt}->{replicated
};
5978 $res->{drives
}->{$opt} = $nbd->{$opt};
5979 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
5983 if ($migratedfrom) {
5985 set_migration_caps
($vmid);
5990 print "spice listens on port $spice_port\n";
5991 $res->{spice_port
} = $spice_port;
5992 if ($migrate_opts->{spice_ticket
}) {
5993 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
5994 $migrate_opts->{spice_ticket
});
5995 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
6000 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
6001 if !$statefile && $conf->{balloon
};
6003 foreach my $opt (keys %$conf) {
6004 next if $opt !~ m/^net\d+$/;
6005 my $nicconf = parse_net
($conf->{$opt});
6006 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
6008 add_nets_bridge_fdb
($conf, $vmid);
6011 mon_cmd
($vmid, 'qom-set',
6012 path
=> "machine/peripheral/balloon0",
6013 property
=> "guest-stats-polling-interval",
6014 value
=> 2) if (!defined($conf->{balloon
}) || $conf->{balloon
});
6017 print "Resumed VM, removing state\n";
6018 if (my $vmstate = $conf->{vmstate
}) {
6019 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6020 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6022 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6023 PVE
::QemuConfig-
>write_config($vmid, $conf);
6026 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
6031 sub vm_commandline
{
6032 my ($storecfg, $vmid, $snapname) = @_;
6034 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6036 my ($forcemachine, $forcecpu);
6038 my $snapshot = $conf->{snapshots
}->{$snapname};
6039 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6041 # check for machine or CPU overrides in snapshot
6042 $forcemachine = $snapshot->{runningmachine
};
6043 $forcecpu = $snapshot->{runningcpu
};
6045 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
6050 my $defaults = load_defaults
();
6052 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6054 return PVE
::Tools
::cmd2string
($cmd);
6058 my ($vmid, $skiplock) = @_;
6060 PVE
::QemuConfig-
>lock_config($vmid, sub {
6062 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6064 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6066 mon_cmd
($vmid, "system_reset");
6070 sub get_vm_volumes
{
6074 foreach_volid
($conf, sub {
6075 my ($volid, $attr) = @_;
6077 return if $volid =~ m
|^/|;
6079 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6082 push @$vollist, $volid;
6088 sub cleanup_pci_devices
{
6089 my ($vmid, $conf) = @_;
6091 foreach my $key (keys %$conf) {
6092 next if $key !~ m/^hostpci(\d+)$/;
6093 my $hostpciindex = $1;
6094 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
6095 my $d = parse_hostpci
($conf->{$key});
6097 # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6098 # don't want to break ABI just for this two liner
6099 my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6100 PVE
::SysFSTools
::file_write
("$dev_sysfs_dir/remove", "1") if -e
$dev_sysfs_dir;
6103 PVE
::QemuServer
::PCI
::remove_pci_reservation
($vmid);
6106 sub vm_stop_cleanup
{
6107 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6112 my $vollist = get_vm_volumes
($conf);
6113 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6115 if (my $tpmdrive = $conf->{tpmstate0
}) {
6116 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
6117 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
6119 PVE
::Storage
::unmap_volume
($storecfg, $tpm->{file
});
6124 foreach my $ext (qw(mon qmp pid vnc qga)) {
6125 unlink "/var/run/qemu-server/${vmid}.$ext";
6128 if ($conf->{ivshmem
}) {
6129 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
6130 # just delete it for now, VMs which have this already open do not
6131 # are affected, but new VMs will get a separated one. If this
6132 # becomes an issue we either add some sort of ref-counting or just
6133 # add a "don't delete on stop" flag to the ivshmem format.
6134 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
6137 cleanup_pci_devices
($vmid, $conf);
6139 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
6141 warn $@ if $@; # avoid errors - just warn
6144 # call only in locked context
6146 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6148 my $pid = check_running
($vmid, $nocheck);
6153 $conf = PVE
::QemuConfig-
>load_config($vmid);
6154 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6155 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
6156 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
6157 $timeout = $opts->{down
} if $opts->{down
};
6159 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
6164 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
6165 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
6167 mon_cmd
($vmid, "system_powerdown");
6170 mon_cmd
($vmid, "quit");
6176 $timeout = 60 if !defined($timeout);
6179 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6184 if ($count >= $timeout) {
6186 warn "VM still running - terminating now with SIGTERM\n";
6189 die "VM quit/powerdown failed - got timeout\n";
6192 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6196 if (!check_running
($vmid, $nocheck)) {
6197 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6201 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6204 die "VM quit/powerdown failed\n";
6212 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6217 if ($count >= $timeout) {
6218 warn "VM still running - terminating now with SIGKILL\n";
6223 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6226 # Note: use $nocheck to skip tests if VM configuration file exists.
6227 # We need that when migration VMs to other nodes (files already moved)
6228 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6230 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6232 $force = 1 if !defined($force) && !$shutdown;
6235 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
6236 kill 15, $pid if $pid;
6237 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
6238 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
6242 PVE
::QemuConfig-
>lock_config($vmid, sub {
6243 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6248 my ($vmid, $timeout) = @_;
6250 PVE
::QemuConfig-
>lock_config($vmid, sub {
6253 # only reboot if running, as qmeventd starts it again on a stop event
6254 return if !check_running
($vmid);
6256 create_reboot_request
($vmid);
6258 my $storecfg = PVE
::Storage
::config
();
6259 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
6263 # avoid that the next normal shutdown will be confused for a reboot
6264 clear_reboot_request
($vmid);
6270 # note: if using the statestorage parameter, the caller has to check privileges
6272 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6279 PVE
::QemuConfig-
>lock_config($vmid, sub {
6281 $conf = PVE
::QemuConfig-
>load_config($vmid);
6283 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
6284 PVE
::QemuConfig-
>check_lock($conf)
6285 if !($skiplock || $is_backing_up);
6287 die "cannot suspend to disk during backup\n"
6288 if $is_backing_up && $includestate;
6290 if ($includestate) {
6291 $conf->{lock} = 'suspending';
6292 my $date = strftime
("%Y-%m-%d", localtime(time()));
6293 $storecfg = PVE
::Storage
::config
();
6294 if (!$statestorage) {
6295 $statestorage = find_vmstate_storage
($conf, $storecfg);
6296 # check permissions for the storage
6297 my $rpcenv = PVE
::RPCEnvironment
::get
();
6298 if ($rpcenv->{type
} ne 'cli') {
6299 my $authuser = $rpcenv->get_user();
6300 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6305 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
6306 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6307 $path = PVE
::Storage
::path
($storecfg, $vmstate);
6308 PVE
::QemuConfig-
>write_config($vmid, $conf);
6310 mon_cmd
($vmid, "stop");
6314 if ($includestate) {
6316 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
6319 set_migration_caps
($vmid, 1);
6320 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
6322 my $state = mon_cmd
($vmid, "query-savevm");
6323 if (!$state->{status
}) {
6324 die "savevm not active\n";
6325 } elsif ($state->{status
} eq 'active') {
6328 } elsif ($state->{status
} eq 'completed') {
6329 print "State saved, quitting\n";
6331 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
6332 die "query-savevm failed with error '$state->{error}'\n"
6334 die "query-savevm returned status '$state->{status}'\n";
6340 PVE
::QemuConfig-
>lock_config($vmid, sub {
6341 $conf = PVE
::QemuConfig-
>load_config($vmid);
6343 # cleanup, but leave suspending lock, to indicate something went wrong
6345 mon_cmd
($vmid, "savevm-end");
6346 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6347 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6348 delete $conf->@{qw(vmstate runningmachine runningcpu)};
6349 PVE
::QemuConfig-
>write_config($vmid, $conf);
6355 die "lock changed unexpectedly\n"
6356 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
6358 mon_cmd
($vmid, "quit");
6359 $conf->{lock} = 'suspended';
6360 PVE
::QemuConfig-
>write_config($vmid, $conf);
6365 # $nocheck is set when called as part of a migration - in this context the
6366 # location of the config file (source or target node) is not deterministic,
6367 # since migration cannot wait for pmxcfs to process the rename
6369 my ($vmid, $skiplock, $nocheck) = @_;
6371 PVE
::QemuConfig-
>lock_config($vmid, sub {
6372 my $res = mon_cmd
($vmid, 'query-status');
6373 my $resume_cmd = 'cont';
6377 $conf = eval { PVE
::QemuConfig-
>load_config($vmid) }; # try on target node
6379 my $vmlist = PVE
::Cluster
::get_vmlist
();
6380 if (exists($vmlist->{ids
}->{$vmid})) {
6381 my $node = $vmlist->{ids
}->{$vmid}->{node
};
6382 $conf = eval { PVE
::QemuConfig-
>load_config($vmid, $node) }; # try on source node
6385 PVE
::Cluster
::cfs_update
(); # vmlist was wrong, invalidate cache
6386 $conf = PVE
::QemuConfig-
>load_config($vmid); # last try on target node again
6390 $conf = PVE
::QemuConfig-
>load_config($vmid);
6393 if ($res->{status
}) {
6394 return if $res->{status
} eq 'running'; # job done, go home
6395 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
6396 $reset = 1 if $res->{status
} eq 'shutdown';
6400 PVE
::QemuConfig-
>check_lock($conf)
6401 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
6405 # required if a VM shuts down during a backup and we get a resume
6406 # request before the backup finishes for example
6407 mon_cmd
($vmid, "system_reset");
6410 add_nets_bridge_fdb
($conf, $vmid) if $resume_cmd eq 'cont';
6412 mon_cmd
($vmid, $resume_cmd);
6417 my ($vmid, $skiplock, $key) = @_;
6419 PVE
::QemuConfig-
>lock_config($vmid, sub {
6421 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6423 # there is no qmp command, so we use the human monitor command
6424 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
6425 die $res if $res ne '';
6429 # vzdump restore implementaion
6431 sub tar_archive_read_firstfile
{
6432 my $archive = shift;
6434 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
6436 # try to detect archive type first
6437 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6438 die "unable to open file '$archive'\n";
6439 my $firstfile = <$fh>;
6443 die "ERROR: archive contaions no data\n" if !$firstfile;
6449 sub tar_restore_cleanup
{
6450 my ($storecfg, $statfile) = @_;
6452 print STDERR
"starting cleanup\n";
6454 if (my $fd = IO
::File-
>new($statfile, "r")) {
6455 while (defined(my $line = <$fd>)) {
6456 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6459 if ($volid =~ m
|^/|) {
6460 unlink $volid || die 'unlink failed\n';
6462 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6464 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6466 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6468 print STDERR
"unable to parse line in statfile - $line";
6475 sub restore_file_archive
{
6476 my ($archive, $vmid, $user, $opts) = @_;
6478 return restore_vma_archive
($archive, $vmid, $user, $opts)
6481 my $info = PVE
::Storage
::archive_info
($archive);
6482 my $format = $opts->{format
} // $info->{format
};
6483 my $comp = $info->{compression
};
6485 # try to detect archive format
6486 if ($format eq 'tar') {
6487 return restore_tar_archive
($archive, $vmid, $user, $opts);
6489 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
6493 # hepler to remove disks that will not be used after restore
6494 my $restore_cleanup_oldconf = sub {
6495 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6497 my $kept_disks = {};
6499 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
6500 my ($ds, $drive) = @_;
6502 return if drive_is_cdrom
($drive, 1);
6504 my $volid = $drive->{file
};
6505 return if !$volid || $volid =~ m
|^/|;
6507 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
6508 return if !$path || !$owner || ($owner != $vmid);
6510 # Note: only delete disk we want to restore
6511 # other volumes will become unused
6512 if ($virtdev_hash->{$ds}) {
6513 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
6518 $kept_disks->{$volid} = 1;
6522 # after the restore we have no snapshots anymore
6523 for my $snapname (keys $oldconf->{snapshots
}->%*) {
6524 my $snap = $oldconf->{snapshots
}->{$snapname};
6525 if ($snap->{vmstate
}) {
6526 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
6532 for my $volid (keys $kept_disks->%*) {
6533 eval { PVE
::Storage
::volume_snapshot_delete
($storecfg, $volid, $snapname); };
6539 # Helper to parse vzdump backup device hints
6541 # $rpcenv: Environment, used to ckeck storage permissions
6542 # $user: User ID, to check storage permissions
6543 # $storecfg: Storage configuration
6544 # $fh: the file handle for reading the configuration
6545 # $devinfo: should contain device sizes for all backu-up'ed devices
6546 # $options: backup options (pool, default storage)
6548 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6549 my $parse_backup_hints = sub {
6550 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6552 my $check_storage = sub { # assert if an image can be allocate
6553 my ($storeid, $scfg) = @_;
6554 die "Content type 'images' is not available on storage '$storeid'\n"
6555 if !$scfg->{content
}->{images
};
6556 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6557 if $user ne 'root@pam';
6560 my $virtdev_hash = {};
6561 while (defined(my $line = <$fh>)) {
6562 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6563 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6564 die "archive does not contain data for drive '$virtdev'\n"
6565 if !$devinfo->{$devname};
6567 if (defined($options->{storage
})) {
6568 $storeid = $options->{storage
} || 'local';
6569 } elsif (!$storeid) {
6572 $format = 'raw' if !$format;
6573 $devinfo->{$devname}->{devname
} = $devname;
6574 $devinfo->{$devname}->{virtdev
} = $virtdev;
6575 $devinfo->{$devname}->{format
} = $format;
6576 $devinfo->{$devname}->{storeid
} = $storeid;
6578 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6579 $check_storage->($storeid, $scfg); # permission and content type check
6581 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6582 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6584 my $drive = parse_drive
($virtdev, $2);
6586 if (drive_is_cloudinit
($drive)) {
6587 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6588 $storeid = $options->{storage
} if defined ($options->{storage
});
6589 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6590 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6592 $check_storage->($storeid, $scfg); # permission and content type check
6594 $virtdev_hash->{$virtdev} = {
6596 storeid
=> $storeid,
6597 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6604 return $virtdev_hash;
6607 # Helper to allocate and activate all volumes required for a restore
6609 # $storecfg: Storage configuration
6610 # $virtdev_hash: as returned by parse_backup_hints()
6612 # Returns: { $virtdev => $volid }
6613 my $restore_allocate_devices = sub {
6614 my ($storecfg, $virtdev_hash, $vmid) = @_;
6617 foreach my $virtdev (sort keys %$virtdev_hash) {
6618 my $d = $virtdev_hash->{$virtdev};
6619 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6620 my $storeid = $d->{storeid
};
6621 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6623 # test if requested format is supported
6624 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6625 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6626 $d->{format
} = $defFormat if !$supported;
6629 if ($d->{is_cloudinit
}) {
6630 $name = "vm-$vmid-cloudinit";
6631 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6632 if ($scfg->{path
}) {
6633 $name .= ".$d->{format}";
6637 my $volid = PVE
::Storage
::vdisk_alloc
(
6638 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6640 print STDERR
"new volume ID is '$volid'\n";
6641 $d->{volid
} = $volid;
6643 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6645 $map->{$virtdev} = $volid;
6651 sub restore_update_config_line
{
6652 my ($cookie, $map, $line, $unique) = @_;
6654 return '' if $line =~ m/^\#qmdump\#/;
6655 return '' if $line =~ m/^\#vzdump\#/;
6656 return '' if $line =~ m/^lock:/;
6657 return '' if $line =~ m/^unused\d+:/;
6658 return '' if $line =~ m/^parent:/;
6662 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6663 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6664 # try to convert old 1.X settings
6665 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6666 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6667 my ($model, $macaddr) = split(/\=/, $devconfig);
6668 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6671 bridge
=> "vmbr$ind",
6672 macaddr
=> $macaddr,
6674 my $netstr = print_net
($net);
6676 $res .= "net$cookie->{netcount}: $netstr\n";
6677 $cookie->{netcount
}++;
6679 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6680 my ($id, $netstr) = ($1, $2);
6681 my $net = parse_net
($netstr);
6682 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6683 $netstr = print_net
($net);
6684 $res .= "$id: $netstr\n";
6685 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6688 my $di = parse_drive
($virtdev, $value);
6689 if (defined($di->{backup
}) && !$di->{backup
}) {
6691 } elsif ($map->{$virtdev}) {
6692 delete $di->{format
}; # format can change on restore
6693 $di->{file
} = $map->{$virtdev};
6694 $value = print_drive
($di);
6695 $res .= "$virtdev: $value\n";
6699 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6701 if ($vmgenid ne '0') {
6702 # always generate a new vmgenid if there was a valid one setup
6703 $vmgenid = generate_uuid
();
6705 $res .= "vmgenid: $vmgenid\n";
6706 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6707 my ($uuid, $uuid_str);
6708 UUID
::generate
($uuid);
6709 UUID
::unparse
($uuid, $uuid_str);
6710 my $smbios1 = parse_smbios1
($2);
6711 $smbios1->{uuid
} = $uuid_str;
6712 $res .= $1.print_smbios1
($smbios1)."\n";
6720 my $restore_deactivate_volumes = sub {
6721 my ($storecfg, $virtdev_hash) = @_;
6724 for my $dev (values $virtdev_hash->%*) {
6725 push $vollist->@*, $dev->{volid
} if $dev->{volid
};
6728 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
6729 print STDERR
$@ if $@;
6732 my $restore_destroy_volumes = sub {
6733 my ($storecfg, $virtdev_hash) = @_;
6735 for my $dev (values $virtdev_hash->%*) {
6736 my $volid = $dev->{volid
} or next;
6738 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6739 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6741 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6745 my $restore_merge_config = sub {
6746 my ($filename, $backup_conf_raw, $override_conf) = @_;
6748 my $backup_conf = parse_vm_config
($filename, $backup_conf_raw);
6749 for my $key (keys $override_conf->%*) {
6750 $backup_conf->{$key} = $override_conf->{$key};
6753 return $backup_conf;
6757 my ($cfg, $vmid) = @_;
6759 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6761 my $volid_hash = {};
6762 foreach my $storeid (keys %$info) {
6763 foreach my $item (@{$info->{$storeid}}) {
6764 next if !($item->{volid
} && $item->{size
});
6765 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6766 $volid_hash->{$item->{volid
}} = $item;
6773 sub update_disk_config
{
6774 my ($vmid, $conf, $volid_hash) = @_;
6777 my $prefix = "VM $vmid";
6779 # used and unused disks
6780 my $referenced = {};
6782 # Note: it is allowed to define multiple storages with same path (alias), so
6783 # we need to check both 'volid' and real 'path' (two different volid can point
6784 # to the same path).
6786 my $referencedpath = {};
6789 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6790 my ($opt, $drive) = @_;
6792 my $volid = $drive->{file
};
6794 my $volume = $volid_hash->{$volid};
6796 # mark volid as "in-use" for next step
6797 $referenced->{$volid} = 1;
6798 if ($volume && (my $path = $volume->{path
})) {
6799 $referencedpath->{$path} = 1;
6802 return if drive_is_cdrom
($drive);
6805 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6806 if (defined($updated)) {
6808 $conf->{$opt} = print_drive
($updated);
6809 print "$prefix ($opt): $msg\n";
6813 # remove 'unusedX' entry if volume is used
6814 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6815 my ($opt, $drive) = @_;
6817 my $volid = $drive->{file
};
6821 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6822 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6823 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6825 delete $conf->{$opt};
6828 $referenced->{$volid} = 1;
6829 $referencedpath->{$path} = 1 if $path;
6832 foreach my $volid (sort keys %$volid_hash) {
6833 next if $volid =~ m/vm-$vmid-state-/;
6834 next if $referenced->{$volid};
6835 my $path = $volid_hash->{$volid}->{path
};
6836 next if !$path; # just to be sure
6837 next if $referencedpath->{$path};
6839 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6840 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6841 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6848 my ($vmid, $nolock, $dryrun) = @_;
6850 my $cfg = PVE
::Storage
::config
();
6852 print "rescan volumes...\n";
6853 my $volid_hash = scan_volids
($cfg, $vmid);
6855 my $updatefn = sub {
6858 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6860 PVE
::QemuConfig-
>check_lock($conf);
6863 foreach my $volid (keys %$volid_hash) {
6864 my $info = $volid_hash->{$volid};
6865 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6868 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6870 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6873 if (defined($vmid)) {
6877 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6880 my $vmlist = config_list
();
6881 foreach my $vmid (keys %$vmlist) {
6885 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6891 sub restore_proxmox_backup_archive
{
6892 my ($archive, $vmid, $user, $options) = @_;
6894 my $storecfg = PVE
::Storage
::config
();
6896 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6897 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6899 my $fingerprint = $scfg->{fingerprint
};
6900 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6902 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6903 my $namespace = $scfg->{namespace
};
6905 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6906 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6907 local $ENV{PBS_PASSWORD
} = $password;
6908 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6910 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6911 PVE
::Storage
::parse_volname
($storecfg, $archive);
6913 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6915 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6917 my $tmpdir = "/var/tmp/vzdumptmp$$";
6921 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6922 # disable interrupts (always do cleanups)
6926 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6928 # Note: $oldconf is undef if VM does not exists
6929 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6930 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6931 my $new_conf_raw = '';
6933 my $rpcenv = PVE
::RPCEnvironment
::get
();
6934 my $devinfo = {}; # info about drives included in backup
6935 my $virtdev_hash = {}; # info about allocated drives
6943 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6945 my $cfgfn = "$tmpdir/qemu-server.conf";
6946 my $firewall_config_fn = "$tmpdir/fw.conf";
6947 my $index_fn = "$tmpdir/index.json";
6949 my $cmd = "restore";
6951 my $param = [$pbs_backup_name, "index.json", $index_fn];
6952 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6953 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6954 $index = decode_json
($index);
6956 foreach my $info (@{$index->{files
}}) {
6957 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6959 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6960 $devinfo->{$devname}->{size
} = $1;
6962 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6967 my $is_qemu_server_backup = scalar(
6968 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6970 if (!$is_qemu_server_backup) {
6971 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6973 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6975 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6976 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6978 if ($has_firewall_config) {
6979 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6980 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6982 my $pve_firewall_dir = '/etc/pve/firewall';
6983 mkdir $pve_firewall_dir; # make sure the dir exists
6984 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6987 my $fh = IO
::File-
>new($cfgfn, "r") ||
6988 die "unable to read qemu-server.conf - $!\n";
6990 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6992 # fixme: rate limit?
6994 # create empty/temp config
6995 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6997 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7000 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7002 foreach my $virtdev (sort keys %$virtdev_hash) {
7003 my $d = $virtdev_hash->{$virtdev};
7004 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7006 # this fails if storage is unavailable
7007 my $volid = $d->{volid
};
7008 my $path = PVE
::Storage
::path
($storecfg, $volid);
7010 # for live-restore we only want to preload the efidisk and TPM state
7011 next if $options->{live
} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7014 if (defined(my $ns = $scfg->{namespace
})) {
7015 @ns_arg = ('--ns', $ns);
7018 my $pbs_restore_cmd = [
7019 '/usr/bin/pbs-restore',
7020 '--repository', $repo,
7023 "$d->{devname}.img.fidx",
7028 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
7029 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
7031 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
7032 push @$pbs_restore_cmd, '--skip-zero';
7035 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
7036 print "restore proxmox backup image: $dbg_cmdstring\n";
7037 run_command
($pbs_restore_cmd);
7040 $fh->seek(0, 0) || die "seek failed - $!\n";
7042 my $cookie = { netcount
=> 0 };
7043 while (defined(my $line = <$fh>)) {
7044 $new_conf_raw .= restore_update_config_line
(
7056 if ($err || !$options->{live
}) {
7057 $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7063 $restore_destroy_volumes->($storecfg, $virtdev_hash);
7067 if ($options->{live
}) {
7068 # keep lock during live-restore
7069 $new_conf_raw .= "\nlock: create";
7072 my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf
});
7073 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7075 eval { rescan
($vmid, 1); };
7078 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
7080 if ($options->{live
}) {
7086 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
7088 my $conf = PVE
::QemuConfig-
>load_config($vmid);
7089 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
7091 # these special drives are already restored before start
7092 delete $devinfo->{'drive-efidisk0'};
7093 delete $devinfo->{'drive-tpmstate0-backup'};
7097 keyfile
=> $keyfile,
7098 snapshot
=> $pbs_backup_name,
7099 namespace
=> $namespace,
7101 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7103 PVE
::QemuConfig-
>remove_lock($vmid, "create");
7107 sub pbs_live_restore
{
7108 my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7110 print "starting VM for live-restore\n";
7111 print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7113 my $pbs_backing = {};
7114 for my $ds (keys %$restored_disks) {
7115 $ds =~ m/^drive-(.*)$/;
7117 $pbs_backing->{$confname} = {
7118 repository
=> $opts->{repo
},
7119 snapshot
=> $opts->{snapshot
},
7120 archive
=> "$ds.img.fidx",
7122 $pbs_backing->{$confname}->{keyfile
} = $opts->{keyfile
} if -e
$opts->{keyfile
};
7123 $pbs_backing->{$confname}->{namespace
} = $opts->{namespace
} if defined($opts->{namespace
});
7125 my $drive = parse_drive
($confname, $conf->{$confname});
7126 print "restoring '$ds' to '$drive->{file}'\n";
7129 my $drives_streamed = 0;
7131 # make sure HA doesn't interrupt our restore by stopping the VM
7132 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
7133 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7136 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7137 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7138 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'pbs-backing' => $pbs_backing}, {});
7140 my $qmeventd_fd = register_qmeventd_handle
($vmid);
7142 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7143 # this will effectively collapse the backing image chain consisting of
7144 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7145 # removes itself once all backing images vanish with 'auto-remove=on')
7147 for my $ds (sort keys %$restored_disks) {
7148 my $job_id = "restore-$ds";
7149 mon_cmd
($vmid, 'block-stream',
7150 'job-id' => $job_id,
7153 $jobs->{$job_id} = {};
7156 mon_cmd
($vmid, 'cont');
7157 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
7159 print "restore-drive jobs finished successfully, removing all tracking block devices"
7160 ." to disconnect from Proxmox Backup Server\n";
7162 for my $ds (sort keys %$restored_disks) {
7163 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7166 close($qmeventd_fd);
7172 warn "An error occurred during live-restore: $err\n";
7173 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
7174 die "live-restore failed\n";
7178 sub restore_vma_archive
{
7179 my ($archive, $vmid, $user, $opts, $comp) = @_;
7181 my $readfrom = $archive;
7183 my $cfg = PVE
::Storage
::config
();
7185 my $bwlimit = $opts->{bwlimit
};
7187 my $dbg_cmdstring = '';
7188 my $add_pipe = sub {
7190 push @$commands, $cmd;
7191 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7192 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
7197 if ($archive eq '-') {
7200 # If we use a backup from a PVE defined storage we also consider that
7201 # storage's rate limit:
7202 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
7203 if (defined($volid)) {
7204 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
7205 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
7207 print STDERR
"applying read rate limit: $readlimit\n";
7208 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7209 $add_pipe->($cstream);
7215 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
7216 my $cmd = $info->{decompressor
};
7217 push @$cmd, $readfrom;
7221 my $tmpdir = "/var/tmp/vzdumptmp$$";
7224 # disable interrupts (always do cleanups)
7228 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
7230 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7231 POSIX
::mkfifo
($mapfifo, 0600);
7233 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7235 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7240 my $devinfo = {}; # info about drives included in backup
7241 my $virtdev_hash = {}; # info about allocated drives
7243 my $rpcenv = PVE
::RPCEnvironment
::get
();
7245 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7247 # Note: $oldconf is undef if VM does not exist
7248 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7249 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7250 my $new_conf_raw = '';
7254 my $print_devmap = sub {
7255 my $cfgfn = "$tmpdir/qemu-server.conf";
7257 # we can read the config - that is already extracted
7258 my $fh = IO
::File-
>new($cfgfn, "r") ||
7259 die "unable to read qemu-server.conf - $!\n";
7261 my $fwcfgfn = "$tmpdir/qemu-server.fw";
7263 my $pve_firewall_dir = '/etc/pve/firewall';
7264 mkdir $pve_firewall_dir; # make sure the dir exists
7265 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7268 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7270 foreach my $info (values %{$virtdev_hash}) {
7271 my $storeid = $info->{storeid
};
7272 next if defined($storage_limits{$storeid});
7274 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
7275 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7276 $storage_limits{$storeid} = $limit * 1024;
7279 foreach my $devname (keys %$devinfo) {
7280 die "found no device mapping information for device '$devname'\n"
7281 if !$devinfo->{$devname}->{virtdev
};
7284 # create empty/temp config
7286 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
7287 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7291 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7293 # print restore information to $fifofh
7294 foreach my $virtdev (sort keys %$virtdev_hash) {
7295 my $d = $virtdev_hash->{$virtdev};
7296 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7298 my $storeid = $d->{storeid
};
7299 my $volid = $d->{volid
};
7302 if (my $limit = $storage_limits{$storeid}) {
7303 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7306 my $write_zeros = 1;
7307 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
7311 my $path = PVE
::Storage
::path
($cfg, $volid);
7313 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7315 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7318 $fh->seek(0, 0) || die "seek failed - $!\n";
7320 my $cookie = { netcount
=> 0 };
7321 while (defined(my $line = <$fh>)) {
7322 $new_conf_raw .= restore_update_config_line
(
7339 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7340 local $SIG{ALRM
} = sub { die "got timeout\n"; };
7342 $oldtimeout = alarm($timeout);
7349 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7350 my ($dev_id, $size, $devname) = ($1, $2, $3);
7351 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
7352 } elsif ($line =~ m/^CTIME: /) {
7353 # we correctly received the vma config, so we can disable
7354 # the timeout now for disk allocation (set to 10 minutes, so
7355 # that we always timeout if something goes wrong)
7358 print $fifofh "done\n";
7359 my $tmp = $oldtimeout || 0;
7360 $oldtimeout = undef;
7367 print "restore vma archive: $dbg_cmdstring\n";
7368 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
7372 alarm($oldtimeout) if $oldtimeout;
7374 $restore_deactivate_volumes->($cfg, $virtdev_hash);
7376 close($fifofh) if $fifofh;
7381 $restore_destroy_volumes->($cfg, $virtdev_hash);
7385 my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf
});
7386 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7388 eval { rescan
($vmid, 1); };
7391 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
7394 sub restore_tar_archive
{
7395 my ($archive, $vmid, $user, $opts) = @_;
7397 if (scalar(keys $opts->{override_conf
}->%*) > 0) {
7398 my $keystring = join(' ', keys $opts->{override_conf
}->%*);
7399 die "cannot pass along options ($keystring) when restoring from tar archive\n";
7402 if ($archive ne '-') {
7403 my $firstfile = tar_archive_read_firstfile
($archive);
7404 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7405 if $firstfile ne 'qemu-server.conf';
7408 my $storecfg = PVE
::Storage
::config
();
7410 # avoid zombie disks when restoring over an existing VM -> cleanup first
7411 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7412 # skiplock=1 because qmrestore has set the 'create' lock itself already
7413 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
7414 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
7416 my $tocmd = "/usr/lib/qemu-server/qmextract";
7418 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
7419 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
7420 $tocmd .= ' --prealloc' if $opts->{prealloc
};
7421 $tocmd .= ' --info' if $opts->{info
};
7423 # tar option "xf" does not autodetect compression when read from STDIN,
7424 # so we pipe to zcat
7425 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
7426 PVE
::Tools
::shellquote
("--to-command=$tocmd");
7428 my $tmpdir = "/var/tmp/vzdumptmp$$";
7431 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
7432 local $ENV{VZDUMP_VMID
} = $vmid;
7433 local $ENV{VZDUMP_USER
} = $user;
7435 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7436 my $new_conf_raw = '';
7438 # disable interrupts (always do cleanups)
7442 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7450 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7452 if ($archive eq '-') {
7453 print "extracting archive from STDIN\n";
7454 run_command
($cmd, input
=> "<&STDIN");
7456 print "extracting archive '$archive'\n";
7460 return if $opts->{info
};
7464 my $statfile = "$tmpdir/qmrestore.stat";
7465 if (my $fd = IO
::File-
>new($statfile, "r")) {
7466 while (defined (my $line = <$fd>)) {
7467 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7468 $map->{$1} = $2 if $1;
7470 print STDERR
"unable to parse line in statfile - $line\n";
7476 my $confsrc = "$tmpdir/qemu-server.conf";
7478 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7480 my $cookie = { netcount
=> 0 };
7481 while (defined (my $line = <$srcfd>)) {
7482 $new_conf_raw .= restore_update_config_line
(
7493 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
7499 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7501 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7503 eval { rescan
($vmid, 1); };
7507 sub foreach_storage_used_by_vm
{
7508 my ($conf, $func) = @_;
7512 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7513 my ($ds, $drive) = @_;
7514 return if drive_is_cdrom
($drive);
7516 my $volid = $drive->{file
};
7518 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
7519 $sidhash->{$sid} = $sid if $sid;
7522 foreach my $sid (sort keys %$sidhash) {
7527 my $qemu_snap_storage = {
7530 sub do_snapshots_with_qemu
{
7531 my ($storecfg, $volid, $deviceid) = @_;
7533 return if $deviceid =~ m/tpmstate0/;
7535 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
7536 my $scfg = $storecfg->{ids
}->{$storage_name};
7537 die "could not find storage '$storage_name'\n" if !defined($scfg);
7539 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
7543 if ($volid =~ m/\.(qcow2|qed)$/){
7550 sub qga_check_running
{
7551 my ($vmid, $nowarn) = @_;
7553 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
7555 warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7561 sub template_create
{
7562 my ($vmid, $conf, $disk) = @_;
7564 my $storecfg = PVE
::Storage
::config
();
7566 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7567 my ($ds, $drive) = @_;
7569 return if drive_is_cdrom
($drive);
7570 return if $disk && $ds ne $disk;
7572 my $volid = $drive->{file
};
7573 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
7575 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
7576 $drive->{file
} = $voliddst;
7577 $conf->{$ds} = print_drive
($drive);
7578 PVE
::QemuConfig-
>write_config($vmid, $conf);
7582 sub convert_iscsi_path
{
7585 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
7590 my $initiator_name = get_initiator_name
();
7592 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7593 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7596 die "cannot convert iscsi path '$path', unkown format\n";
7599 sub qemu_img_convert
{
7600 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7602 my $storecfg = PVE
::Storage
::config
();
7603 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
7604 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
7606 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7610 my $src_is_iscsi = 0;
7614 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
7615 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7616 $src_format = qemu_img_format
($src_scfg, $src_volname);
7617 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
7618 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
7619 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
7620 } elsif (-f
$src_volid || -b
$src_volid) {
7621 $src_path = $src_volid;
7622 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7627 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7629 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7630 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7631 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7632 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7635 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7636 push @$cmd, '-l', "snapshot.name=$snapname"
7637 if $snapname && $src_format && $src_format eq "qcow2";
7638 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7639 push @$cmd, '-T', $cachemode if defined($cachemode);
7641 if ($src_is_iscsi) {
7642 push @$cmd, '--image-opts';
7643 $src_path = convert_iscsi_path
($src_path);
7644 } elsif ($src_format) {
7645 push @$cmd, '-f', $src_format;
7648 if ($dst_is_iscsi) {
7649 push @$cmd, '--target-image-opts';
7650 $dst_path = convert_iscsi_path
($dst_path);
7652 push @$cmd, '-O', $dst_format;
7655 push @$cmd, $src_path;
7657 if (!$dst_is_iscsi && $is_zero_initialized) {
7658 push @$cmd, "zeroinit:$dst_path";
7660 push @$cmd, $dst_path;
7665 if($line =~ m/\((\S+)\/100\
%\)/){
7667 my $transferred = int($size * $percent / 100);
7668 my $total_h = render_bytes
($size, 1);
7669 my $transferred_h = render_bytes
($transferred, 1);
7671 print "transferred $transferred_h of $total_h ($percent%)\n";
7676 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7678 die "copy failed: $err" if $err;
7681 sub qemu_img_format
{
7682 my ($scfg, $volname) = @_;
7684 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7691 sub qemu_drive_mirror
{
7692 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7694 $jobs = {} if !$jobs;
7698 $jobs->{"drive-$drive"} = {};
7700 if ($dst_volid =~ /^nbd:/) {
7701 $qemu_target = $dst_volid;
7704 my $storecfg = PVE
::Storage
::config
();
7705 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7707 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7709 $format = qemu_img_format
($dst_scfg, $dst_volname);
7711 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7713 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7716 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7717 $opts->{format
} = $format if $format;
7719 if (defined($src_bitmap)) {
7720 $opts->{sync
} = 'incremental';
7721 $opts->{bitmap
} = $src_bitmap;
7722 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7725 if (defined($bwlimit)) {
7726 $opts->{speed
} = $bwlimit * 1024;
7727 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7729 print "drive mirror is starting for drive-$drive\n";
7732 # if a job already runs for this device we get an error, catch it for cleanup
7733 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7735 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7737 die "mirroring error: $err\n";
7740 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7743 # $completion can be either
7744 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7745 # 'cancel': wait until all jobs are ready, block-job-cancel them
7746 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7747 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7748 sub qemu_drive_mirror_monitor
{
7749 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7751 $completion //= 'complete';
7755 my $err_complete = 0;
7757 my $starttime = time ();
7759 die "block job ('$op') timed out\n" if $err_complete > 300;
7761 my $stats = mon_cmd
($vmid, "query-block-jobs");
7764 my $running_jobs = {};
7765 for my $stat (@$stats) {
7766 next if $stat->{type
} ne $op;
7767 $running_jobs->{$stat->{device
}} = $stat;
7770 my $readycounter = 0;
7772 for my $job_id (sort keys %$jobs) {
7773 my $job = $running_jobs->{$job_id};
7775 my $vanished = !defined($job);
7776 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7777 if($complete || ($vanished && $completion eq 'auto')) {
7778 print "$job_id: $op-job finished\n";
7779 delete $jobs->{$job_id};
7783 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7785 my $busy = $job->{busy
};
7786 my $ready = $job->{ready
};
7787 if (my $total = $job->{len
}) {
7788 my $transferred = $job->{offset
} || 0;
7789 my $remaining = $total - $transferred;
7790 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7792 my $duration = $ctime - $starttime;
7793 my $total_h = render_bytes
($total, 1);
7794 my $transferred_h = render_bytes
($transferred, 1);
7796 my $status = sprintf(
7797 "transferred $transferred_h of $total_h ($percent%%) in %s",
7798 render_duration
($duration),
7803 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7805 $status .= ", ready";
7808 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
7809 $jobs->{$job_id}->{ready
} = $ready;
7812 $readycounter++ if $job->{ready
};
7815 last if scalar(keys %$jobs) == 0;
7817 if ($readycounter == scalar(keys %$jobs)) {
7818 print "all '$op' jobs are ready\n";
7820 # do the complete later (or has already been done)
7821 last if $completion eq 'skip' || $completion eq 'auto';
7823 if ($vmiddst && $vmiddst != $vmid) {
7824 my $agent_running = $qga && qga_check_running
($vmid);
7825 if ($agent_running) {
7826 print "freeze filesystem\n";
7827 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
7830 print "suspend vm\n";
7831 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
7835 # if we clone a disk for a new target vm, we don't switch the disk
7836 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
7838 if ($agent_running) {
7839 print "unfreeze filesystem\n";
7840 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
7843 print "resume vm\n";
7844 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
7851 for my $job_id (sort keys %$jobs) {
7852 # try to switch the disk if source and destination are on the same guest
7853 print "$job_id: Completing block job_id...\n";
7856 if ($completion eq 'complete') {
7857 $op = 'block-job-complete';
7858 } elsif ($completion eq 'cancel') {
7859 $op = 'block-job-cancel';
7861 die "invalid completion value: $completion\n";
7863 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
7864 if ($@ =~ m/cannot be completed/) {
7865 print "$job_id: block job cannot be completed, trying again.\n";
7868 print "$job_id: Completed successfully.\n";
7869 $jobs->{$job_id}->{complete
} = 1;
7880 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7881 die "block job ($op) error: $err";
7885 sub qemu_blockjobs_cancel
{
7886 my ($vmid, $jobs) = @_;
7888 foreach my $job (keys %$jobs) {
7889 print "$job: Cancelling block job\n";
7890 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
7891 $jobs->{$job}->{cancel
} = 1;
7895 my $stats = mon_cmd
($vmid, "query-block-jobs");
7897 my $running_jobs = {};
7898 foreach my $stat (@$stats) {
7899 $running_jobs->{$stat->{device
}} = $stat;
7902 foreach my $job (keys %$jobs) {
7904 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
7905 print "$job: Done.\n";
7906 delete $jobs->{$job};
7910 last if scalar(keys %$jobs) == 0;
7917 my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7919 my ($vmid, $running) = $source->@{qw(vmid running)};
7920 my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7922 my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7923 my ($storage, $format) = $dest->@{qw(storage format)};
7925 my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7927 if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7928 die "cloning from/to EFI disk requires EFI disk\n"
7929 if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7930 die "cloning from/to TPM state requires TPM state\n"
7931 if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7933 # This would lead to two device nodes in QEMU pointing to the same backing image!
7934 die "cannot change drive name when cloning disk from/to the same VM\n"
7935 if $use_drive_mirror && $vmid == $newvmid;
7938 die "cannot move TPM state while VM is running\n"
7939 if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7943 print "create " . ($full ?
'full' : 'linked') . " clone of drive ";
7944 print "$src_drivename " if $src_drivename;
7945 print "($drive->{file})\n";
7948 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
7949 push @$newvollist, $newvolid;
7952 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
7953 $storeid = $storage if $storage;
7955 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
7959 if (drive_is_cloudinit
($drive)) {
7960 $name = "vm-$newvmid-cloudinit";
7961 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7962 if ($scfg->{path
}) {
7963 $name .= ".$dst_format";
7966 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
7967 } elsif ($dst_drivename eq 'efidisk0') {
7968 $size = $efisize or die "internal error - need to specify EFI disk size\n";
7969 } elsif ($dst_drivename eq 'tpmstate0') {
7970 $dst_format = 'raw';
7971 $size = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
7973 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
7975 $newvolid = PVE
::Storage
::vdisk_alloc
(
7976 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7978 push @$newvollist, $newvolid;
7980 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
7982 if (drive_is_cloudinit
($drive)) {
7983 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7984 # if this is the case, we have to complete any block-jobs still there from
7985 # previous drive-mirrors
7986 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7987 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
7992 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
7993 if ($use_drive_mirror) {
7994 qemu_drive_mirror
($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7995 $completion, $qga, $bwlimit);
7997 # TODO: handle bwlimits
7998 if ($dst_drivename eq 'efidisk0') {
7999 # the relevant data on the efidisk may be smaller than the source
8000 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8001 # that is given by the OVMF_VARS.fd
8002 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
8003 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
8005 my $src_format = (PVE
::Storage
::parse_volname
($storecfg, $drive->{file
}))[6];
8007 # better for Ceph if block size is not too small, see bug #3324
8010 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8012 if ($src_format eq 'qcow2' && $snapname) {
8013 die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8014 if !min_version
(kvm_user_version
(), 6, 2);
8015 push $cmd->@*, '-l', $snapname;
8017 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8020 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
8026 my ($size) = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
8028 my $disk = dclone
($drive);
8029 delete $disk->{format
};
8030 $disk->{file
} = $newvolid;
8031 $disk->{size
} = $size if defined($size);
8036 sub get_running_qemu_version
{
8038 my $res = mon_cmd
($vmid, "query-version");
8039 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8042 sub qemu_use_old_bios_files
{
8043 my ($machine_type) = @_;
8045 return if !$machine_type;
8047 my $use_old_bios_files = undef;
8049 if ($machine_type =~ m/^(\S+)\.pxe$/) {
8051 $use_old_bios_files = 1;
8053 my $version = extract_version
($machine_type, kvm_user_version
());
8054 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8055 # load new efi bios files on migration. So this hack is required to allow
8056 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8057 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8058 $use_old_bios_files = !min_version
($version, 2, 4);
8061 return ($use_old_bios_files, $machine_type);
8064 sub get_efivars_size
{
8065 my ($conf, $efidisk) = @_;
8067 my $arch = get_vm_arch
($conf);
8068 $efidisk //= $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
8069 my $smm = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
8070 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8071 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
8072 return -s
$ovmf_vars;
8075 sub update_efidisk_size
{
8078 return if !defined($conf->{efidisk0
});
8080 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
8081 $disk->{size
} = get_efivars_size
($conf);
8082 $conf->{efidisk0
} = print_drive
($disk);
8087 sub update_tpmstate_size
{
8090 my $disk = PVE
::QemuServer
::parse_drive
('tpmstate0', $conf->{tpmstate0
});
8091 $disk->{size
} = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8092 $conf->{tpmstate0
} = print_drive
($disk);
8095 sub create_efidisk
($$$$$$$) {
8096 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8098 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8099 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
8101 my $vars_size_b = -s
$ovmf_vars;
8102 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
8103 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8104 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
8106 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
8107 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
8109 return ($volid, $size/1024);
8112 sub vm_iothreads_list
{
8115 my $res = mon_cmd
($vmid, 'query-iothreads');
8118 foreach my $iothread (@$res) {
8119 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
8126 my ($conf, $drive) = @_;
8130 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
8132 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
8138 my $controller = int($drive->{index} / $maxdev);
8139 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
8143 return ($maxdev, $controller, $controller_prefix);
8146 sub resolve_dst_disk_format
{
8147 my ($storecfg, $storeid, $src_volname, $format) = @_;
8148 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
8151 # if no target format is specified, use the source disk format as hint
8153 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8154 $format = qemu_img_format
($scfg, $src_volname);
8160 # test if requested format is supported - else use default
8161 my $supported = grep { $_ eq $format } @$validFormats;
8162 $format = $defFormat if !$supported;
8166 # NOTE: if this logic changes, please update docs & possibly gui logic
8167 sub find_vmstate_storage
{
8168 my ($conf, $storecfg) = @_;
8170 # first, return storage from conf if set
8171 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
8173 my ($target, $shared, $local);
8175 foreach_storage_used_by_vm
($conf, sub {
8177 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
8178 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
8179 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
8182 # second, use shared storage where VM has at least one disk
8183 # third, use local storage where VM has at least one disk
8184 # fall back to local storage
8185 $target = $shared // $local // 'local';
8191 my ($uuid, $uuid_str);
8192 UUID
::generate
($uuid);
8193 UUID
::unparse
($uuid, $uuid_str);
8197 sub generate_smbios1_uuid
{
8198 return "uuid=".generate_uuid
();
8204 mon_cmd
($vmid, 'nbd-server-stop');
8207 sub create_reboot_request
{
8209 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8210 or die "failed to create reboot trigger file: $!\n";
8214 sub clear_reboot_request
{
8216 my $path = "/run/qemu-server/$vmid.reboot";
8219 $res = unlink($path);
8220 die "could not remove reboot request for $vmid: $!"
8221 if !$res && $! != POSIX
::ENOENT
;
8226 sub bootorder_from_legacy
{
8227 my ($conf, $bootcfg) = @_;
8229 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
8230 my $bootindex_hash = {};
8232 foreach my $o (split(//, $boot)) {
8233 $bootindex_hash->{$o} = $i*100;
8239 PVE
::QemuConfig-
>foreach_volume($conf, sub {
8240 my ($ds, $drive) = @_;
8242 if (drive_is_cdrom
($drive, 1)) {
8243 if ($bootindex_hash->{d
}) {
8244 $bootorder->{$ds} = $bootindex_hash->{d
};
8245 $bootindex_hash->{d
} += 1;
8247 } elsif ($bootindex_hash->{c
}) {
8248 $bootorder->{$ds} = $bootindex_hash->{c
}
8249 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
8250 $bootindex_hash->{c
} += 1;
8254 if ($bootindex_hash->{n
}) {
8255 for (my $i = 0; $i < $MAX_NETS; $i++) {
8256 my $netname = "net$i";
8257 next if !$conf->{$netname};
8258 $bootorder->{$netname} = $bootindex_hash->{n
};
8259 $bootindex_hash->{n
} += 1;
8266 # Generate default device list for 'boot: order=' property. Matches legacy
8267 # default boot order, but with explicit device names. This is important, since
8268 # the fallback for when neither 'order' nor the old format is specified relies
8269 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8270 sub get_default_bootdevices
{
8276 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
8277 push @ret, $first if $first;
8280 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
8281 push @ret, $first if $first;
8284 for (my $i = 0; $i < $MAX_NETS; $i++) {
8285 my $netname = "net$i";
8286 next if !$conf->{$netname};
8287 push @ret, $netname;
8294 sub device_bootorder
{
8297 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
8299 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
8302 if (!defined($boot) || $boot->{legacy
}) {
8303 $bootorder = bootorder_from_legacy
($conf, $boot);
8304 } elsif ($boot->{order
}) {
8305 my $i = 100; # start at 100 to allow user to insert devices before us with -args
8306 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
8307 $bootorder->{$dev} = $i++;
8314 sub register_qmeventd_handle
{
8318 my $peer = "/var/run/qmeventd.sock";
8323 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
8325 if ($! != EINTR
&& $! != EAGAIN
) {
8326 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8329 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8330 . "after $count retries\n";
8335 # send handshake to mark VM as backing up
8336 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
8338 # return handle to be closed later when inhibit is no longer required
8342 # bash completion helper
8344 sub complete_backup_archives
{
8345 my ($cmdname, $pname, $cvalue) = @_;
8347 my $cfg = PVE
::Storage
::config
();
8351 if ($cvalue =~ m/^([^:]+):/) {
8355 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
8358 foreach my $id (keys %$data) {
8359 foreach my $item (@{$data->{$id}}) {
8360 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8361 push @$res, $item->{volid
} if defined($item->{volid
});
8368 my $complete_vmid_full = sub {
8371 my $idlist = vmstatus
();
8375 foreach my $id (keys %$idlist) {
8376 my $d = $idlist->{$id};
8377 if (defined($running)) {
8378 next if $d->{template
};
8379 next if $running && $d->{status
} ne 'running';
8380 next if !$running && $d->{status
} eq 'running';
8389 return &$complete_vmid_full();
8392 sub complete_vmid_stopped
{
8393 return &$complete_vmid_full(0);
8396 sub complete_vmid_running
{
8397 return &$complete_vmid_full(1);
8400 sub complete_storage
{
8402 my $cfg = PVE
::Storage
::config
();
8403 my $ids = $cfg->{ids
};
8406 foreach my $sid (keys %$ids) {
8407 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
8408 next if !$ids->{$sid}->{content
}->{images
};
8415 sub complete_migration_storage
{
8416 my ($cmd, $param, $current_value, $all_args) = @_;
8418 my $targetnode = @$all_args[1];
8420 my $cfg = PVE
::Storage
::config
();
8421 my $ids = $cfg->{ids
};
8424 foreach my $sid (keys %$ids) {
8425 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
8426 next if !$ids->{$sid}->{content
}->{images
};
8435 my $qmpstatus = eval {
8436 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
8437 mon_cmd
($vmid, "query-status");
8440 return $qmpstatus && $qmpstatus->{status
} eq "paused";
8443 sub check_volume_storage_type
{
8444 my ($storecfg, $vol) = @_;
8446 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
8447 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8448 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
8450 die "storage '$storeid' does not support content-type '$vtype'\n"
8451 if !$scfg->{content
}->{$vtype};
8456 sub add_nets_bridge_fdb
{
8457 my ($conf, $vmid) = @_;
8459 for my $opt (keys %$conf) {
8460 next if $opt !~ m/^net(\d+)$/;
8461 my $iface = "tap${vmid}i$1";
8462 # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8463 my $net = parse_net
($conf->{$opt}, 1) or next;
8465 my $mac = $net->{macaddr
};
8467 log_warn
("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8468 if !file_read_firstline
("/sys/class/net/$iface/brport/learning");
8472 my $bridge = $net->{bridge
};
8474 PVE
::Network
::SDN
::Zones
::add_bridge_fdb
($iface, $mac, $bridge, $net->{firewall
});
8475 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8476 PVE
::Network
::add_bridge_fdb
($iface, $mac, $net->{firewall
});
8481 sub del_nets_bridge_fdb
{
8482 my ($conf, $vmid) = @_;
8484 for my $opt (keys %$conf) {
8485 next if $opt !~ m/^net(\d+)$/;
8486 my $iface = "tap${vmid}i$1";
8488 my $net = parse_net
($conf->{$opt}) or next;
8489 my $mac = $net->{macaddr
} or next;
8491 my $bridge = $net->{bridge
};
8493 PVE
::Network
::SDN
::Zones
::del_bridge_fdb
($iface, $mac, $bridge, $net->{firewall
});
8494 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8495 PVE
::Network
::del_bridge_fdb
($iface, $mac, $net->{firewall
});