1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
22 use List
::Util
qw(first);
25 use Storable
qw(dclone);
26 use Time
::HiRes
qw(gettimeofday usleep);
30 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
33 use PVE
::DataCenterConfig
;
34 use PVE
::Exception
qw(raise raise_param_exc);
35 use PVE
::Format
qw(render_duration render_bytes);
36 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
38 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
41 use PVE
::RESTEnvironment
qw(log_warn);
42 use PVE
::RPCEnvironment
;
46 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
50 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout windows_version);
51 use PVE
::QemuServer
::Cloudinit
;
52 use PVE
::QemuServer
::CGroup
;
53 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
54 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
55 use PVE
::QemuServer
::Machine
;
56 use PVE
::QemuServer
::Memory
;
57 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
58 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
59 use PVE
::QemuServer
::USB
qw(parse_usb_device);
63 require PVE
::Network
::SDN
::Zones
;
67 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
71 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
72 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
75 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
76 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
79 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
80 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
83 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
84 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
87 "$EDK2_FW_BASE/OVMF_CODE.fd",
88 "$EDK2_FW_BASE/OVMF_VARS.fd",
93 "$EDK2_FW_BASE/AAVMF_CODE.fd",
94 "$EDK2_FW_BASE/AAVMF_VARS.fd",
99 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
101 # Note about locking: we use flock on the config file protect against concurent actions.
102 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
103 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
104 # But you can ignore this kind of lock with the --skiplock flag.
112 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
113 description
=> "Some command save/restore state from this location.",
119 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
120 description
=> "Specifies the QEMU machine type.",
122 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
130 $nodename_cache //= PVE
::INotify
::nodename
();
131 return $nodename_cache;
138 enum
=> [qw(i6300esb ib700)],
139 description
=> "Watchdog type to emulate.",
140 default => 'i6300esb',
145 enum
=> [qw(reset shutdown poweroff pause debug none)],
146 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
150 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
154 description
=> "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
159 fstrim_cloned_disks
=> {
160 description
=> "Run fstrim after moving a disk or migrating the VM.",
166 description
=> "Select the agent type",
170 enum
=> [qw(virtio isa)],
176 description
=> "Select the VGA type.",
181 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
184 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
196 description
=> "The size of the file in MB.",
200 pattern
=> '[a-zA-Z0-9\-]+',
202 format_description
=> 'string',
203 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
210 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
211 description
=> "Configure an audio device."
215 enum
=> ['spice', 'none'],
218 description
=> "Driver backend for the audio device."
222 my $spice_enhancements_fmt = {
227 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
231 enum
=> ['off', 'all', 'filter'],
234 description
=> "Enable video streaming. Uses compression for detected video streams."
241 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
243 description
=> "The file on the host to gather entropy from. In most cases '/dev/urandom'"
244 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
245 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
246 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
247 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
248 ." a hardware RNG from the host.",
252 description
=> "Maximum bytes of entropy allowed to get injected into the guest every"
253 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
254 ." `0` to disable limiting (potentially dangerous!).",
257 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
258 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
259 # reading from /dev/urandom
264 description
=> "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
265 ." the guest to retrieve another 'max_bytes' of entropy.",
271 my $meta_info_fmt = {
274 description
=> "The guest creation timestamp as UNIX epoch time",
280 description
=> "The QEMU (machine) version from the time this VM was created.",
281 pattern
=> '\d+(\.\d+)+',
290 description
=> "Specifies whether a VM will be started during system bootup.",
296 description
=> "Automatic restart after crash (currently ignored).",
301 type
=> 'string', format
=> 'pve-hotplug-features',
302 description
=> "Selectively enable hotplug features. This is a comma separated list of"
303 ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
304 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
305 ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
307 default => 'network,disk,usb',
312 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
318 description
=> "Lock/unlock the VM.",
319 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
324 description
=> "Limit of CPU usage.",
325 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
326 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
334 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
335 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
336 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
337 ." weights of all the other running VMs.",
340 default => 'cgroup v1: 1024, cgroup v2: 100',
345 description
=> "Amount of RAM for the VM in MiB. This is the maximum available memory when"
346 ." you use the balloon device.",
353 description
=> "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
359 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the"
360 ." more memory this VM gets. Number is relative to weights of all other running VMs."
361 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
369 description
=> "Keyboard layout for VNC server. This option is generally not required and"
370 ." is often better handled from within the guest OS.",
371 enum
=> PVE
::Tools
::kvmkeymaplist
(),
376 type
=> 'string', format
=> 'dns-name',
377 description
=> "Set a name for the VM. Only used on the configuration web interface.",
382 description
=> "SCSI controller model",
383 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
389 description
=> "Description for the VM. Shown in the web-interface VM's summary."
390 ." This is saved as comment inside the configuration file.",
391 maxLength
=> 1024 * 8,
396 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
397 description
=> "Specify guest operating system.",
398 verbose_description
=> <<EODESC,
399 Specify guest operating system. This is used to enable special
400 optimization/features for specific operating systems:
403 other;; unspecified OS
404 wxp;; Microsoft Windows XP
405 w2k;; Microsoft Windows 2000
406 w2k3;; Microsoft Windows 2003
407 w2k8;; Microsoft Windows 2008
408 wvista;; Microsoft Windows Vista
409 win7;; Microsoft Windows 7
410 win8;; Microsoft Windows 8/2012/2012r2
411 win10;; Microsoft Windows 10/2016/2019
412 win11;; Microsoft Windows 11/2022
413 l24;; Linux 2.4 Kernel
414 l26;; Linux 2.6 - 6.X Kernel
415 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
420 type
=> 'string', format
=> 'pve-qm-boot',
421 description
=> "Specify guest boot order. Use the 'order=' sub-property as usage with no"
422 ." key or 'legacy=' is deprecated.",
426 type
=> 'string', format
=> 'pve-qm-bootdisk',
427 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
428 pattern
=> '(ide|sata|scsi|virtio)\d+',
433 description
=> "The number of CPUs. Please use option -sockets instead.",
440 description
=> "The number of CPU sockets.",
447 description
=> "The number of cores per socket.",
454 description
=> "Enable/disable NUMA.",
460 description
=> "Enable/disable hugepages memory.",
461 enum
=> [qw(any 2 1024)],
467 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
468 ." after VM shutdown and can be used for subsequent starts.",
473 description
=> "Number of hotplugged vcpus.",
480 description
=> "Enable/disable ACPI.",
485 description
=> "Enable/disable communication with the QEMU Guest Agent and its properties.",
487 format
=> $agent_fmt,
492 description
=> "Enable/disable KVM hardware virtualization.",
498 description
=> "Enable/disable time drift fix.",
504 description
=> "Set the real time clock (RTC) to local time. This is enabled by default if"
505 ." the `ostype` indicates a Microsoft Windows OS.",
510 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
514 type
=> 'string', format
=> $vga_fmt,
515 description
=> "Configure the VGA hardware.",
516 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
517 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
518 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
519 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
520 ." display server. For win* OS you can select how many independent displays you want,"
521 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
522 ." using a serial device as terminal.",
526 type
=> 'string', format
=> 'pve-qm-watchdog',
527 description
=> "Create a virtual hardware watchdog device.",
528 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
529 ." action), the watchdog must be periodically polled by an agent inside the guest or"
530 ." else the watchdog will reset the guest (or execute the respective action specified)",
535 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
536 description
=> "Set the initial date of the real time clock. Valid format for date are:"
537 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
538 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
541 startup
=> get_standard_option
('pve-startup-order'),
545 description
=> "Enable/disable Template.",
551 description
=> "Arbitrary arguments passed to kvm.",
552 verbose_description
=> <<EODESCR,
553 Arbitrary arguments passed to kvm, for example:
555 args: -no-reboot -no-hpet
557 NOTE: this option is for experts only.
564 description
=> "Enable/disable the USB tablet device.",
565 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
566 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
567 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
568 ." may consider disabling this to save some context switches. This is turned off by"
569 ." default if you use spice (`qm set <vmid> --vga qxl`).",
574 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
578 migrate_downtime
=> {
581 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
587 type
=> 'string', format
=> 'pve-qm-ide',
588 typetext
=> '<volume>',
589 description
=> "This is an alias for option -ide2",
593 description
=> "Emulated CPU type.",
595 format
=> 'pve-vm-cpu-conf',
597 parent
=> get_standard_option
('pve-snapshot-name', {
599 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
603 description
=> "Timestamp for snapshots.",
609 type
=> 'string', format
=> 'pve-volume-id',
610 description
=> "Reference to a volume which stores the VM state. This is used internally"
613 vmstatestorage
=> get_standard_option
('pve-storage-id', {
614 description
=> "Default storage for VM state volumes/files.",
617 runningmachine
=> get_standard_option
('pve-qemu-machine', {
618 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
622 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
623 ." internally for snapshots.",
626 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
627 format_description
=> 'QEMU -cpu parameter'
629 machine
=> get_standard_option
('pve-qemu-machine'),
631 description
=> "Virtual processor architecture. Defaults to the host.",
634 enum
=> [qw(x86_64 aarch64)],
637 description
=> "Specify SMBIOS type 1 fields.",
638 type
=> 'string', format
=> 'pve-qm-smbios1',
645 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
646 ." remove disk operations.",
652 enum
=> [ qw(seabios ovmf) ],
653 description
=> "Select BIOS implementation.",
654 default => 'seabios',
658 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
659 format_description
=> 'UUID',
660 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
661 ." to disable explicitly.",
662 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
663 ." value identifier to the guest OS. This allows to notify the guest operating system"
664 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
665 ." execution or creation from a template). The guest operating system notices the"
666 ." change, and is then able to react as appropriate by marking its copies of"
667 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
668 ."Note that auto-creation only works when done through API/CLI create or update methods"
669 .", but not when manually editing the config file.",
670 default => "1 (autogenerated)",
675 format
=> 'pve-volume-id',
677 description
=> "Script that will be executed during various steps in the vms lifetime.",
681 format
=> $ivshmem_fmt,
682 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
688 format
=> $audio_fmt,
689 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
692 spice_enhancements
=> {
694 format
=> $spice_enhancements_fmt,
695 description
=> "Configure additional enhancements for SPICE.",
699 type
=> 'string', format
=> 'pve-tag-list',
700 description
=> 'Tags of the VM. This is only meta information.',
706 description
=> "Configure a VirtIO-based Random Number Generator.",
711 format
=> $meta_info_fmt,
712 description
=> "Some (read-only) meta-information about this guest.",
716 type
=> 'string', format
=> 'pve-cpuset',
717 description
=> "List of host cores used to execute guest processes, for example: 0,5,8-11",
726 description
=> 'Specify a custom file containing all meta data passed to the VM via"
727 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
728 format
=> 'pve-volume-id',
729 format_description
=> 'volume',
734 description
=> 'To pass a custom file containing all network data to the VM via cloud-init.',
735 format
=> 'pve-volume-id',
736 format_description
=> 'volume',
741 description
=> 'To pass a custom file containing all user data to the VM via cloud-init.',
742 format
=> 'pve-volume-id',
743 format_description
=> 'volume',
748 description
=> 'To pass a custom file containing all vendor data to the VM via cloud-init.',
749 format
=> 'pve-volume-id',
750 format_description
=> 'volume',
753 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
755 my $confdesc_cloudinit = {
759 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
760 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
761 .' and `configdrive2` for windows.',
762 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
767 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
768 ." image's configured default user.",
773 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
774 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
775 .' support hashed passwords.',
780 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
782 format
=> 'pve-qm-cicustom',
787 description
=> 'cloud-init: Sets DNS search domains for a container. Create will'
788 .' automatically use the setting from the host if neither searchdomain nor nameserver'
793 type
=> 'string', format
=> 'address-list',
794 description
=> 'cloud-init: Sets DNS server IP address for a container. Create will'
795 .' automatically use the setting from the host if neither searchdomain nor nameserver'
801 format
=> 'urlencoded',
802 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
806 # what about other qemu settings ?
808 #machine => 'string',
821 ##soundhw => 'string',
823 while (my ($k, $v) = each %$confdesc) {
824 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
827 my $MAX_USB_DEVICES = 14;
829 my $MAX_SERIAL_PORTS = 4;
830 my $MAX_PARALLEL_PORTS = 3;
836 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
837 description
=> "CPUs accessing this NUMA node.",
838 format_description
=> "id[-id];...",
842 description
=> "Amount of memory this NUMA node provides.",
847 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
848 description
=> "Host NUMA nodes to use.",
849 format_description
=> "id[-id];...",
854 enum
=> [qw(preferred bind interleave)],
855 description
=> "NUMA allocation policy.",
859 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
862 type
=> 'string', format
=> $numa_fmt,
863 description
=> "NUMA topology.",
865 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
867 for (my $i = 0; $i < $MAX_NUMA; $i++) {
868 $confdesc->{"numa$i"} = $numadesc;
871 my $nic_model_list = [
887 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
889 my $net_fmt_bridge_descr = <<__EOD__;
890 Bridge to attach the network device to. The Proxmox VE standard bridge
893 If you do not specify a bridge, we create a kvm user (NATed) network
894 device, which provides DHCP and DNS services. The following addresses
901 The DHCP server assign addresses to the guest starting from 10.0.2.15.
905 macaddr
=> get_standard_option
('mac-addr', {
906 description
=> "MAC address. That address must be unique withing your network. This is"
907 ." automatically generated if not specified.",
911 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
912 ." very low CPU overhead. If your guest does not support this driver, it is usually"
913 ." best to use 'e1000'.",
914 enum
=> $nic_model_list,
917 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
918 bridge
=> get_standard_option
('pve-bridge-id', {
919 description
=> $net_fmt_bridge_descr,
924 minimum
=> 0, maximum
=> 64,
925 description
=> 'Number of packet queues to be used on the device.',
931 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
936 minimum
=> 1, maximum
=> 4094,
937 description
=> 'VLAN tag to apply to packets on this interface.',
942 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
943 description
=> 'VLAN trunks to pass through this interface.',
944 format_description
=> 'vlanid[;vlanid...]',
949 description
=> 'Whether this interface should be protected by the firewall.',
954 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
959 minimum
=> 1, maximum
=> 65520,
960 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
967 type
=> 'string', format
=> $net_fmt,
968 description
=> "Specify network devices.",
971 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
976 format
=> 'pve-ipv4-config',
977 format_description
=> 'IPv4Format/CIDR',
978 description
=> 'IPv4 address in CIDR format.',
985 format_description
=> 'GatewayIPv4',
986 description
=> 'Default gateway for IPv4 traffic.',
992 format
=> 'pve-ipv6-config',
993 format_description
=> 'IPv6Format/CIDR',
994 description
=> 'IPv6 address in CIDR format.',
1001 format_description
=> 'GatewayIPv6',
1002 description
=> 'Default gateway for IPv6 traffic.',
1007 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
1008 my $ipconfigdesc = {
1010 type
=> 'string', format
=> 'pve-qm-ipconfig',
1011 description
=> <<'EODESCR',
1012 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1014 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1016 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1017 gateway should be provided.
1018 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1019 cloud-init 19.4 or newer.
1021 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1025 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
1027 for (my $i = 0; $i < $MAX_NETS; $i++) {
1028 $confdesc->{"net$i"} = $netdesc;
1029 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1032 foreach my $key (keys %$confdesc_cloudinit) {
1033 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1036 PVE
::JSONSchema
::register_format
('pve-cpuset', \
&pve_verify_cpuset
);
1037 sub pve_verify_cpuset
{
1038 my ($set_text, $noerr) = @_;
1040 my ($count, $members) = eval { PVE
::CpuSet
::parse_cpuset
($set_text) };
1044 die "unable to parse cpuset option\n";
1047 return PVE
::CpuSet-
>new($members)->short_string();
1050 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1051 sub verify_volume_id_or_qm_path
{
1052 my ($volid, $noerr) = @_;
1054 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1056 return verify_volume_id_or_absolute_path
($volid, $noerr);
1059 PVE
::JSONSchema
::register_format
('pve-volume-id-or-absolute-path', \
&verify_volume_id_or_absolute_path
);
1060 sub verify_volume_id_or_absolute_path
{
1061 my ($volid, $noerr) = @_;
1063 return $volid if $volid =~ m
|^/|;
1065 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1076 type
=> 'string', format
=> 'pve-qm-usb-device',
1077 format_description
=> 'HOSTUSBDEVICE|spice',
1078 description
=> <<EODESCR,
1079 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1081 'bus-port(.port)*' (decimal numbers) or
1082 'vendor_id:product_id' (hexadeciaml numbers) or
1085 You can use the 'lsusb -t' command to list existing usb devices.
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1090 The value 'spice' can be used to add a usb redirection devices for spice.
1096 description
=> "Specifies whether if given host option is a USB3 device or port."
1097 ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1098 ." is irrelevant (all devices are plugged into a xhci controller).",
1105 type
=> 'string', format
=> $usb_fmt,
1106 description
=> "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1107 ." l26 or windows > 7, n can be up to 14).",
1109 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1114 pattern
=> '(/dev/.+|socket)',
1115 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1116 verbose_description
=> <<EODESCR,
1117 Create a serial device inside the VM (n is 0 to 3), and pass through a
1118 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1119 host side (use 'qm terminal' to open a terminal connection).
1121 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1122 use with special care.
1124 CAUTION: Experimental! User reported problems with this option.
1131 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1132 description
=> "Map host parallel devices (n is 0 to 2).",
1133 verbose_description
=> <<EODESCR,
1134 Map host parallel devices (n is 0 to 2).
1136 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1137 machines - use with special care.
1139 CAUTION: Experimental! User reported problems with this option.
1143 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1144 $confdesc->{"parallel$i"} = $paralleldesc;
1147 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1148 $confdesc->{"serial$i"} = $serialdesc;
1151 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1152 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1155 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1156 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1159 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1160 $confdesc->{"usb$i"} = $usbdesc;
1168 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1169 . " Deprecated, use 'order=' instead.",
1170 pattern
=> '[acdn]{1,4}',
1171 format_description
=> "[acdn]{1,4}",
1173 # note: this is also the fallback if boot: is not given at all
1179 format
=> 'pve-qm-bootdev-list',
1180 format_description
=> "device[;device...]",
1181 description
=> <<EODESC,
1182 The guest will attempt to boot from devices in the order they appear here.
1184 Disks, optical drives and passed-through storage USB devices will be directly
1185 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1186 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1188 Note that only devices in this list will be marked as bootable and thus loaded
1189 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1190 (e.g. software-raid), you need to specify all of them here.
1192 Overrides the deprecated 'legacy=[acdn]*' value when given.
1196 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1198 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1199 sub verify_bootdev
{
1200 my ($dev, $noerr) = @_;
1202 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1203 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && !$special;
1207 return 0 if $dev !~ m/^$base\d+$/;
1208 return 0 if !$confdesc->{$dev};
1212 return $dev if $check->("net");
1213 return $dev if $check->("usb");
1214 return $dev if $check->("hostpci");
1217 die "invalid boot device '$dev'\n";
1220 sub print_bootorder
{
1222 return "" if !@$devs;
1223 my $data = { order
=> join(';', @$devs) };
1224 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1227 my $kvm_api_version = 0;
1230 return $kvm_api_version if $kvm_api_version;
1232 open my $fh, '<', '/dev/kvm' or return;
1234 # 0xae00 => KVM_GET_API_VERSION
1235 $kvm_api_version = ioctl($fh, 0xae00, 0);
1238 return $kvm_api_version;
1241 my $kvm_user_version = {};
1244 sub kvm_user_version
{
1247 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1248 my $st = stat($binary);
1250 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1251 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1252 $cachedmtime == $st->mtime;
1254 $kvm_user_version->{$binary} = 'unknown';
1255 $kvm_mtime->{$binary} = $st->mtime;
1259 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1260 $kvm_user_version->{$binary} = $2;
1264 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1267 return $kvm_user_version->{$binary};
1270 my sub extract_version
{
1271 my ($machine_type, $version) = @_;
1272 $version = kvm_user_version
() if !defined($version);
1273 return PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1276 sub kernel_has_vhost_net
{
1277 return -c
'/dev/vhost-net';
1282 return defined($confdesc->{$key});
1286 sub get_cdrom_path
{
1288 return $cdrom_path if defined($cdrom_path);
1290 $cdrom_path = first
{ -l
$_ } map { "/dev/cdrom$_" } ('', '1', '2');
1292 if (!defined($cdrom_path)) {
1293 log_warn
("no physical CD-ROM available, ignoring");
1301 my ($storecfg, $vmid, $cdrom) = @_;
1303 if ($cdrom eq 'cdrom') {
1304 return get_cdrom_path
();
1305 } elsif ($cdrom eq 'none') {
1307 } elsif ($cdrom =~ m
|^/|) {
1310 return PVE
::Storage
::path
($storecfg, $cdrom);
1314 # try to convert old style file names to volume IDs
1315 sub filename_to_volume_id
{
1316 my ($vmid, $file, $media) = @_;
1318 if (!($file eq 'none' || $file eq 'cdrom' ||
1319 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1321 return if $file =~ m
|/|;
1323 if ($media && $media eq 'cdrom') {
1324 $file = "local:iso/$file";
1326 $file = "local:$vmid/$file";
1333 sub verify_media_type
{
1334 my ($opt, $vtype, $media) = @_;
1339 if ($media eq 'disk') {
1341 } elsif ($media eq 'cdrom') {
1344 die "internal error";
1347 return if ($vtype eq $etype);
1349 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1352 sub cleanup_drive_path
{
1353 my ($opt, $storecfg, $drive) = @_;
1355 # try to convert filesystem paths to volume IDs
1357 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1358 ($drive->{file
} !~ m
|^/dev/.+|) &&
1359 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1360 ($drive->{file
} !~ m/^\d+$/)) {
1361 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1362 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1364 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1365 verify_media_type
($opt, $vtype, $drive->{media
});
1366 $drive->{file
} = $volid;
1369 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1372 sub parse_hotplug_features
{
1377 return $res if $data eq '0';
1379 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1381 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1382 if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1385 die "invalid hotplug feature '$feature'\n";
1391 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1392 sub pve_verify_hotplug_features
{
1393 my ($value, $noerr) = @_;
1395 return $value if parse_hotplug_features
($value);
1399 die "unable to parse hotplug option\n";
1403 my($fh, $noerr) = @_;
1406 my $SG_GET_VERSION_NUM = 0x2282;
1408 my $versionbuf = "\x00" x
8;
1409 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1411 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1414 my $version = unpack("I", $versionbuf);
1415 if ($version < 30000) {
1416 die "scsi generic interface too old\n" if !$noerr;
1420 my $buf = "\x00" x
36;
1421 my $sensebuf = "\x00" x
8;
1422 my $cmd = pack("C x3 C x1", 0x12, 36);
1424 # see /usr/include/scsi/sg.h
1425 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1428 $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1431 $ret = ioctl($fh, $SG_IO, $packet);
1433 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1437 my @res = unpack($sg_io_hdr_t, $packet);
1438 if ($res[17] || $res[18]) {
1439 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1444 $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1446 $res->{removable
} = $res->{removable
} & 128 ?
1 : 0;
1447 $res->{type
} &= 0x1F;
1455 my $fh = IO
::File-
>new("+<$path") || return;
1456 my $res = scsi_inquiry
($fh, 1);
1462 sub print_tabletdevice_full
{
1463 my ($conf, $arch) = @_;
1465 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1467 # we use uhci for old VMs because tablet driver was buggy in older qemu
1469 if ($q35 || $arch eq 'aarch64') {
1475 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1478 sub print_keyboarddevice_full
{
1479 my ($conf, $arch) = @_;
1481 return if $arch ne 'aarch64';
1483 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1486 my sub get_drive_id
{
1488 return "$drive->{interface}$drive->{index}";
1491 sub print_drivedevice_full
{
1492 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1497 my $drive_id = get_drive_id
($drive);
1498 if ($drive->{interface
} eq 'virtio') {
1499 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1500 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1501 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1502 } elsif ($drive->{interface
} eq 'scsi') {
1504 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1505 my $unit = $drive->{index} % $maxdev;
1506 my $devicetype = 'hd';
1508 if (drive_is_cdrom
($drive)) {
1511 if ($drive->{file
} =~ m
|^/|) {
1512 $path = $drive->{file
};
1513 if (my $info = path_is_scsi
($path)) {
1514 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1515 $devicetype = 'block';
1516 } elsif ($info->{type
} == 1) { # tape
1517 $devicetype = 'generic';
1521 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1524 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1525 my $version = extract_version
($machine_type, kvm_user_version
());
1526 if ($path =~ m/^iscsi\:\/\
// &&
1527 !min_version
($version, 4, 1)) {
1528 $devicetype = 'generic';
1532 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1533 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1535 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1536 .",lun=$drive->{index}";
1538 $device .= ",drive=drive-$drive_id,id=$drive_id";
1540 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1541 $device .= ",rotation_rate=1";
1543 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1545 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1546 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1547 my $controller = int($drive->{index} / $maxdev);
1548 my $unit = $drive->{index} % $maxdev;
1549 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1551 $device = "ide-$devicetype";
1552 if ($drive->{interface
} eq 'ide') {
1553 $device .= ",bus=ide.$controller,unit=$unit";
1555 $device .= ",bus=ahci$controller.$unit";
1557 $device .= ",drive=drive-$drive_id,id=$drive_id";
1559 if ($devicetype eq 'hd') {
1560 if (my $model = $drive->{model
}) {
1561 $model = URI
::Escape
::uri_unescape
($model);
1562 $device .= ",model=$model";
1564 if ($drive->{ssd
}) {
1565 $device .= ",rotation_rate=1";
1568 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1569 } elsif ($drive->{interface
} eq 'usb') {
1571 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1573 die "unsupported interface type";
1576 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1578 if (my $serial = $drive->{serial
}) {
1579 $serial = URI
::Escape
::uri_unescape
($serial);
1580 $device .= ",serial=$serial";
1587 sub get_initiator_name
{
1590 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1591 while (defined(my $line = <$fh>)) {
1592 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1601 my sub storage_allows_io_uring_default
{
1602 my ($scfg, $cache_direct) = @_;
1604 # io_uring with cache mode writeback or writethrough on krbd will hang...
1605 return if $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1607 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1608 # sometimes, just plain disable...
1609 return if $scfg && $scfg->{type
} eq 'lvm';
1611 # io_uring causes problems when used with CIFS since kernel 5.15
1612 # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1613 return if $scfg && $scfg->{type
} eq 'cifs';
1618 sub print_drive_commandline_full
{
1619 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1622 my $volid = $drive->{file
};
1623 my $format = $drive->{format
};
1624 my $drive_id = get_drive_id
($drive);
1626 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1627 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1629 if (drive_is_cdrom
($drive)) {
1630 $path = get_iso_path
($storecfg, $vmid, $volid);
1631 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1634 $path = PVE
::Storage
::path
($storecfg, $volid);
1635 $format //= qemu_img_format
($scfg, $volname);
1642 my $is_rbd = $path =~ m/^rbd:/;
1645 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1646 foreach my $o (@qemu_drive_options) {
1647 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1650 # snapshot only accepts on|off
1651 if (defined($drive->{snapshot
})) {
1652 my $v = $drive->{snapshot
} ?
'on' : 'off';
1653 $opts .= ",snapshot=$v";
1656 if (defined($drive->{ro
})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1657 $opts .= ",readonly=" . ($drive->{ro
} ?
'on' : 'off');
1660 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1661 my ($dir, $qmpname) = @$type;
1662 if (my $v = $drive->{"mbps$dir"}) {
1663 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1665 if (my $v = $drive->{"mbps${dir}_max"}) {
1666 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1668 if (my $v = $drive->{"bps${dir}_max_length"}) {
1669 $opts .= ",throttling.bps$qmpname-max-length=$v";
1671 if (my $v = $drive->{"iops${dir}"}) {
1672 $opts .= ",throttling.iops$qmpname=$v";
1674 if (my $v = $drive->{"iops${dir}_max"}) {
1675 $opts .= ",throttling.iops$qmpname-max=$v";
1677 if (my $v = $drive->{"iops${dir}_max_length"}) {
1678 $opts .= ",throttling.iops$qmpname-max-length=$v";
1683 $format = "rbd" if $is_rbd;
1684 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1686 $opts .= ",format=alloc-track,file.driver=$format";
1688 $opts .= ",format=$format";
1691 my $cache_direct = 0;
1693 if (my $cache = $drive->{cache
}) {
1694 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1695 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1696 $opts .= ",cache=none";
1700 if (!$drive->{aio
}) {
1701 if ($io_uring && storage_allows_io_uring_default
($scfg, $cache_direct)) {
1702 # io_uring supports all cache modes
1703 $opts .= ",aio=io_uring";
1705 # aio native works only with O_DIRECT
1707 $opts .= ",aio=native";
1709 $opts .= ",aio=threads";
1714 if (!drive_is_cdrom
($drive)) {
1716 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1717 $detectzeroes = 'off';
1718 } elsif ($drive->{discard
}) {
1719 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1721 # This used to be our default with discard not being specified:
1722 $detectzeroes = 'on';
1725 # note: 'detect-zeroes' works per blockdev and we want it to persist
1726 # after the alloc-track is removed, so put it on 'file' directly
1727 my $dz_param = $pbs_name ?
"file.detect-zeroes" : "detect-zeroes";
1728 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1732 $opts .= ",backing=$pbs_name";
1733 $opts .= ",auto-remove=on";
1736 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1737 my $file_param = "file";
1739 # non-rbd drivers require the underlying file to be a seperate block
1740 # node, so add a second .file indirection
1741 $file_param .= ".file" if !$is_rbd;
1742 $file_param .= ".filename";
1744 my $pathinfo = $path ?
"$file_param=$path," : '';
1746 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1749 sub print_pbs_blockdev
{
1750 my ($pbs_conf, $pbs_name) = @_;
1751 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1752 $blockdev .= ",repository=$pbs_conf->{repository}";
1753 $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace
};
1754 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1755 $blockdev .= ",archive=$pbs_conf->{archive}";
1756 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1760 sub print_netdevice_full
{
1761 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1763 my $device = $net->{model
};
1764 if ($net->{model
} eq 'virtio') {
1765 $device = 'virtio-net-pci';
1768 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1769 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1770 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1771 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1772 # and out of each queue plus one config interrupt and control vector queue
1773 my $vectors = $net->{queues
} * 2 + 2;
1774 $tmpstr .= ",vectors=$vectors,mq=on";
1775 if (min_version
($machine_version, 7, 1)) {
1776 $tmpstr .= ",packed=on";
1780 if (min_version
($machine_version, 7, 1) && $net->{model
} eq 'virtio'){
1781 $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1784 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1786 if (my $mtu = $net->{mtu
}) {
1787 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1788 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1791 } elsif ($mtu < 576) {
1792 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1793 } elsif ($mtu > $bridge_mtu) {
1794 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1796 $tmpstr .= ",host_mtu=$mtu";
1798 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1802 if ($use_old_bios_files) {
1804 if ($device eq 'virtio-net-pci') {
1805 $romfile = 'pxe-virtio.rom';
1806 } elsif ($device eq 'e1000') {
1807 $romfile = 'pxe-e1000.rom';
1808 } elsif ($device eq 'e1000e') {
1809 $romfile = 'pxe-e1000e.rom';
1810 } elsif ($device eq 'ne2k') {
1811 $romfile = 'pxe-ne2k_pci.rom';
1812 } elsif ($device eq 'pcnet') {
1813 $romfile = 'pxe-pcnet.rom';
1814 } elsif ($device eq 'rtl8139') {
1815 $romfile = 'pxe-rtl8139.rom';
1817 $tmpstr .= ",romfile=$romfile" if $romfile;
1823 sub print_netdev_full
{
1824 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1827 if ($netid =~ m/^net(\d+)$/) {
1831 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1833 my $ifname = "tap${vmid}i$i";
1835 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1836 die "interface name '$ifname' is too long (max 15 character)\n"
1837 if length($ifname) >= 16;
1839 my $vhostparam = '';
1840 if (is_native
($arch)) {
1841 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1844 my $vmname = $conf->{name
} || "vm$vmid";
1847 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1849 if ($net->{bridge
}) {
1850 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1851 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1853 $netdev = "type=user,id=$netid,hostname=$vmname";
1856 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1862 'cirrus' => 'cirrus-vga',
1864 'vmware' => 'vmware-svga',
1865 'virtio' => 'virtio-vga',
1866 'virtio-gl' => 'virtio-vga-gl',
1869 sub print_vga_device
{
1870 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1872 my $type = $vga_map->{$vga->{type
}};
1873 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1874 $type = 'virtio-gpu';
1876 my $vgamem_mb = $vga->{memory
};
1878 my $max_outputs = '';
1880 $type = $id ?
'qxl' : 'qxl-vga';
1882 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1883 # set max outputs so linux can have up to 4 qxl displays with one device
1884 if (min_version
($machine_version, 4, 1)) {
1885 $max_outputs = ",max_outputs=4";
1890 die "no devicetype for $vga->{type}\n" if !$type;
1894 if ($vga->{type
} =~ /^virtio/) {
1895 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1896 $memory = ",max_hostmem=$bytes";
1898 # from https://www.spice-space.org/multiple-monitors.html
1899 $memory = ",vgamem_mb=$vga->{memory}";
1900 my $ram = $vgamem_mb * 4;
1901 my $vram = $vgamem_mb * 2;
1902 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1904 $memory = ",vgamem_mb=$vga->{memory}";
1906 } elsif ($qxlnum && $id) {
1907 $memory = ",ram_size=67108864,vram_size=33554432";
1911 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1912 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1915 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1916 my $vgaid = "vga" . ($id // '');
1918 if ($q35 && $vgaid eq 'vga') {
1919 # the first display uses pcie.0 bus on q35 machines
1920 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1922 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1925 if ($vga->{type
} eq 'virtio-gl') {
1926 my $base = '/usr/lib/x86_64-linux-gnu/lib';
1927 die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1928 if !-e
"${base}EGL.so.1" || !-e
"${base}GL.so.1";
1930 die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1931 if !PVE
::Tools
::dir_glob_regex
('/dev/dri/', "renderD.*");
1934 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1937 sub parse_number_sets
{
1940 foreach my $part (split(/;/, $set)) {
1941 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1942 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1943 push @$res, [ $1, $2 ];
1945 die "invalid range: $part\n";
1954 my $res = parse_property_string
($numa_fmt, $data);
1955 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1956 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1960 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1962 my ($data, $disable_mac_autogen) = @_;
1964 my $res = eval { parse_property_string
($net_fmt, $data) };
1969 if (!defined($res->{macaddr
}) && !$disable_mac_autogen) {
1970 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1971 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1976 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1977 sub parse_ipconfig
{
1980 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1986 if ($res->{gw
} && !$res->{ip
}) {
1987 warn 'gateway specified without specifying an IP address';
1990 if ($res->{gw6
} && !$res->{ip6
}) {
1991 warn 'IPv6 gateway specified without specifying an IPv6 address';
1994 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1995 warn 'gateway specified together with DHCP';
1998 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
2000 warn "IPv6 gateway specified together with $res->{ip6} address";
2004 if (!$res->{ip
} && !$res->{ip6
}) {
2005 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
2014 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
2017 sub add_random_macs
{
2018 my ($settings) = @_;
2020 foreach my $opt (keys %$settings) {
2021 next if $opt !~ m/^net(\d+)$/;
2022 my $net = parse_net
($settings->{$opt});
2024 $settings->{$opt} = print_net
($net);
2028 sub vm_is_volid_owner
{
2029 my ($storecfg, $vmid, $volid) = @_;
2031 if ($volid !~ m
|^/|) {
2033 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
2034 if ($owner && ($owner == $vmid)) {
2042 sub vmconfig_register_unused_drive
{
2043 my ($storecfg, $vmid, $conf, $drive) = @_;
2045 if (drive_is_cloudinit
($drive)) {
2046 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2048 delete $conf->{cloudinit
};
2049 } elsif (!drive_is_cdrom
($drive)) {
2050 my $volid = $drive->{file
};
2051 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2052 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2057 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2061 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2062 format_description
=> 'UUID',
2063 description
=> "Set SMBIOS1 UUID.",
2068 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2069 format_description
=> 'Base64 encoded string',
2070 description
=> "Set SMBIOS1 version.",
2075 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2076 format_description
=> 'Base64 encoded string',
2077 description
=> "Set SMBIOS1 serial number.",
2082 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2083 format_description
=> 'Base64 encoded string',
2084 description
=> "Set SMBIOS1 manufacturer.",
2089 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2090 format_description
=> 'Base64 encoded string',
2091 description
=> "Set SMBIOS1 product ID.",
2096 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2097 format_description
=> 'Base64 encoded string',
2098 description
=> "Set SMBIOS1 SKU string.",
2103 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2104 format_description
=> 'Base64 encoded string',
2105 description
=> "Set SMBIOS1 family string.",
2110 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2118 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2125 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2128 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2130 sub parse_watchdog
{
2135 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2140 sub parse_guest_agent
{
2143 return {} if !defined($conf->{agent
});
2145 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2148 # if the agent is disabled ignore the other potentially set properties
2149 return {} if !$res->{enabled
};
2154 my ($conf, $key) = @_;
2155 return undef if !defined($conf->{agent
});
2157 my $agent = parse_guest_agent
($conf);
2158 return $agent->{$key};
2164 return {} if !$value;
2165 my $res = eval { parse_property_string
($vga_fmt, $value) };
2175 my $res = eval { parse_property_string
($rng_fmt, $value) };
2180 sub parse_meta_info
{
2185 my $res = eval { parse_property_string
($meta_info_fmt, $value) };
2190 sub new_meta_info_string
{
2191 my () = @_; # for now do not allow to override any value
2193 return PVE
::JSONSchema
::print_property_string
(
2195 'creation-qemu' => kvm_user_version
(),
2196 ctime
=> "". int(time()),
2202 sub qemu_created_version_fixups
{
2203 my ($conf, $forcemachine, $kvmver) = @_;
2205 my $meta = parse_meta_info
($conf->{meta
}) // {};
2206 my $forced_vers = PVE
::QemuServer
::Machine
::extract_version
($forcemachine);
2208 # check if we need to apply some handling for VMs that always use the latest machine version but
2209 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2210 # would be required (we do not want to pin machine version for non-windows OS type)
2212 (!defined($conf->{machine
}) || $conf->{machine
} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2213 && (!defined($meta->{'creation-qemu'}) || !min_version
($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2214 && (!$forced_vers || min_version
($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2215 && min_version
($kvmver, 6, 1) # only need to apply the change since 6.1
2217 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2218 if ($q35 && $conf->{ostype
} && $conf->{ostype
} eq 'l26') {
2219 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2220 # and thus with the predictable interface naming of systemd
2221 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2227 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2228 sub verify_usb_device
{
2229 my ($value, $noerr) = @_;
2231 return $value if parse_usb_device
($value);
2235 die "unable to parse usb device\n";
2238 # add JSON properties for create and set function
2239 sub json_config_properties
{
2240 my ($prop, $with_disk_alloc) = @_;
2242 my $skip_json_config_opts = {
2246 runningmachine
=> 1,
2251 foreach my $opt (keys %$confdesc) {
2252 next if $skip_json_config_opts->{$opt};
2254 if ($with_disk_alloc && is_valid_drivename
($opt)) {
2255 $prop->{$opt} = $PVE::QemuServer
::Drive
::drivedesc_hash_with_alloc-
>{$opt};
2257 $prop->{$opt} = $confdesc->{$opt};
2264 # Properties that we can read from an OVF file
2265 sub json_ovf_properties
{
2268 for my $device (PVE
::QemuServer
::Drive
::valid_drive_names
()) {
2269 $prop->{$device} = {
2271 format
=> 'pve-volume-id-or-absolute-path',
2272 description
=> "Disk image that gets imported to $device",
2279 description
=> "The number of CPU cores.",
2284 description
=> "Amount of RAM for the VM in MB.",
2289 description
=> "Name of the VM.",
2296 # return copy of $confdesc_cloudinit to generate documentation
2297 sub cloudinit_config_properties
{
2299 return dclone
($confdesc_cloudinit);
2302 sub cloudinit_pending_properties
{
2304 map { $_ => 1 } keys $confdesc_cloudinit->%*,
2307 $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2312 my ($key, $value) = @_;
2314 die "unknown setting '$key'\n" if !$confdesc->{$key};
2316 my $type = $confdesc->{$key}->{type
};
2318 if (!defined($value)) {
2319 die "got undefined value\n";
2322 if ($value =~ m/[\n\r]/) {
2323 die "property contains a line feed\n";
2326 if ($type eq 'boolean') {
2327 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2328 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2329 die "type check ('boolean') failed - got '$value'\n";
2330 } elsif ($type eq 'integer') {
2331 return int($1) if $value =~ m/^(\d+)$/;
2332 die "type check ('integer') failed - got '$value'\n";
2333 } elsif ($type eq 'number') {
2334 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2335 die "type check ('number') failed - got '$value'\n";
2336 } elsif ($type eq 'string') {
2337 if (my $fmt = $confdesc->{$key}->{format
}) {
2338 PVE
::JSONSchema
::check_format
($fmt, $value);
2341 $value =~ s/^\"(.*)\"$/$1/;
2344 die "internal error"
2349 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2351 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2353 if (!$skiplock && !PVE
::QemuConfig-
>has_lock($conf, 'suspended')) {
2354 PVE
::QemuConfig-
>check_lock($conf);
2357 if ($conf->{template
}) {
2358 # check if any base image is still used by a linked clone
2359 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2360 my ($ds, $drive) = @_;
2361 return if drive_is_cdrom
($drive);
2363 my $volid = $drive->{file
};
2364 return if !$volid || $volid =~ m
|^/|;
2366 die "base volume '$volid' is still in use by linked cloned\n"
2367 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2373 my $remove_owned_drive = sub {
2374 my ($ds, $drive) = @_;
2375 return if drive_is_cdrom
($drive, 1);
2377 my $volid = $drive->{file
};
2378 return if !$volid || $volid =~ m
|^/|;
2379 return if $volids->{$volid};
2381 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2382 return if !$path || !$owner || ($owner != $vmid);
2384 $volids->{$volid} = 1;
2385 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2386 warn "Could not remove disk '$volid', check manually: $@" if $@;
2389 # only remove disks owned by this VM (referenced in the config)
2390 my $include_opts = {
2391 include_unused
=> 1,
2392 extra_keys
=> ['vmstate'],
2394 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2396 for my $snap (values %{$conf->{snapshots
}}) {
2397 next if !defined($snap->{vmstate
});
2398 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2399 next if !defined($drive);
2400 $remove_owned_drive->('vmstate', $drive);
2403 PVE
::QemuConfig-
>foreach_volume_full($conf->{pending
}, $include_opts, $remove_owned_drive);
2405 if ($purge_unreferenced) { # also remove unreferenced disk
2406 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2407 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2408 my ($volid, $sid, $volname, $d) = @_;
2409 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2414 if (defined $replacement_conf) {
2415 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2417 PVE
::QemuConfig-
>destroy_config($vmid);
2421 sub parse_vm_config
{
2422 my ($filename, $raw, $strict) = @_;
2424 return if !defined($raw);
2427 digest
=> Digest
::SHA
::sha1_hex
($raw),
2433 my $handle_error = sub {
2443 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2444 || die "got strange filename '$filename'";
2450 my $finish_description = sub {
2451 if (defined($descr)) {
2453 $conf->{description
} = $descr;
2459 my @lines = split(/\n/, $raw);
2460 foreach my $line (@lines) {
2461 next if $line =~ m/^\s*$/;
2463 if ($line =~ m/^\[PENDING\]\s*$/i) {
2464 $section = 'pending';
2465 $finish_description->();
2466 $conf = $res->{$section} = {};
2468 } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2469 $section = 'cloudinit';
2470 $finish_description->();
2471 $conf = $res->{$section} = {};
2474 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2476 $finish_description->();
2477 $conf = $res->{snapshots
}->{$section} = {};
2481 if ($line =~ m/^\#(.*)$/) {
2482 $descr = '' if !defined($descr);
2483 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2487 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2488 $descr = '' if !defined($descr);
2489 $descr .= PVE
::Tools
::decode_text
($2);
2490 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2491 $conf->{snapstate
} = $1;
2492 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2495 $conf->{$key} = $value;
2496 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2498 if ($section eq 'pending') {
2499 $conf->{delete} = $value; # we parse this later
2501 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2503 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2506 if ($section eq 'cloudinit') {
2507 # ignore validation only used for informative purpose
2508 $conf->{$key} = $value;
2511 eval { $value = check_type
($key, $value); };
2513 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2515 $key = 'ide2' if $key eq 'cdrom';
2516 my $fmt = $confdesc->{$key}->{format
};
2517 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2518 my $v = parse_drive
($key, $value);
2519 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2520 $v->{file
} = $volid;
2521 $value = print_drive
($v);
2523 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2528 $conf->{$key} = $value;
2531 $handle_error->("vm $vmid - unable to parse config: $line\n");
2535 $finish_description->();
2536 delete $res->{snapstate
}; # just to be sure
2541 sub write_vm_config
{
2542 my ($filename, $conf) = @_;
2544 delete $conf->{snapstate
}; # just to be sure
2546 if ($conf->{cdrom
}) {
2547 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2548 $conf->{ide2
} = $conf->{cdrom
};
2549 delete $conf->{cdrom
};
2552 # we do not use 'smp' any longer
2553 if ($conf->{sockets
}) {
2554 delete $conf->{smp
};
2555 } elsif ($conf->{smp
}) {
2556 $conf->{sockets
} = $conf->{smp
};
2557 delete $conf->{cores
};
2558 delete $conf->{smp
};
2561 my $used_volids = {};
2563 my $cleanup_config = sub {
2564 my ($cref, $pending, $snapname) = @_;
2566 foreach my $key (keys %$cref) {
2567 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2568 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2569 my $value = $cref->{$key};
2570 if ($key eq 'delete') {
2571 die "propertry 'delete' is only allowed in [PENDING]\n"
2573 # fixme: check syntax?
2576 eval { $value = check_type
($key, $value); };
2577 die "unable to parse value of '$key' - $@" if $@;
2579 $cref->{$key} = $value;
2581 if (!$snapname && is_valid_drivename
($key)) {
2582 my $drive = parse_drive
($key, $value);
2583 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2588 &$cleanup_config($conf);
2590 &$cleanup_config($conf->{pending
}, 1);
2592 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2593 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2594 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2597 # remove 'unusedX' settings if we re-add a volume
2598 foreach my $key (keys %$conf) {
2599 my $value = $conf->{$key};
2600 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2601 delete $conf->{$key};
2605 my $generate_raw_config = sub {
2606 my ($conf, $pending) = @_;
2610 # add description as comment to top of file
2611 if (defined(my $descr = $conf->{description
})) {
2613 foreach my $cl (split(/\n/, $descr)) {
2614 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2617 $raw .= "#\n" if $pending;
2621 foreach my $key (sort keys %$conf) {
2622 next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2623 $raw .= "$key: $conf->{$key}\n";
2628 my $raw = &$generate_raw_config($conf);
2630 if (scalar(keys %{$conf->{pending
}})){
2631 $raw .= "\n[PENDING]\n";
2632 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2635 if (scalar(keys %{$conf->{cloudinit
}}) && PVE
::QemuConfig-
>has_cloudinit($conf)){
2636 $raw .= "\n[special:cloudinit]\n";
2637 $raw .= &$generate_raw_config($conf->{cloudinit
});
2640 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2641 $raw .= "\n[$snapname]\n";
2642 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2652 # we use static defaults from our JSON schema configuration
2653 foreach my $key (keys %$confdesc) {
2654 if (defined(my $default = $confdesc->{$key}->{default})) {
2655 $res->{$key} = $default;
2663 my $vmlist = PVE
::Cluster
::get_vmlist
();
2665 return $res if !$vmlist || !$vmlist->{ids
};
2666 my $ids = $vmlist->{ids
};
2667 my $nodename = nodename
();
2669 foreach my $vmid (keys %$ids) {
2670 my $d = $ids->{$vmid};
2671 next if !$d->{node
} || $d->{node
} ne $nodename;
2672 next if !$d->{type
} || $d->{type
} ne 'qemu';
2673 $res->{$vmid}->{exists} = 1;
2678 # test if VM uses local resources (to prevent migration)
2679 sub check_local_resources
{
2680 my ($conf, $noerr) = @_;
2684 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2685 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2687 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2689 foreach my $k (keys %$conf) {
2690 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2691 # sockets are safe: they will recreated be on the target side post-migrate
2692 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2693 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2696 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2701 # check if used storages are available on all nodes (use by migrate)
2702 sub check_storage_availability
{
2703 my ($storecfg, $conf, $node) = @_;
2705 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2706 my ($ds, $drive) = @_;
2708 my $volid = $drive->{file
};
2711 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2714 # check if storage is available on both nodes
2715 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2716 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2718 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2720 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2721 if !$scfg->{content
}->{$vtype};
2725 # list nodes where all VM images are available (used by has_feature API)
2727 my ($conf, $storecfg) = @_;
2729 my $nodelist = PVE
::Cluster
::get_nodelist
();
2730 my $nodehash = { map { $_ => 1 } @$nodelist };
2731 my $nodename = nodename
();
2733 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2734 my ($ds, $drive) = @_;
2736 my $volid = $drive->{file
};
2739 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2741 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2742 if ($scfg->{disable
}) {
2744 } elsif (my $avail = $scfg->{nodes
}) {
2745 foreach my $node (keys %$nodehash) {
2746 delete $nodehash->{$node} if !$avail->{$node};
2748 } elsif (!$scfg->{shared
}) {
2749 foreach my $node (keys %$nodehash) {
2750 delete $nodehash->{$node} if $node ne $nodename
2759 sub check_local_storage_availability
{
2760 my ($conf, $storecfg) = @_;
2762 my $nodelist = PVE
::Cluster
::get_nodelist
();
2763 my $nodehash = { map { $_ => {} } @$nodelist };
2765 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2766 my ($ds, $drive) = @_;
2768 my $volid = $drive->{file
};
2771 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2773 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2775 if ($scfg->{disable
}) {
2776 foreach my $node (keys %$nodehash) {
2777 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2779 } elsif (my $avail = $scfg->{nodes
}) {
2780 foreach my $node (keys %$nodehash) {
2781 if (!$avail->{$node}) {
2782 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2789 foreach my $node (values %$nodehash) {
2790 if (my $unavail = $node->{unavailable_storages
}) {
2791 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2798 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2800 my ($vmid, $nocheck, $node) = @_;
2802 # $nocheck is set when called during a migration, in which case the config
2803 # file might still or already reside on the *other* node
2804 # - because rename has already happened, and current node is source
2805 # - because rename hasn't happened yet, and current node is target
2806 # - because rename has happened, current node is target, but hasn't yet
2808 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2809 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2814 my $vzlist = config_list
();
2816 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2818 while (defined(my $de = $fd->read)) {
2819 next if $de !~ m/^(\d+)\.pid$/;
2821 next if !defined($vzlist->{$vmid});
2822 if (my $pid = check_running
($vmid)) {
2823 $vzlist->{$vmid}->{pid
} = $pid;
2830 our $vmstatus_return_properties = {
2831 vmid
=> get_standard_option
('pve-vmid'),
2833 description
=> "QEMU process status.",
2835 enum
=> ['stopped', 'running'],
2838 description
=> "Maximum memory in bytes.",
2841 renderer
=> 'bytes',
2844 description
=> "Root disk size in bytes.",
2847 renderer
=> 'bytes',
2850 description
=> "VM name.",
2855 description
=> "QEMU QMP agent status.",
2860 description
=> "PID of running qemu process.",
2865 description
=> "Uptime.",
2868 renderer
=> 'duration',
2871 description
=> "Maximum usable CPUs.",
2876 description
=> "The current config lock, if any.",
2881 description
=> "The current configured tags, if any",
2885 'running-machine' => {
2886 description
=> "The currently running machine type (if running).",
2891 description
=> "The currently running QEMU version (if running).",
2897 my $last_proc_pid_stat;
2899 # get VM status information
2900 # This must be fast and should not block ($full == false)
2901 # We only query KVM using QMP if $full == true (this can be slow)
2903 my ($opt_vmid, $full) = @_;
2907 my $storecfg = PVE
::Storage
::config
();
2909 my $list = vzlist
();
2910 my $defaults = load_defaults
();
2912 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2914 my $cpucount = $cpuinfo->{cpus
} || 1;
2916 foreach my $vmid (keys %$list) {
2917 next if $opt_vmid && ($vmid ne $opt_vmid);
2919 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2921 my $d = { vmid
=> int($vmid) };
2922 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2924 # fixme: better status?
2925 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2927 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2928 if (defined($size)) {
2929 $d->{disk
} = 0; # no info available
2930 $d->{maxdisk
} = $size;
2936 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2937 * ($conf->{cores
} || $defaults->{cores
});
2938 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2939 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2941 $d->{name
} = $conf->{name
} || "VM $vmid";
2942 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2943 : $defaults->{memory
}*(1024*1024);
2945 if ($conf->{balloon
}) {
2946 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2947 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2948 : $defaults->{shares
};
2959 $d->{diskwrite
} = 0;
2961 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2963 $d->{serial
} = 1 if conf_has_serial
($conf);
2964 $d->{lock} = $conf->{lock} if $conf->{lock};
2965 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2970 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2971 foreach my $dev (keys %$netdev) {
2972 next if $dev !~ m/^tap([1-9]\d*)i/;
2974 my $d = $res->{$vmid};
2977 $d->{netout
} += $netdev->{$dev}->{receive
};
2978 $d->{netin
} += $netdev->{$dev}->{transmit
};
2981 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2982 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2987 my $ctime = gettimeofday
;
2989 foreach my $vmid (keys %$list) {
2991 my $d = $res->{$vmid};
2992 my $pid = $d->{pid
};
2995 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2996 next if !$pstat; # not running
2998 my $used = $pstat->{utime} + $pstat->{stime
};
3000 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
3002 if ($pstat->{vsize
}) {
3003 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
3006 my $old = $last_proc_pid_stat->{$pid};
3008 $last_proc_pid_stat->{$pid} = {
3016 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
3018 if ($dtime > 1000) {
3019 my $dutime = $used - $old->{used
};
3021 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
3022 $last_proc_pid_stat->{$pid} = {
3028 $d->{cpu
} = $old->{cpu
};
3032 return $res if !$full;
3034 my $qmpclient = PVE
::QMPClient-
>new();
3036 my $ballooncb = sub {
3037 my ($vmid, $resp) = @_;
3039 my $info = $resp->{'return'};
3040 return if !$info->{max_mem
};
3042 my $d = $res->{$vmid};
3044 # use memory assigned to VM
3045 $d->{maxmem
} = $info->{max_mem
};
3046 $d->{balloon
} = $info->{actual
};
3048 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
3049 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
3050 $d->{freemem
} = $info->{free_mem
};
3053 $d->{ballooninfo
} = $info;
3056 my $blockstatscb = sub {
3057 my ($vmid, $resp) = @_;
3058 my $data = $resp->{'return'} || [];
3059 my $totalrdbytes = 0;
3060 my $totalwrbytes = 0;
3062 for my $blockstat (@$data) {
3063 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
3064 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
3066 $blockstat->{device
} =~ s/drive-//;
3067 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
3069 $res->{$vmid}->{diskread
} = $totalrdbytes;
3070 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
3073 my $machinecb = sub {
3074 my ($vmid, $resp) = @_;
3075 my $data = $resp->{'return'} || [];
3077 $res->{$vmid}->{'running-machine'} =
3078 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
3081 my $versioncb = sub {
3082 my ($vmid, $resp) = @_;
3083 my $data = $resp->{'return'} // {};
3084 my $version = 'unknown';
3086 if (my $v = $data->{qemu
}) {
3087 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
3090 $res->{$vmid}->{'running-qemu'} = $version;
3093 my $statuscb = sub {
3094 my ($vmid, $resp) = @_;
3096 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3097 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3098 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3099 # this fails if ballon driver is not loaded, so this must be
3100 # the last commnand (following command are aborted if this fails).
3101 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3103 my $status = 'unknown';
3104 if (!defined($status = $resp->{'return'}->{status
})) {
3105 warn "unable to get VM status\n";
3109 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3112 foreach my $vmid (keys %$list) {
3113 next if $opt_vmid && ($vmid ne $opt_vmid);
3114 next if !$res->{$vmid}->{pid
}; # not running
3115 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3118 $qmpclient->queue_execute(undef, 2);
3120 foreach my $vmid (keys %$list) {
3121 next if $opt_vmid && ($vmid ne $opt_vmid);
3122 next if !$res->{$vmid}->{pid
}; #not running
3124 # we can't use the $qmpclient since it might have already aborted on
3125 # 'query-balloon', but this might also fail for older versions...
3126 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
3127 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3130 foreach my $vmid (keys %$list) {
3131 next if $opt_vmid && ($vmid ne $opt_vmid);
3132 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3138 sub conf_has_serial
{
3141 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3142 if ($conf->{"serial$i"}) {
3150 sub conf_has_audio
{
3151 my ($conf, $id) = @_;
3154 my $audio = $conf->{"audio$id"};
3155 return if !defined($audio);
3157 my $audioproperties = parse_property_string
($audio_fmt, $audio);
3158 my $audiodriver = $audioproperties->{driver
} // 'spice';
3161 dev
=> $audioproperties->{device
},
3162 dev_id
=> "audiodev$id",
3163 backend
=> $audiodriver,
3164 backend_id
=> "$audiodriver-backend${id}",
3169 my ($audio, $audiopciaddr, $machine_version) = @_;
3173 my $id = $audio->{dev_id
};
3175 if (min_version
($machine_version, 4, 2)) {
3176 $audiodev = ",audiodev=$audio->{backend_id}";
3179 if ($audio->{dev
} eq 'AC97') {
3180 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3181 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3182 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3183 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3184 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3186 die "unkown audio device '$audio->{dev}', implement me!";
3189 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3197 socket => "/var/run/qemu-server/$vmid.swtpm",
3198 pid
=> "/var/run/qemu-server/$vmid.swtpm.pid",
3202 sub add_tpm_device
{
3203 my ($vmid, $devices, $conf) = @_;
3205 return if !$conf->{tpmstate0
};
3207 my $paths = get_tpm_paths
($vmid);
3209 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3210 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3211 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3215 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3217 return if !$tpmdrive;
3220 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
3221 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
3223 $state = PVE
::Storage
::map_volume
($storecfg, $tpm->{file
});
3225 $state = $tpm->{file
};
3228 my $paths = get_tpm_paths
($vmid);
3230 # during migration, we will get state from remote
3233 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3240 "--create-platform-cert",
3243 "/etc/swtpm_setup.conf", # do not use XDG configs
3245 "0", # force creation as root, error if not possible
3246 "--not-overwrite", # ignore existing state, do not modify
3249 push @$setup_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3250 # TPM 2.0 supports ECC crypto, use if possible
3251 push @$setup_cmd, "--ecc" if $tpm->{version
} eq 'v2.0';
3253 run_command
($setup_cmd, outfunc
=> sub {
3254 print "swtpm_setup: $1\n";
3258 my $emulator_cmd = [
3262 "backend-uri=file://$state,mode=0600",
3264 "type=unixio,path=$paths->{socket},mode=0600",
3266 "file=$paths->{pid}",
3267 "--terminate", # terminate on QEMU disconnect
3270 push @$emulator_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3271 run_command
($emulator_cmd, outfunc
=> sub { print $1; });
3273 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3274 while (! -e
$paths->{pid
}) {
3275 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3279 # return untainted PID of swtpm daemon so it can be killed on error
3280 file_read_firstline
($paths->{pid
}) =~ m/(\d+)/;
3284 sub vga_conf_has_spice
{
3287 my $vgaconf = parse_vga
($vga);
3288 my $vgatype = $vgaconf->{type
};
3289 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3296 return get_host_arch
() eq $arch;
3301 return $conf->{arch
} // get_host_arch
();
3304 my $default_machines = {
3309 sub get_installed_machine_version
{
3310 my ($kvmversion) = @_;
3311 $kvmversion = kvm_user_version
() if !defined($kvmversion);
3312 $kvmversion =~ m/^(\d+\.\d+)/;
3316 sub windows_get_pinned_machine_version
{
3317 my ($machine, $base_version, $kvmversion) = @_;
3319 my $pin_version = $base_version;
3320 if (!defined($base_version) ||
3321 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3323 $pin_version = get_installed_machine_version
($kvmversion);
3325 if (!$machine || $machine eq 'pc') {
3326 $machine = "pc-i440fx-$pin_version";
3327 } elsif ($machine eq 'q35') {
3328 $machine = "pc-q35-$pin_version";
3329 } elsif ($machine eq 'virt') {
3330 $machine = "virt-$pin_version";
3332 warn "unknown machine type '$machine', not touching that!\n";
3338 sub get_vm_machine
{
3339 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3341 my $machine = $forcemachine || $conf->{machine
};
3343 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3344 $kvmversion //= kvm_user_version
();
3345 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3346 # layout which confuses windows quite a bit and may result in various regressions..
3347 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3348 if (windows_version
($conf->{ostype
})) {
3349 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3352 $machine ||= $default_machines->{$arch};
3353 if ($add_pve_version) {
3354 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3355 $machine .= "+pve$pvever";
3359 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3360 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3361 $machine = $1 if $is_pxe;
3363 # for version-pinned machines that do not include a pve-version (e.g.
3364 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3365 $machine .= '+pve0';
3367 $machine .= '.pxe' if $is_pxe;
3373 sub get_ovmf_files
($$$) {
3374 my ($arch, $efidisk, $smm) = @_;
3376 my $types = $OVMF->{$arch}
3377 or die "no OVMF images known for architecture '$arch'\n";
3379 my $type = 'default';
3380 if (defined($efidisk->{efitype
}) && $efidisk->{efitype
} eq '4m') {
3381 $type = $smm ?
"4m" : "4m-no-smm";
3382 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3385 my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3386 die "EFI base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3387 die "EFI vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
3389 return ($ovmf_code, $ovmf_vars);
3393 aarch64
=> '/usr/bin/qemu-system-aarch64',
3394 x86_64
=> '/usr/bin/qemu-system-x86_64',
3396 sub get_command_for_arch
($) {
3398 return '/usr/bin/kvm' if is_native
($arch);
3400 my $cmd = $Arch2Qemu->{$arch}
3401 or die "don't know how to emulate architecture '$arch'\n";
3405 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3406 # to use in a QEMU command line (-cpu element), first array_intersect the result
3407 # of query_supported_ with query_understood_. This is necessary because:
3409 # a) query_understood_ returns flags the host cannot use and
3410 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3411 # flags, but CPU settings - with most of them being flags. Those settings
3412 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3414 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3415 # expensive. If you need the value returned from this, you can get it much
3416 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3417 # $accel being 'kvm' or 'tcg'.
3419 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3420 # changes, automatically populating pmxcfs.
3422 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3423 # since kvm and tcg machines support different flags
3425 sub query_supported_cpu_flags
{
3428 $arch //= get_host_arch
();
3429 my $default_machine = $default_machines->{$arch};
3433 # FIXME: Once this is merged, the code below should work for ARM as well:
3434 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3435 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3438 my $kvm_supported = defined(kvm_version
());
3439 my $qemu_cmd = get_command_for_arch
($arch);
3441 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3443 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3444 my $query_supported_run_qemu = sub {
3450 '-machine', $default_machine,
3452 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3453 '-mon', 'chardev=qmp,mode=control',
3454 '-pidfile', $pidfile,
3459 push @$cmd, '-accel', 'tcg';
3462 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3463 die "QEMU flag querying VM exited with code " . $rc if $rc;
3466 my $cmd_result = mon_cmd
(
3468 'query-cpu-model-expansion',
3470 model
=> { name
=> 'host' }
3473 my $props = $cmd_result->{model
}->{props
};
3474 foreach my $prop (keys %$props) {
3475 next if $props->{$prop} ne '1';
3476 # QEMU returns some flags multiple times, with '_', '.' or '-'
3477 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3478 # We only keep those with underscores, to match /proc/cpuinfo
3479 $prop =~ s/\.|-/_/g;
3480 $flags->{$prop} = 1;
3485 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3486 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3490 return [ sort keys %$flags ];
3493 # We need to query QEMU twice, since KVM and TCG have different supported flags
3494 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3495 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3496 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3498 if ($kvm_supported) {
3499 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3500 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3507 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3508 my $understood_cpu_flag_dir = "/usr/share/kvm";
3509 sub query_understood_cpu_flags
{
3510 my $arch = get_host_arch
();
3511 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3513 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3516 my $raw = file_get_contents
($filepath);
3517 $raw =~ s/^\s+|\s+$//g;
3518 my @flags = split(/\s+/, $raw);
3523 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3524 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3525 my sub should_disable_smm
{
3526 my ($conf, $vga) = @_;
3528 return (!defined($conf->{bios
}) || $conf->{bios
} eq 'seabios') &&
3529 $vga->{type
} && $vga->{type
} =~ m/^(serial\d+|none)$/;
3532 my sub print_ovmf_drive_commandlines
{
3533 my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3535 my $d = $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
3537 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch, $d, $q35);
3539 my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3541 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3542 my ($path, $format) = $d->@{'file', 'format'};
3544 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3545 if (!defined($format)) {
3546 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3547 $format = qemu_img_format
($scfg, $volname);
3549 } elsif (!defined($format)) {
3550 die "efidisk format must be specified\n";
3552 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3553 if ($path =~ m/^rbd:/) {
3554 $var_drive_str .= ',cache=writeback';
3555 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3557 $var_drive_str .= ",format=$format,file=$path";
3559 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3560 $var_drive_str .= ',readonly=on' if drive_is_read_only
($conf, $d);
3562 log_warn
("no efidisk configured! Using temporary efivars disk.");
3563 my $path = "/tmp/$vmid-ovmf.fd";
3564 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3565 $var_drive_str .= ",format=raw,file=$path";
3566 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $version_guard->(4, 1, 2);
3569 return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3572 sub config_to_command
{
3573 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3576 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3579 my $ostype = $conf->{ostype
};
3580 my $winversion = windows_version
($ostype);
3581 my $kvm = $conf->{kvm
};
3582 my $nodename = nodename
();
3584 my $arch = get_vm_arch
($conf);
3585 my $kvm_binary = get_command_for_arch
($arch);
3586 my $kvmver = kvm_user_version
($kvm_binary);
3588 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3589 $kvmver //= "undefined";
3590 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3593 my $add_pve_version = min_version
($kvmver, 4, 1);
3595 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3596 my $machine_version = extract_version
($machine_type, $kvmver);
3597 $kvm //= 1 if is_native
($arch);
3599 $machine_version =~ m/(\d+)\.(\d+)/;
3600 my ($machine_major, $machine_minor) = ($1, $2);
3602 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3603 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3604 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3605 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3606 ." please upgrade node '$nodename'\n"
3607 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3608 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3609 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3610 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3611 ." node '$nodename'\n";
3614 # if a specific +pve version is required for a feature, use $version_guard
3615 # instead of min_version to allow machines to be run with the minimum
3617 my $required_pve_version = 0;
3618 my $version_guard = sub {
3619 my ($major, $minor, $pve) = @_;
3620 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3621 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3622 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3623 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3627 if ($kvm && !defined kvm_version
()) {
3628 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3629 ." or enable in BIOS.\n";
3632 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3633 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3634 my $use_old_bios_files = undef;
3635 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3638 if ($conf->{affinity
}) {
3639 push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity
};
3642 push @$cmd, $kvm_binary;
3644 push @$cmd, '-id', $vmid;
3646 my $vmname = $conf->{name
} || "vm$vmid";
3648 push @$cmd, '-name', "$vmname,debug-threads=on";
3650 push @$cmd, '-no-shutdown';
3654 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3655 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3656 push @$cmd, '-mon', "chardev=qmp,mode=control";
3658 if (min_version
($machine_version, 2, 12)) {
3659 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3660 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3663 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3665 push @$cmd, '-daemonize';
3667 if ($conf->{smbios1
}) {
3668 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3669 if ($smbios_conf->{base64
}) {
3670 # Do not pass base64 flag to qemu
3671 delete $smbios_conf->{base64
};
3672 my $smbios_string = "";
3673 foreach my $key (keys %$smbios_conf) {
3675 if ($key eq "uuid") {
3676 $value = $smbios_conf->{uuid
}
3678 $value = decode_base64
($smbios_conf->{$key});
3680 # qemu accepts any binary data, only commas need escaping by double comma
3682 $smbios_string .= "," . $key . "=" . $value if $value;
3684 push @$cmd, '-smbios', "type=1" . $smbios_string;
3686 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3690 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3691 my ($code_drive_str, $var_drive_str) =
3692 print_ovmf_drive_commandlines
($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3693 push $cmd->@*, '-drive', $code_drive_str;
3694 push $cmd->@*, '-drive', $var_drive_str;
3697 if ($q35) { # tell QEMU to load q35 config early
3698 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3699 if (min_version
($machine_version, 4, 0)) {
3700 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3702 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3706 if (defined(my $fixups = qemu_created_version_fixups
($conf, $forcemachine, $kvmver))) {
3707 push @$cmd, $fixups->@*;
3710 if ($conf->{vmgenid
}) {
3711 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3714 # add usb controllers
3715 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3716 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES, $machine_version);
3717 push @$devices, @usbcontrollers if @usbcontrollers;
3718 my $vga = parse_vga
($conf->{vga
});
3720 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3721 $vga->{type
} = 'qxl' if $qxlnum;
3723 if (!$vga->{type
}) {
3724 if ($arch eq 'aarch64') {
3725 $vga->{type
} = 'virtio';
3726 } elsif (min_version
($machine_version, 2, 9)) {
3727 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3729 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3733 # enable absolute mouse coordinates (needed by vnc)
3734 my $tablet = $conf->{tablet
};
3735 if (!defined($tablet)) {
3736 $tablet = $defaults->{tablet
};
3737 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3738 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3742 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3743 my $kbd = print_keyboarddevice_full
($conf, $arch);
3744 push @$devices, '-device', $kbd if defined($kbd);
3747 my $bootorder = device_bootorder
($conf);
3749 # host pci device passthrough
3750 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3751 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3754 my $usb_dev_features = {};
3755 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3757 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3758 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3759 push @$devices, @usbdevices if @usbdevices;
3762 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3763 my $path = $conf->{"serial$i"} or next;
3764 if ($path eq 'socket') {
3765 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3766 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3767 # On aarch64, serial0 is the UART device. QEMU only allows
3768 # connecting UART devices via the '-serial' command line, as
3769 # the device has a fixed slot on the hardware...
3770 if ($arch eq 'aarch64' && $i == 0) {
3771 push @$devices, '-serial', "chardev:serial$i";
3773 push @$devices, '-device', "isa-serial,chardev=serial$i";
3776 die "no such serial device\n" if ! -c
$path;
3777 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3778 push @$devices, '-device', "isa-serial,chardev=serial$i";
3783 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3784 if (my $path = $conf->{"parallel$i"}) {
3785 die "no such parallel device\n" if ! -c
$path;
3786 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3787 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3788 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3792 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3793 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3794 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3795 push @$devices, @$audio_devs;
3798 add_tpm_device
($vmid, $devices, $conf);
3801 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3802 $sockets = $conf->{sockets
} if $conf->{sockets
};
3804 my $cores = $conf->{cores
} || 1;
3806 my $maxcpus = $sockets * $cores;
3808 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3810 my $allowed_vcpus = $cpuinfo->{cpus
};
3812 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3814 if ($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3815 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3816 for (my $i = 2; $i <= $vcpus; $i++) {
3817 my $cpustr = print_cpu_device
($conf,$i);
3818 push @$cmd, '-device', $cpustr;
3823 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3825 push @$cmd, '-nodefaults';
3827 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3829 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3831 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3833 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3834 push @$devices, '-device', print_vga_device
(
3835 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3837 push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type
} eq 'virtio-gl'; # VIRGL
3839 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3840 push @$cmd, '-vnc', "unix:$socket,password=on";
3842 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3843 push @$cmd, '-nographic';
3847 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3848 my $useLocaltime = $conf->{localtime};
3850 if ($winversion >= 5) { # windows
3851 $useLocaltime = 1 if !defined($conf->{localtime});
3853 # use time drift fix when acpi is enabled
3854 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3855 $tdf = 1 if !defined($conf->{tdf
});
3859 if ($winversion >= 6) {
3860 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3861 push @$cmd, '-no-hpet';
3864 push @$rtcFlags, 'driftfix=slew' if $tdf;
3866 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3867 push @$rtcFlags, "base=$conf->{startdate}";
3868 } elsif ($useLocaltime) {
3869 push @$rtcFlags, 'base=localtime';
3873 push @$cmd, '-cpu', $forcecpu;
3875 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3878 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3880 push @$cmd, '-S' if $conf->{freeze
};
3882 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3884 my $guest_agent = parse_guest_agent
($conf);
3886 if ($guest_agent->{enabled
}) {
3887 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3888 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3890 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3891 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3892 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3893 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3894 } elsif ($guest_agent->{type
} eq 'isa') {
3895 push @$devices, '-device', "isa-serial,chardev=qga0";
3899 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3900 if ($rng && $version_guard->(4, 1, 2)) {
3901 check_rng_source
($rng->{source
});
3903 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3904 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3905 my $limiter_str = "";
3907 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3910 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3911 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3912 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3917 if ($qxlnum || $vga->{type
} =~ /^virtio/) {
3920 for (my $i = 1; $i < $qxlnum; $i++){
3921 push @$devices, '-device', print_vga_device
(
3922 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3925 # assume other OS works like Linux
3926 my ($ram, $vram) = ("134217728", "67108864");
3927 if ($vga->{memory
}) {
3928 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3929 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3931 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3932 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3936 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3938 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3939 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3940 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3942 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3943 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3944 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3946 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3947 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3949 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3950 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3951 if ($spice_enhancement->{foldersharing
}) {
3952 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3953 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3956 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3957 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3958 if $spice_enhancement->{videostreaming
};
3960 push @$devices, '-spice', "$spice_opts";
3963 # enable balloon by default, unless explicitly disabled
3964 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3965 my $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3966 my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3967 $ballooncmd .= ",free-page-reporting=on" if min_version
($machine_version, 6, 2);
3968 push @$devices, '-device', $ballooncmd;
3971 if ($conf->{watchdog
}) {
3972 my $wdopts = parse_watchdog
($conf->{watchdog
});
3973 my $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3974 my $watchdog = $wdopts->{model
} || 'i6300esb';
3975 push @$devices, '-device', "$watchdog$pciaddr";
3976 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3980 my $scsicontroller = {};
3981 my $ahcicontroller = {};
3982 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3984 # Add iscsi initiator name if available
3985 if (my $initiator = get_initiator_name
()) {
3986 push @$devices, '-iscsi', "initiator-name=$initiator";
3989 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3990 my ($ds, $drive) = @_;
3992 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3993 check_volume_storage_type
($storecfg, $drive->{file
});
3994 push @$vollist, $drive->{file
};
3997 # ignore efidisk here, already added in bios/fw handling code above
3998 return if $drive->{interface
} eq 'efidisk';
4000 return if $drive->{interface
} eq 'tpmstate';
4002 $use_virtio = 1 if $ds =~ m/^virtio/;
4004 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
4006 if ($drive->{interface
} eq 'virtio'){
4007 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
4010 if ($drive->{interface
} eq 'scsi') {
4012 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
4014 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4015 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4017 my $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
4018 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
4021 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
4022 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4023 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4024 } elsif ($drive->{iothread
}) {
4026 "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4031 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
4032 $queues = ",num_queues=$drive->{queues}";
4035 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4036 if !$scsicontroller->{$controller};
4037 $scsicontroller->{$controller}=1;
4040 if ($drive->{interface
} eq 'sata') {
4041 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
4042 my $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
4043 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4044 if !$ahcicontroller->{$controller};
4045 $ahcicontroller->{$controller}=1;
4048 my $pbs_conf = $pbs_backing->{$ds};
4049 my $pbs_name = undef;
4051 $pbs_name = "drive-$ds-pbs";
4052 push @$devices, '-blockdev', print_pbs_blockdev
($pbs_conf, $pbs_name);
4055 my $drive_cmd = print_drive_commandline_full
(
4056 $storecfg, $vmid, $drive, $pbs_name, min_version
($kvmver, 6, 0));
4058 # extra protection for templates, but SATA and IDE don't support it..
4059 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
4061 push @$devices, '-drive',$drive_cmd;
4062 push @$devices, '-device', print_drivedevice_full
(
4063 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4066 for (my $i = 0; $i < $MAX_NETS; $i++) {
4067 my $netname = "net$i";
4069 next if !$conf->{$netname};
4070 my $d = parse_net
($conf->{$netname});
4072 # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4074 $use_virtio = 1 if $d->{model
} eq 'virtio';
4076 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
4078 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
4079 push @$devices, '-netdev', $netdevfull;
4081 my $netdevicefull = print_netdevice_full
(
4082 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4084 push @$devices, '-device', $netdevicefull;
4087 if ($conf->{ivshmem
}) {
4088 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4092 $bus = print_pcie_addr
("ivshmem");
4094 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4097 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4098 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4100 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4101 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4102 .",size=$ivshmem->{size}M";
4105 # pci.4 is nested in pci.1
4106 $bridges->{1} = 1 if $bridges->{4};
4108 if (!$q35) { # add pci bridges
4109 if (min_version
($machine_version, 2, 3)) {
4113 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4116 for my $k (sort {$b cmp $a} keys %$bridges) {
4117 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4120 if ($k == 2 && $legacy_igd) {
4123 my $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
4124 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4126 if ($q35) { # add after -readconfig pve-q35.cfg
4127 splice @$devices, 2, 0, '-device', $devstr;
4129 unshift @$devices, '-device', $devstr if $k > 0;
4134 push @$machineFlags, 'accel=tcg';
4137 push @$machineFlags, 'smm=off' if should_disable_smm
($conf, $vga);
4139 my $machine_type_min = $machine_type;
4140 if ($add_pve_version) {
4141 $machine_type_min =~ s/\+pve\d+$//;
4142 $machine_type_min .= "+pve$required_pve_version";
4144 push @$machineFlags, "type=${machine_type_min}";
4146 push @$cmd, @$devices;
4147 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4148 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4149 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4151 if (my $vmstate = $conf->{vmstate
}) {
4152 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4153 push @$vollist, $vmstate;
4154 push @$cmd, '-loadstate', $statepath;
4155 print "activating and using '$vmstate' as vmstate\n";
4158 if (PVE
::QemuConfig-
>is_template($conf)) {
4159 # needed to workaround base volumes being read-only
4160 push @$cmd, '-snapshot';
4164 if ($conf->{args
}) {
4165 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4169 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
4172 sub check_rng_source
{
4175 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4176 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4179 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4180 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
4181 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4182 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4183 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4184 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4192 my $res = mon_cmd
($vmid, 'query-spice');
4194 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4197 sub vm_devices_list
{
4200 my $res = mon_cmd
($vmid, 'query-pci');
4201 my $devices_to_check = [];
4203 foreach my $pcibus (@$res) {
4204 push @$devices_to_check, @{$pcibus->{devices
}},
4207 while (@$devices_to_check) {
4209 for my $d (@$devices_to_check) {
4210 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4211 next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices
};
4213 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4214 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4216 $devices_to_check = $to_check;
4219 my $resblock = mon_cmd
($vmid, 'query-block');
4220 foreach my $block (@$resblock) {
4221 if($block->{device
} =~ m/^drive-(\S+)/){
4226 my $resmice = mon_cmd
($vmid, 'query-mice');
4227 foreach my $mice (@$resmice) {
4228 if ($mice->{name
} eq 'QEMU HID Tablet') {
4229 $devices->{tablet
} = 1;
4234 # for usb devices there is no query-usb
4235 # but we can iterate over the entries in
4236 # qom-list path=/machine/peripheral
4237 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4238 foreach my $per (@$resperipheral) {
4239 if ($per->{name
} =~ m/^usb(?:redirdev)?\d+$/) {
4240 $devices->{$per->{name
}} = 1;
4248 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4250 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4252 my $devices_list = vm_devices_list
($vmid);
4253 return 1 if defined($devices_list->{$deviceid});
4255 # add PCI bridge if we need it for the device
4256 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4258 if ($deviceid eq 'tablet') {
4259 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4260 } elsif ($deviceid eq 'keyboard') {
4261 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4262 } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4264 qemu_spice_usbredir_chardev_add
($vmid, "usbredirchardev$id");
4265 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_spice_usbdevice
($id, "xhci", $id + 1));
4266 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4267 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device, {}, $1 + 1));
4268 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4269 qemu_iothread_add
($vmid, $deviceid, $device);
4271 qemu_driveadd
($storecfg, $vmid, $device);
4272 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4274 qemu_deviceadd
($vmid, $devicefull);
4275 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4277 eval { qemu_drivedel
($vmid, $deviceid); };
4281 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4282 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4283 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4284 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4286 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4288 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4289 qemu_iothread_add
($vmid, $deviceid, $device);
4290 $devicefull .= ",iothread=iothread-$deviceid";
4293 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4294 $devicefull .= ",num_queues=$device->{queues}";
4297 qemu_deviceadd
($vmid, $devicefull);
4298 qemu_deviceaddverify
($vmid, $deviceid);
4299 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4300 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4301 qemu_driveadd
($storecfg, $vmid, $device);
4303 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4304 eval { qemu_deviceadd
($vmid, $devicefull); };
4306 eval { qemu_drivedel
($vmid, $deviceid); };
4310 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4311 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4313 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4314 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type);
4315 my $use_old_bios_files = undef;
4316 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4318 my $netdevicefull = print_netdevice_full
(
4319 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4320 qemu_deviceadd
($vmid, $netdevicefull);
4322 qemu_deviceaddverify
($vmid, $deviceid);
4323 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4326 eval { qemu_netdevdel
($vmid, $deviceid); };
4330 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4332 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4333 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4335 qemu_deviceadd
($vmid, $devicefull);
4336 qemu_deviceaddverify
($vmid, $deviceid);
4338 die "can't hotplug device '$deviceid'\n";
4344 # fixme: this should raise exceptions on error!
4345 sub vm_deviceunplug
{
4346 my ($vmid, $conf, $deviceid) = @_;
4348 my $devices_list = vm_devices_list
($vmid);
4349 return 1 if !defined($devices_list->{$deviceid});
4351 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4352 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4354 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4355 qemu_devicedel
($vmid, $deviceid);
4356 } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4357 qemu_devicedel
($vmid, $deviceid);
4358 qemu_devicedelverify
($vmid, $deviceid);
4359 } elsif ($deviceid =~ m/^usb\d+$/) {
4360 qemu_devicedel
($vmid, $deviceid);
4361 qemu_devicedelverify
($vmid, $deviceid);
4362 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4363 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4365 qemu_devicedel
($vmid, $deviceid);
4366 qemu_devicedelverify
($vmid, $deviceid);
4367 qemu_drivedel
($vmid, $deviceid);
4368 qemu_iothread_del
($vmid, $deviceid, $device);
4369 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4370 qemu_devicedel
($vmid, $deviceid);
4371 qemu_devicedelverify
($vmid, $deviceid);
4372 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4373 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4375 qemu_devicedel
($vmid, $deviceid);
4376 qemu_devicedelverify
($vmid, $deviceid);
4377 qemu_drivedel
($vmid, $deviceid);
4378 qemu_deletescsihw
($conf, $vmid, $deviceid);
4380 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4381 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4382 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4383 qemu_devicedel
($vmid, $deviceid);
4384 qemu_devicedelverify
($vmid, $deviceid);
4385 qemu_netdevdel
($vmid, $deviceid);
4387 die "can't unplug device '$deviceid'\n";
4393 sub qemu_spice_usbredir_chardev_add
{
4394 my ($vmid, $id) = @_;
4396 mon_cmd
($vmid, "chardev-add" , (
4407 sub qemu_deviceadd
{
4408 my ($vmid, $devicefull) = @_;
4410 $devicefull = "driver=".$devicefull;
4411 my %options = split(/[=,]/, $devicefull);
4413 mon_cmd
($vmid, "device_add" , %options);
4416 sub qemu_devicedel
{
4417 my ($vmid, $deviceid) = @_;
4419 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4422 sub qemu_iothread_add
{
4423 my ($vmid, $deviceid, $device) = @_;
4425 if ($device->{iothread
}) {
4426 my $iothreads = vm_iothreads_list
($vmid);
4427 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4431 sub qemu_iothread_del
{
4432 my ($vmid, $deviceid, $device) = @_;
4434 if ($device->{iothread
}) {
4435 my $iothreads = vm_iothreads_list
($vmid);
4436 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4440 sub qemu_objectadd
{
4441 my ($vmid, $objectid, $qomtype) = @_;
4443 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4448 sub qemu_objectdel
{
4449 my ($vmid, $objectid) = @_;
4451 mon_cmd
($vmid, "object-del", id
=> $objectid);
4457 my ($storecfg, $vmid, $device) = @_;
4459 my $kvmver = get_running_qemu_version
($vmid);
4460 my $io_uring = min_version
($kvmver, 6, 0);
4461 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4462 $drive =~ s/\\/\\\\/g;
4463 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4465 # If the command succeeds qemu prints: "OK
"
4466 return 1 if $ret =~ m/OK/s;
4468 die "adding drive failed
: $ret\n";
4472 my ($vmid, $deviceid) = @_;
4474 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4477 return 1 if $ret eq "";
4479 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4480 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4482 die "deleting drive
$deviceid failed
: $ret\n";
4485 sub qemu_deviceaddverify {
4486 my ($vmid, $deviceid) = @_;
4488 for (my $i = 0; $i <= 5; $i++) {
4489 my $devices_list = vm_devices_list($vmid);
4490 return 1 if defined($devices_list->{$deviceid});
4494 die "error on hotplug device
'$deviceid'\n";
4498 sub qemu_devicedelverify {
4499 my ($vmid, $deviceid) = @_;
4501 # need to verify that the device is correctly removed as device_del
4502 # is async and empty return is not reliable
4504 for (my $i = 0; $i <= 5; $i++) {
4505 my $devices_list = vm_devices_list($vmid);
4506 return 1 if !defined($devices_list->{$deviceid});
4510 die "error on hot-unplugging device
'$deviceid'\n";
4513 sub qemu_findorcreatescsihw {
4514 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4516 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4518 my $scsihwid="$controller_prefix$controller";
4519 my $devices_list = vm_devices_list($vmid);
4521 if (!defined($devices_list->{$scsihwid})) {
4522 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4528 sub qemu_deletescsihw {
4529 my ($conf, $vmid, $opt) = @_;
4531 my $device = parse_drive($opt, $conf->{$opt});
4533 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4534 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4538 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4540 my $devices_list = vm_devices_list($vmid);
4541 foreach my $opt (keys %{$devices_list}) {
4542 if (is_valid_drivename($opt)) {
4543 my $drive = parse_drive($opt, $conf->{$opt});
4544 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4550 my $scsihwid="scsihw
$controller";
4552 vm_deviceunplug($vmid, $conf, $scsihwid);
4557 sub qemu_add_pci_bridge {
4558 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4564 print_pci_addr($device, $bridges, $arch, $machine_type);
4566 while (my ($k, $v) = each %$bridges) {
4569 return 1 if !defined($bridgeid) || $bridgeid < 1;
4571 my $bridge = "pci
.$bridgeid";
4572 my $devices_list = vm_devices_list($vmid);
4574 if (!defined($devices_list->{$bridge})) {
4575 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4581 sub qemu_set_link_status {
4582 my ($vmid, $device, $up) = @_;
4584 mon_cmd($vmid, "set_link
", name => $device,
4585 up => $up ? JSON::true : JSON::false);
4588 sub qemu_netdevadd {
4589 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4591 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4592 my %options = split(/[=,]/, $netdev);
4594 if (defined(my $vhost = $options{vhost})) {
4595 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4598 if (defined(my $queues = $options{queues})) {
4599 $options{queues} = $queues + 0;
4602 mon_cmd($vmid, "netdev_add
", %options);
4606 sub qemu_netdevdel {
4607 my ($vmid, $deviceid) = @_;
4609 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4612 sub qemu_usb_hotplug {
4613 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4617 # remove the old one first
4618 vm_deviceunplug($vmid, $conf, $deviceid);
4620 # check if xhci controller is necessary and available
4621 my $devicelist = vm_devices_list($vmid);
4623 if (!$devicelist->{xhci}) {
4624 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4625 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4628 # print_usbdevice_full expects the parsed device
4629 my $d = parse_usb_device($device->{host});
4630 $d->{usb3} = $device->{usb3};
4633 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4636 sub qemu_cpu_hotplug {
4637 my ($vmid, $conf, $vcpus) = @_;
4639 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4642 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4643 $sockets = $conf->{sockets} if $conf->{sockets};
4644 my $cores = $conf->{cores} || 1;
4645 my $maxcpus = $sockets * $cores;
4647 $vcpus = $maxcpus if !$vcpus;
4649 die "you can
't add more vcpus than maxcpus\n"
4650 if $vcpus > $maxcpus;
4652 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4654 if ($vcpus < $currentvcpus) {
4656 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4658 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4659 qemu_devicedel($vmid, "cpu$i");
4661 my $currentrunningvcpus = undef;
4663 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4664 last if scalar(@{$currentrunningvcpus}) == $i-1;
4665 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4669 #update conf after each succesfull cpu unplug
4670 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4671 PVE::QemuConfig->write_config($vmid, $conf);
4674 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4680 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4681 die "vcpus in running vm does not match its configuration\n"
4682 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4684 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4686 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4687 my $cpustr = print_cpu_device($conf, $i);
4688 qemu_deviceadd($vmid, $cpustr);
4691 my $currentrunningvcpus = undef;
4693 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4694 last if scalar(@{$currentrunningvcpus}) == $i;
4695 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4699 #update conf after each succesfull cpu hotplug
4700 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4701 PVE::QemuConfig->write_config($vmid, $conf);
4705 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4706 mon_cmd($vmid, "cpu-add", id => int($i));
4711 sub qemu_block_set_io_throttle {
4712 my ($vmid, $deviceid,
4713 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4714 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4715 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4716 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4718 return if !check_running($vmid) ;
4720 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4722 bps_rd => int($bps_rd),
4723 bps_wr => int($bps_wr),
4725 iops_rd => int($iops_rd),
4726 iops_wr => int($iops_wr),
4727 bps_max => int($bps_max),
4728 bps_rd_max => int($bps_rd_max),
4729 bps_wr_max => int($bps_wr_max),
4730 iops_max => int($iops_max),
4731 iops_rd_max => int($iops_rd_max),
4732 iops_wr_max => int($iops_wr_max),
4733 bps_max_length => int($bps_max_length),
4734 bps_rd_max_length => int($bps_rd_max_length),
4735 bps_wr_max_length => int($bps_wr_max_length),
4736 iops_max_length => int($iops_max_length),
4737 iops_rd_max_length => int($iops_rd_max_length),
4738 iops_wr_max_length => int($iops_wr_max_length),
4743 sub qemu_block_resize {
4744 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4746 my $running = check_running($vmid);
4748 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4750 return if !$running;
4752 my $padding = (1024 - $size % 1024) % 1024;
4753 $size = $size + $padding;
4758 device => $deviceid,
4764 sub qemu_volume_snapshot {
4765 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4767 my $running = check_running($vmid);
4769 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4770 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4772 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4776 sub qemu_volume_snapshot_delete {
4777 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4779 my $running = check_running($vmid);
4784 my $conf = PVE::QemuConfig->load_config($vmid);
4785 PVE::QemuConfig->foreach_volume($conf, sub {
4786 my ($ds, $drive) = @_;
4787 $running = 1 if $drive->{file} eq $volid;
4791 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4792 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4794 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4798 sub set_migration_caps {
4799 my ($vmid, $savevm) = @_;
4801 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4803 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4804 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4809 "auto-converge" => 1,
4811 "x-rdma-pin-all" => 0,
4814 "dirty-bitmaps" => $dirty_bitmaps,
4817 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4819 for my $supported_capability (@$supported_capabilities) {
4821 capability => $supported_capability->{capability},
4822 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4826 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4830 my ($conf, $func, @param) = @_;
4834 my $test_volid = sub {
4835 my ($key, $drive, $snapname) = @_;
4837 my $volid = $drive->{file};
4840 $volhash->{$volid}->{cdrom} //= 1;
4841 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4843 my $replicate = $drive->{replicate} // 1;
4844 $volhash->{$volid}->{replicate} //= 0;
4845 $volhash->{$volid}->{replicate} = 1 if $replicate;
4847 $volhash->{$volid}->{shared} //= 0;
4848 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4850 $volhash->{$volid}->{referenced_in_config} //= 0;
4851 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4853 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4854 if defined($snapname);
4856 my $size = $drive->{size};
4857 $volhash->{$volid}->{size} //= $size if $size;
4859 $volhash->{$volid}->{is_vmstate} //= 0;
4860 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4862 $volhash->{$volid}->{is_tpmstate} //= 0;
4863 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0
';
4865 $volhash->{$volid}->{is_unused} //= 0;
4866 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4868 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4871 my $include_opts = {
4872 extra_keys => ['vmstate
'],
4873 include_unused => 1,
4876 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4877 foreach my $snapname (keys %{$conf->{snapshots}}) {
4878 my $snap = $conf->{snapshots}->{$snapname};
4879 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4882 foreach my $volid (keys %$volhash) {
4883 &$func($volid, $volhash->{$volid}, @param);
4887 my $fast_plug_option = {
4895 'vmstatestorage
' => 1,
4900 for my $opt (keys %$confdesc_cloudinit) {
4901 $fast_plug_option->{$opt} = 1;
4904 # hotplug changes in [PENDING]
4905 # $selection hash can be used to only apply specified options, for
4906 # example: { cores => 1 } (only apply changed 'cores
')
4907 # $errors ref is used to return error messages
4908 sub vmconfig_hotplug_pending {
4909 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4911 my $defaults = load_defaults();
4912 my $arch = get_vm_arch($conf);
4913 my $machine_type = get_vm_machine($conf, undef, $arch);
4915 # commit values which do not have any impact on running VM first
4916 # Note: those option cannot raise errors, we we do not care about
4917 # $selection and always apply them.
4919 my $add_error = sub {
4920 my ($opt, $msg) = @_;
4921 $errors->{$opt} = "hotplug problem - $msg";
4924 my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4926 my $cloudinit_record_changed = sub {
4927 my ($conf, $opt, $old, $new) = @_;
4928 return if !$cloudinit_pending_properties->{$opt};
4930 my $ci = ($conf->{cloudinit} //= {});
4932 my $recorded = $ci->{$opt};
4933 my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4935 if (defined($new)) {
4936 if (defined($old)) {
4937 # an existing value is being modified
4938 if (defined($recorded)) {
4939 # the value was already not in sync
4940 if ($new eq $recorded) {
4941 # a value is being reverted to the cloud-init state:
4943 delete $added{$opt};
4945 # the value was changed multiple times, do nothing
4947 } elsif ($added{$opt}) {
4948 # the value had been marked as added and is being changed, do nothing
4950 # the value is new, record it:
4954 # a new value is being added
4955 if (defined($recorded)) {
4956 # it was already not in sync
4957 if ($new eq $recorded) {
4958 # a value is being reverted to the cloud-init state:
4960 delete $added{$opt};
4962 # the value had temporarily been removed, do nothing
4964 } elsif ($added{$opt}) {
4965 # the value had been marked as added already, do nothing
4967 # the value is new, add it
4971 } elsif (!defined($old)) {
4972 # a non-existent value is being removed? ignore...
4974 # a value is being deleted
4975 if (defined($recorded)) {
4976 # a value was already recorded, just keep it
4977 } elsif ($added{$opt}) {
4978 # the value was marked as added, remove it
4979 delete $added{$opt};
4981 # a previously unrecorded value is being removed, record the old value:
4986 my $added = join(',', sort keys %added);
4987 $ci->{added} = $added if length($added);
4991 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4992 if ($fast_plug_option->{$opt}) {
4993 my $new = delete $conf->{pending}->{$opt};
4994 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4995 $conf->{$opt} = $new;
5001 PVE::QemuConfig->write_config($vmid, $conf);
5004 my $ostype = $conf->{ostype};
5005 my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5006 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5007 my $usb_hotplug = $hotplug_features->{usb}
5008 && min_version($version, 7, 1)
5009 && defined($ostype) && ($ostype eq 'l26
' || windows_version($ostype) > 7);
5011 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5012 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5014 foreach my $opt (sort keys %$pending_delete_hash) {
5015 next if $selection && !$selection->{$opt};
5016 my $force = $pending_delete_hash->{$opt}->{force};
5018 if ($opt eq 'hotplug
') {
5019 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5020 } elsif ($opt eq 'tablet
') {
5021 die "skip\n" if !$hotplug_features->{usb};
5022 if ($defaults->{tablet}) {
5023 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
5024 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
5025 if $arch eq 'aarch64
';
5027 vm_deviceunplug($vmid, $conf, 'tablet
');
5028 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
5030 } elsif ($opt =~ m/^usb(\d+)$/) {
5032 die "skip\n" if !$usb_hotplug;
5033 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5034 vm_deviceunplug
($vmid, $conf, $opt);
5035 } elsif ($opt eq 'vcpus') {
5036 die "skip\n" if !$hotplug_features->{cpu
};
5037 qemu_cpu_hotplug
($vmid, $conf, undef);
5038 } elsif ($opt eq 'balloon') {
5039 # enable balloon device is not hotpluggable
5040 die "skip\n" if defined($conf->{balloon
}) && $conf->{balloon
} == 0;
5041 # here we reset the ballooning value to memory
5042 my $balloon = $conf->{memory
} || $defaults->{memory
};
5043 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5044 } elsif ($fast_plug_option->{$opt}) {
5046 } elsif ($opt =~ m/^net(\d+)$/) {
5047 die "skip\n" if !$hotplug_features->{network
};
5048 vm_deviceunplug
($vmid, $conf, $opt);
5049 } elsif (is_valid_drivename
($opt)) {
5050 die "skip\n" if !$hotplug_features->{disk
} || $opt =~ m/(ide|sata)(\d+)/;
5051 vm_deviceunplug
($vmid, $conf, $opt);
5052 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5053 } elsif ($opt =~ m/^memory$/) {
5054 die "skip\n" if !$hotplug_features->{memory
};
5055 PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $defaults);
5056 } elsif ($opt eq 'cpuunits') {
5057 $cgroup->change_cpu_shares(undef);
5058 } elsif ($opt eq 'cpulimit') {
5059 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5065 &$add_error($opt, $err) if $err ne "skip\n";
5067 my $old = delete $conf->{$opt};
5068 $cloudinit_record_changed->($conf, $opt, $old, undef);
5069 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5074 foreach my $opt (keys %{$conf->{pending
}}) {
5075 next if $selection && !$selection->{$opt};
5076 my $value = $conf->{pending
}->{$opt};
5078 if ($opt eq 'hotplug') {
5079 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug
} =~ /memory/);
5080 } elsif ($opt eq 'tablet') {
5081 die "skip\n" if !$hotplug_features->{usb
};
5083 vm_deviceplug
($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5084 vm_deviceplug
($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5085 if $arch eq 'aarch64';
5086 } elsif ($value == 0) {
5087 vm_deviceunplug
($vmid, $conf, 'tablet');
5088 vm_deviceunplug
($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5090 } elsif ($opt =~ m/^usb(\d+)$/) {
5092 die "skip\n" if !$usb_hotplug;
5093 my $d = eval { parse_property_string
($usbdesc->{format
}, $value) };
5095 if ($d->{host
} eq 'spice') {
5096 $id = "usbredirdev$index";
5098 qemu_usb_hotplug
($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5099 } elsif ($opt eq 'vcpus') {
5100 die "skip\n" if !$hotplug_features->{cpu
};
5101 qemu_cpu_hotplug
($vmid, $conf, $value);
5102 } elsif ($opt eq 'balloon') {
5103 # enable/disable balloning device is not hotpluggable
5104 my $old_balloon_enabled = !!(!defined($conf->{balloon
}) || $conf->{balloon
});
5105 my $new_balloon_enabled = !!(!defined($conf->{pending
}->{balloon
}) || $conf->{pending
}->{balloon
});
5106 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5108 # allow manual ballooning if shares is set to zero
5109 if ((defined($conf->{shares
}) && ($conf->{shares
} == 0))) {
5110 my $balloon = $conf->{pending
}->{balloon
} || $conf->{memory
} || $defaults->{memory
};
5111 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5113 } elsif ($opt =~ m/^net(\d+)$/) {
5114 # some changes can be done without hotplug
5115 vmconfig_update_net
($storecfg, $conf, $hotplug_features->{network
},
5116 $vmid, $opt, $value, $arch, $machine_type);
5117 } elsif (is_valid_drivename
($opt)) {
5118 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5119 # some changes can be done without hotplug
5120 my $drive = parse_drive
($opt, $value);
5121 if (drive_is_cloudinit
($drive)) {
5122 $cloudinit_opt = [$opt, $drive];
5123 # apply all the other changes first, then generate the cloudinit disk
5126 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5127 $vmid, $opt, $value, $arch, $machine_type);
5128 } elsif ($opt =~ m/^memory$/) { #dimms
5129 die "skip\n" if !$hotplug_features->{memory
};
5130 $value = PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $defaults, $value);
5131 } elsif ($opt eq 'cpuunits') {
5132 my $new_cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{pending
}->{$opt}); #clamp
5133 $cgroup->change_cpu_shares($new_cpuunits);
5134 } elsif ($opt eq 'cpulimit') {
5135 my $cpulimit = $conf->{pending
}->{$opt} == 0 ?
-1 : int($conf->{pending
}->{$opt} * 100000);
5136 $cgroup->change_cpu_quota($cpulimit, 100000);
5137 } elsif ($opt eq 'agent') {
5138 vmconfig_update_agent
($conf, $opt, $value);
5140 die "skip\n"; # skip non-hot-pluggable options
5144 &$add_error($opt, $err) if $err ne "skip\n";
5146 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5147 $conf->{$opt} = $value;
5148 delete $conf->{pending
}->{$opt};
5152 if (defined($cloudinit_opt)) {
5153 my ($opt, $drive) = @$cloudinit_opt;
5154 my $value = $conf->{pending
}->{$opt};
5156 my $temp = {%$conf, $opt => $value};
5157 PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($temp, $vmid);
5158 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5159 $vmid, $opt, $value, $arch, $machine_type);
5162 &$add_error($opt, $err) if $err ne "skip\n";
5164 $conf->{$opt} = $value;
5165 delete $conf->{pending
}->{$opt};
5169 # unplug xhci controller if no usb device is left
5172 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5173 next if !defined($conf->{"usb$i"});
5178 vm_deviceunplug
($vmid, $conf, 'xhci');
5182 PVE
::QemuConfig-
>write_config($vmid, $conf);
5184 if ($hotplug_features->{cloudinit
} && PVE
::QemuServer
::Cloudinit
::has_changes
($conf)) {
5185 PVE
::QemuServer
::vmconfig_update_cloudinit_drive
($storecfg, $conf, $vmid);
5189 sub try_deallocate_drive
{
5190 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5192 if (($force || $key =~ /^unused/) && !drive_is_cdrom
($drive, 1)) {
5193 my $volid = $drive->{file
};
5194 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
5195 my $sid = PVE
::Storage
::parse_volume_id
($volid);
5196 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5198 # check if the disk is really unused
5199 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5200 if PVE
::QemuServer
::Drive
::is_volume_in_use
($storecfg, $conf, $key, $volid);
5201 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5204 # If vm is not owner of this disk remove from config
5212 sub vmconfig_delete_or_detach_drive
{
5213 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5215 my $drive = parse_drive
($opt, $conf->{$opt});
5217 my $rpcenv = PVE
::RPCEnvironment
::get
();
5218 my $authuser = $rpcenv->get_user();
5221 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5222 try_deallocate_drive
($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5224 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $drive);
5230 sub vmconfig_apply_pending
{
5231 my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5233 return if !scalar(keys %{$conf->{pending
}});
5235 my $add_apply_error = sub {
5236 my ($opt, $msg) = @_;
5237 my $err_msg = "unable to apply pending change $opt : $msg";
5238 $errors->{$opt} = $err_msg;
5244 my $pending_delete_hash = PVE
::QemuConfig-
>parse_pending_delete($conf->{pending
}->{delete});
5245 foreach my $opt (sort keys %$pending_delete_hash) {
5246 my $force = $pending_delete_hash->{$opt}->{force
};
5248 if ($opt =~ m/^unused/) {
5249 die "internal error";
5250 } elsif (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5251 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5255 $add_apply_error->($opt, $err);
5257 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5258 delete $conf->{$opt};
5262 PVE
::QemuConfig-
>cleanup_pending($conf);
5264 my $generate_cloudinit = $skip_cloud_init ?
0 : undef;
5266 foreach my $opt (keys %{$conf->{pending
}}) { # add/change
5267 next if $opt eq 'delete'; # just to be sure
5269 if (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5270 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, parse_drive
($opt, $conf->{$opt}))
5274 $add_apply_error->($opt, $err);
5277 if (is_valid_drivename
($opt)) {
5278 my $drive = parse_drive
($opt, $conf->{pending
}->{$opt});
5279 $generate_cloudinit //= 1 if drive_is_cloudinit
($drive);
5282 $conf->{$opt} = delete $conf->{pending
}->{$opt};
5286 # write all changes at once to avoid unnecessary i/o
5287 PVE
::QemuConfig-
>write_config($vmid, $conf);
5288 if ($generate_cloudinit) {
5289 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5290 # After successful generation and if there were changes to be applied, update the
5291 # config to drop the {cloudinit} entry.
5292 PVE
::QemuConfig-
>write_config($vmid, $conf);
5297 sub vmconfig_update_net
{
5298 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5300 my $newnet = parse_net
($value);
5302 if ($conf->{$opt}) {
5303 my $oldnet = parse_net
($conf->{$opt});
5305 if (safe_string_ne
($oldnet->{model
}, $newnet->{model
}) ||
5306 safe_string_ne
($oldnet->{macaddr
}, $newnet->{macaddr
}) ||
5307 safe_num_ne
($oldnet->{queues
}, $newnet->{queues
}) ||
5308 !($newnet->{bridge
} && $oldnet->{bridge
})) { # bridge/nat mode change
5310 # for non online change, we try to hot-unplug
5311 die "skip\n" if !$hotplug;
5312 vm_deviceunplug
($vmid, $conf, $opt);
5315 die "internal error" if $opt !~ m/net(\d+)/;
5316 my $iface = "tap${vmid}i$1";
5318 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5319 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
}) ||
5320 safe_string_ne
($oldnet->{trunks
}, $newnet->{trunks
}) ||
5321 safe_num_ne
($oldnet->{firewall
}, $newnet->{firewall
})) {
5322 PVE
::Network
::tap_unplug
($iface);
5325 PVE
::Network
::SDN
::Zones
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5327 PVE
::Network
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5329 } elsif (safe_num_ne
($oldnet->{rate
}, $newnet->{rate
})) {
5330 # Rate can be applied on its own but any change above needs to
5331 # include the rate in tap_plug since OVS resets everything.
5332 PVE
::Network
::tap_rate_limit
($iface, $newnet->{rate
});
5335 if (safe_string_ne
($oldnet->{link_down
}, $newnet->{link_down
})) {
5336 qemu_set_link_status
($vmid, $opt, !$newnet->{link_down
});
5344 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5350 sub vmconfig_update_agent
{
5351 my ($conf, $opt, $value) = @_;
5353 die "skip\n" if !$conf->{$opt};
5355 my $hotplug_options = { fstrim_cloned_disks
=> 1 };
5357 my $old_agent = parse_guest_agent
($conf);
5358 my $agent = parse_guest_agent
({$opt => $value});
5360 for my $option (keys %$agent) { # added/changed options
5361 next if defined($hotplug_options->{$option});
5362 die "skip\n" if safe_string_ne
($agent->{$option}, $old_agent->{$option});
5365 for my $option (keys %$old_agent) { # removed options
5366 next if defined($hotplug_options->{$option});
5367 die "skip\n" if safe_string_ne
($old_agent->{$option}, $agent->{$option});
5370 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5373 sub vmconfig_update_disk
{
5374 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5376 my $drive = parse_drive
($opt, $value);
5378 if ($conf->{$opt} && (my $old_drive = parse_drive
($opt, $conf->{$opt}))) {
5379 my $media = $drive->{media
} || 'disk';
5380 my $oldmedia = $old_drive->{media
} || 'disk';
5381 die "unable to change media type\n" if $media ne $oldmedia;
5383 if (!drive_is_cdrom
($old_drive)) {
5385 if ($drive->{file
} ne $old_drive->{file
}) {
5387 die "skip\n" if !$hotplug;
5389 # unplug and register as unused
5390 vm_deviceunplug
($vmid, $conf, $opt);
5391 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive)
5394 # update existing disk
5396 # skip non hotpluggable value
5397 if (safe_string_ne
($drive->{aio
}, $old_drive->{aio
}) ||
5398 safe_string_ne
($drive->{discard
}, $old_drive->{discard
}) ||
5399 safe_string_ne
($drive->{iothread
}, $old_drive->{iothread
}) ||
5400 safe_string_ne
($drive->{queues
}, $old_drive->{queues
}) ||
5401 safe_string_ne
($drive->{cache
}, $old_drive->{cache
}) ||
5402 safe_string_ne
($drive->{ssd
}, $old_drive->{ssd
}) ||
5403 safe_string_ne
($drive->{ro
}, $old_drive->{ro
})) {
5408 if (safe_num_ne
($drive->{mbps
}, $old_drive->{mbps
}) ||
5409 safe_num_ne
($drive->{mbps_rd
}, $old_drive->{mbps_rd
}) ||
5410 safe_num_ne
($drive->{mbps_wr
}, $old_drive->{mbps_wr
}) ||
5411 safe_num_ne
($drive->{iops
}, $old_drive->{iops
}) ||
5412 safe_num_ne
($drive->{iops_rd
}, $old_drive->{iops_rd
}) ||
5413 safe_num_ne
($drive->{iops_wr
}, $old_drive->{iops_wr
}) ||
5414 safe_num_ne
($drive->{mbps_max
}, $old_drive->{mbps_max
}) ||
5415 safe_num_ne
($drive->{mbps_rd_max
}, $old_drive->{mbps_rd_max
}) ||
5416 safe_num_ne
($drive->{mbps_wr_max
}, $old_drive->{mbps_wr_max
}) ||
5417 safe_num_ne
($drive->{iops_max
}, $old_drive->{iops_max
}) ||
5418 safe_num_ne
($drive->{iops_rd_max
}, $old_drive->{iops_rd_max
}) ||
5419 safe_num_ne
($drive->{iops_wr_max
}, $old_drive->{iops_wr_max
}) ||
5420 safe_num_ne
($drive->{bps_max_length
}, $old_drive->{bps_max_length
}) ||
5421 safe_num_ne
($drive->{bps_rd_max_length
}, $old_drive->{bps_rd_max_length
}) ||
5422 safe_num_ne
($drive->{bps_wr_max_length
}, $old_drive->{bps_wr_max_length
}) ||
5423 safe_num_ne
($drive->{iops_max_length
}, $old_drive->{iops_max_length
}) ||
5424 safe_num_ne
($drive->{iops_rd_max_length
}, $old_drive->{iops_rd_max_length
}) ||
5425 safe_num_ne
($drive->{iops_wr_max_length
}, $old_drive->{iops_wr_max_length
})) {
5427 qemu_block_set_io_throttle
(
5429 ($drive->{mbps
} || 0)*1024*1024,
5430 ($drive->{mbps_rd
} || 0)*1024*1024,
5431 ($drive->{mbps_wr
} || 0)*1024*1024,
5432 $drive->{iops
} || 0,
5433 $drive->{iops_rd
} || 0,
5434 $drive->{iops_wr
} || 0,
5435 ($drive->{mbps_max
} || 0)*1024*1024,
5436 ($drive->{mbps_rd_max
} || 0)*1024*1024,
5437 ($drive->{mbps_wr_max
} || 0)*1024*1024,
5438 $drive->{iops_max
} || 0,
5439 $drive->{iops_rd_max
} || 0,
5440 $drive->{iops_wr_max
} || 0,
5441 $drive->{bps_max_length
} || 1,
5442 $drive->{bps_rd_max_length
} || 1,
5443 $drive->{bps_wr_max_length
} || 1,
5444 $drive->{iops_max_length
} || 1,
5445 $drive->{iops_rd_max_length
} || 1,
5446 $drive->{iops_wr_max_length
} || 1,
5456 if ($drive->{file
} eq 'none') {
5457 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5458 if (drive_is_cloudinit
($old_drive)) {
5459 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive);
5462 my $path = get_iso_path
($storecfg, $vmid, $drive->{file
});
5464 # force eject if locked
5465 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5468 mon_cmd
($vmid, "blockdev-change-medium",
5469 id
=> "$opt", filename
=> "$path");
5477 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5479 PVE
::Storage
::activate_volumes
($storecfg, [$drive->{file
}]) if $drive->{file
} !~ m
|^/dev/.+|;
5480 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5483 sub vmconfig_update_cloudinit_drive
{
5484 my ($storecfg, $conf, $vmid) = @_;
5486 my $cloudinit_ds = undef;
5487 my $cloudinit_drive = undef;
5489 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5490 my ($ds, $drive) = @_;
5491 if (PVE
::QemuServer
::drive_is_cloudinit
($drive)) {
5492 $cloudinit_ds = $ds;
5493 $cloudinit_drive = $drive;
5497 return if !$cloudinit_drive;
5499 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5500 PVE
::QemuConfig-
>write_config($vmid, $conf);
5503 my $running = PVE
::QemuServer
::check_running
($vmid);
5506 my $path = PVE
::Storage
::path
($storecfg, $cloudinit_drive->{file
});
5508 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$cloudinit_ds");
5509 mon_cmd
($vmid, "blockdev-change-medium", id
=> "$cloudinit_ds", filename
=> "$path");
5514 # called in locked context by incoming migration
5515 sub vm_migrate_get_nbd_disks
{
5516 my ($storecfg, $conf, $replicated_volumes) = @_;
5518 my $local_volumes = {};
5519 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5520 my ($ds, $drive) = @_;
5522 return if drive_is_cdrom
($drive);
5523 return if $ds eq 'tpmstate0';
5525 my $volid = $drive->{file
};
5529 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid);
5531 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5532 return if $scfg->{shared
};
5534 # replicated disks re-use existing state via bitmap
5535 my $use_existing = $replicated_volumes->{$volid} ?
1 : 0;
5536 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5538 return $local_volumes;
5541 # called in locked context by incoming migration
5542 sub vm_migrate_alloc_nbd_disks
{
5543 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5546 foreach my $opt (sort keys %$source_volumes) {
5547 my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5549 if ($use_existing) {
5550 $nbd->{$opt}->{drivestr
} = print_drive
($drive);
5551 $nbd->{$opt}->{volid
} = $volid;
5552 $nbd->{$opt}->{replicated
} = 1;
5556 # storage mapping + volname = regular migration
5557 # storage mapping + format = remote migration
5558 # order of precedence, filtered by whether storage supports it:
5559 # 1. explicit requested format
5560 # 2. format of current volume
5561 # 3. default format of storage
5562 if (!$storagemap->{identity
}) {
5563 $storeid = PVE
::JSONSchema
::map_id
($storagemap, $storeid);
5564 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5565 if (!$format || !grep { $format eq $_ } @$validFormats) {
5567 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5568 my $fileFormat = qemu_img_format
($scfg, $volname);
5569 $format = $fileFormat
5570 if grep { $fileFormat eq $_ } @$validFormats;
5572 $format //= $defFormat;
5575 # can't happen for remote migration, so $volname is always defined
5576 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5577 $format = qemu_img_format
($scfg, $volname);
5580 my $size = $drive->{size
} / 1024;
5581 my $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $format, undef, $size);
5582 my $newdrive = $drive;
5583 $newdrive->{format
} = $format;
5584 $newdrive->{file
} = $newvolid;
5585 my $drivestr = print_drive
($newdrive);
5586 $nbd->{$opt}->{drivestr
} = $drivestr;
5587 $nbd->{$opt}->{volid
} = $newvolid;
5593 # see vm_start_nolock for parameters, additionally:
5595 # storagemap = parsed storage map for allocating NBD disks
5597 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5599 return PVE
::QemuConfig-
>lock_config($vmid, sub {
5600 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migrate_opts->{migratedfrom
});
5602 die "you can't start a vm if it's a template\n"
5603 if !$params->{skiptemplate
} && PVE
::QemuConfig-
>is_template($conf);
5605 my $has_suspended_lock = PVE
::QemuConfig-
>has_lock($conf, 'suspended');
5606 my $has_backup_lock = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5608 my $running = check_running
($vmid, undef, $migrate_opts->{migratedfrom
});
5610 if ($has_backup_lock && $running) {
5611 # a backup is currently running, attempt to start the guest in the
5612 # existing QEMU instance
5613 return vm_resume
($vmid);
5616 PVE
::QemuConfig-
>check_lock($conf)
5617 if !($params->{skiplock
} || $has_suspended_lock);
5619 $params->{resume
} = $has_suspended_lock || defined($conf->{vmstate
});
5621 die "VM $vmid already running\n" if $running;
5623 if (my $storagemap = $migrate_opts->{storagemap
}) {
5624 my $replicated = $migrate_opts->{replicated_volumes
};
5625 my $disks = vm_migrate_get_nbd_disks
($storecfg, $conf, $replicated);
5626 $migrate_opts->{nbd
} = vm_migrate_alloc_nbd_disks
($storecfg, $vmid, $disks, $storagemap);
5628 foreach my $opt (keys %{$migrate_opts->{nbd
}}) {
5629 $conf->{$opt} = $migrate_opts->{nbd
}->{$opt}->{drivestr
};
5633 return vm_start_nolock
($storecfg, $vmid, $conf, $params, $migrate_opts);
5639 # statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5640 # skiplock => 0/1, skip checking for config lock
5641 # skiptemplate => 0/1, skip checking whether VM is template
5642 # forcemachine => to force QEMU machine (rollback/migration)
5643 # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5644 # timeout => in seconds
5645 # paused => start VM in paused state (backup)
5646 # resume => resume from hibernation
5657 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5658 # migratedfrom => source node
5659 # spice_ticket => used for spice migration, passed via tunnel/stdin
5660 # network => CIDR of migration network
5661 # type => secure/insecure - tunnel over encrypted connection or plain-text
5662 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5663 # replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5664 # offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5665 # contained in config
5666 sub vm_start_nolock
{
5667 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5669 my $statefile = $params->{statefile
};
5670 my $resume = $params->{resume
};
5672 my $migratedfrom = $migrate_opts->{migratedfrom
};
5673 my $migration_type = $migrate_opts->{type
};
5677 # clean up leftover reboot request files
5678 eval { clear_reboot_request
($vmid); };
5681 if (!$statefile && scalar(keys %{$conf->{pending
}})) {
5682 vmconfig_apply_pending
($vmid, $conf, $storecfg);
5683 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5686 # don't regenerate the ISO if the VM is started as part of a live migration
5687 # this way we can reuse the old ISO with the correct config
5688 if (!$migratedfrom) {
5689 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5690 # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5691 # $conf->{cloudinit}, so we could just not do this?
5692 # But we do it above, so for now let's be consistent.
5693 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5697 # override offline migrated volumes, conf is out of date still
5698 if (my $offline_volumes = $migrate_opts->{offline_volumes
}) {
5699 for my $key (sort keys $offline_volumes->%*) {
5700 my $parsed = parse_drive
($key, $conf->{$key});
5701 $parsed->{file
} = $offline_volumes->{$key};
5702 $conf->{$key} = print_drive
($parsed);
5706 my $defaults = load_defaults
();
5708 # set environment variable useful inside network script
5709 # for remote migration the config is available on the target node!
5710 if (!$migrate_opts->{remote_node
}) {
5711 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom;
5714 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5716 my $forcemachine = $params->{forcemachine
};
5717 my $forcecpu = $params->{forcecpu
};
5719 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5720 $forcemachine = $conf->{runningmachine
};
5721 $forcecpu = $conf->{runningcpu
};
5722 print "Resuming suspended VM\n";
5725 my ($cmd, $vollist, $spice_port) = config_to_command
($storecfg, $vmid,
5726 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5729 my $get_migration_ip = sub {
5730 my ($nodename) = @_;
5732 return $migration_ip if defined($migration_ip);
5734 my $cidr = $migrate_opts->{network
};
5736 if (!defined($cidr)) {
5737 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5738 $cidr = $dc_conf->{migration
}->{network
};
5741 if (defined($cidr)) {
5742 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5744 die "could not get IP: no address configured on local " .
5745 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5747 die "could not get IP: multiple addresses configured on local " .
5748 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5750 $migration_ip = @$ips[0];
5753 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5754 if !defined($migration_ip);
5756 return $migration_ip;
5760 if ($statefile eq 'tcp') {
5761 my $migrate = $res->{migrate
} = { proto
=> 'tcp' };
5762 $migrate->{addr
} = "localhost";
5763 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5764 my $nodename = nodename
();
5766 if (!defined($migration_type)) {
5767 if (defined($datacenterconf->{migration
}->{type
})) {
5768 $migration_type = $datacenterconf->{migration
}->{type
};
5770 $migration_type = 'secure';
5774 if ($migration_type eq 'insecure') {
5775 $migrate->{addr
} = $get_migration_ip->($nodename);
5776 $migrate->{addr
} = "[$migrate->{addr}]" if Net
::IP
::ip_is_ipv6
($migrate->{addr
});
5779 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5780 $migrate->{port
} = PVE
::Tools
::next_migrate_port
($pfamily);
5781 $migrate->{uri
} = "tcp:$migrate->{addr}:$migrate->{port}";
5782 push @$cmd, '-incoming', $migrate->{uri
};
5785 } elsif ($statefile eq 'unix') {
5786 # should be default for secure migrations as a ssh TCP forward
5787 # tunnel is not deterministic reliable ready and fails regurarly
5788 # to set up in time, so use UNIX socket forwards
5789 my $migrate = $res->{migrate
} = { proto
=> 'unix' };
5790 $migrate->{addr
} = "/run/qemu-server/$vmid.migrate";
5791 unlink $migrate->{addr
};
5793 $migrate->{uri
} = "unix:$migrate->{addr}";
5794 push @$cmd, '-incoming', $migrate->{uri
};
5797 } elsif (-e
$statefile) {
5798 push @$cmd, '-loadstate', $statefile;
5800 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5801 push @$vollist, $statefile;
5802 push @$cmd, '-loadstate', $statepath;
5804 } elsif ($params->{paused
}) {
5808 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $resume);
5810 my $pci_devices = {}; # host pci devices
5811 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
5812 my $dev = $conf->{"hostpci$i"} or next;
5813 $pci_devices->{$i} = parse_hostpci
($dev);
5816 # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5817 my $real_pci_devices = [ grep { !(defined($_->{mdev
}) && scalar($_->{pciid
}->@*) == 1) } values $pci_devices->%* ];
5819 # map to a flat list of pci ids
5820 my $pci_id_list = [ map { $_->{id
} } map { $_->{pciid
}->@* } $real_pci_devices->@* ];
5822 # reserve all PCI IDs before actually doing anything with them
5823 PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_id_list, $vmid, $start_timeout);
5827 for my $id (sort keys %$pci_devices) {
5828 my $d = $pci_devices->{$id};
5829 for my $dev ($d->{pciid
}->@*) {
5830 my $info = PVE
::QemuServer
::PCI
::prepare_pci_device
($vmid, $dev->{id
}, $id, $d->{mdev
});
5832 # nvidia grid needs the uuid of the mdev as qemu parameter
5833 if ($d->{mdev
} && !defined($uuid) && $info->{vendor
} eq '10de') {
5834 $uuid = PVE
::QemuServer
::PCI
::generate_mdev_uuid
($vmid, $id);
5838 push @$cmd, '-uuid', $uuid if defined($uuid);
5841 eval { cleanup_pci_devices
($vmid, $conf) };
5846 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5849 run_command
(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc
=> sub{}, errfunc
=> sub{});
5851 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5852 # timeout should be more than enough here...
5853 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 20);
5855 my $cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{cpuunits
});
5858 timeout
=> $statefile ?
undef : $start_timeout,
5863 # when migrating, prefix QEMU output so other side can pick up any
5864 # errors that might occur and show the user
5865 if ($migratedfrom) {
5866 $run_params{quiet
} = 1;
5867 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5870 my %systemd_properties = (
5871 Slice
=> 'qemu.slice',
5872 KillMode
=> 'process',
5874 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5877 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5878 $systemd_properties{CPUWeight
} = $cpuunits;
5880 $systemd_properties{CPUShares
} = $cpuunits;
5883 if (my $cpulimit = $conf->{cpulimit
}) {
5884 $systemd_properties{CPUQuota
} = int($cpulimit * 100);
5886 $systemd_properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5888 my $run_qemu = sub {
5889 PVE
::Tools
::run_fork
sub {
5890 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5893 if (my $tpm = $conf->{tpmstate0
}) {
5894 # start the TPM emulator so QEMU can connect on start
5895 $tpmpid = start_swtpm
($storecfg, $vmid, $tpm, $migratedfrom);
5898 my $exitcode = run_command
($cmd, %run_params);
5901 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5902 kill 'TERM', $tpmpid;
5904 die "QEMU exited with code $exitcode\n";
5909 if ($conf->{hugepages
}) {
5912 my $hugepages_topology = PVE
::QemuServer
::Memory
::hugepages_topology
($conf);
5913 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5915 PVE
::QemuServer
::Memory
::hugepages_mount
();
5916 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5918 eval { $run_qemu->() };
5920 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5921 if !$conf->{keephugepages
};
5925 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5926 if !$conf->{keephugepages
};
5928 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5931 eval { $run_qemu->() };
5935 # deactivate volumes if start fails
5936 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5938 eval { cleanup_pci_devices
($vmid, $conf) };
5941 die "start failed: $err";
5944 # re-reserve all PCI IDs now that we can know the actual VM PID
5945 my $pid = PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
5946 eval { PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_id_list, $vmid, undef, $pid) };
5949 if (defined($res->{migrate
})) {
5950 print "migration listens on $res->{migrate}->{uri}\n";
5951 } elsif ($statefile) {
5952 eval { mon_cmd
($vmid, "cont"); };
5956 #start nbd server for storage migration
5957 if (my $nbd = $migrate_opts->{nbd
}) {
5958 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5960 my $migrate_storage_uri;
5961 # nbd_protocol_version > 0 for unix socket support
5962 if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5963 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5964 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
5965 $migrate_storage_uri = "nbd:unix:$socket_path";
5966 $res->{migrate
}->{unix_sockets
} = [$socket_path];
5968 my $nodename = nodename
();
5969 my $localip = $get_migration_ip->($nodename);
5970 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5971 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5973 mon_cmd
($vmid, "nbd-server-start", addr
=> {
5976 host
=> "${localip}",
5977 port
=> "${storage_migrate_port}",
5980 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5981 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5984 my $block_info = mon_cmd
($vmid, "query-block");
5985 $block_info = { map { $_->{device
} => $_ } $block_info->@* };
5987 foreach my $opt (sort keys %$nbd) {
5988 my $drivestr = $nbd->{$opt}->{drivestr
};
5989 my $volid = $nbd->{$opt}->{volid
};
5991 my $block_node = $block_info->{"drive-$opt"}->{inserted
}->{'node-name'};
5997 'node-name' => $block_node,
5998 writable
=> JSON
::true
,
6000 name
=> "drive-$opt", # NBD export name
6003 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6004 print "storage migration listens on $nbd_uri volume:$drivestr\n";
6005 print "re-using replicated volume: $opt - $volid\n"
6006 if $nbd->{$opt}->{replicated
};
6008 $res->{drives
}->{$opt} = $nbd->{$opt};
6009 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
6013 if ($migratedfrom) {
6015 set_migration_caps
($vmid);
6020 print "spice listens on port $spice_port\n";
6021 $res->{spice_port
} = $spice_port;
6022 if ($migrate_opts->{spice_ticket
}) {
6023 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
6024 $migrate_opts->{spice_ticket
});
6025 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
6030 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
6031 if !$statefile && $conf->{balloon
};
6033 foreach my $opt (keys %$conf) {
6034 next if $opt !~ m/^net\d+$/;
6035 my $nicconf = parse_net
($conf->{$opt});
6036 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
6038 add_nets_bridge_fdb
($conf, $vmid);
6041 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
6045 path
=> "machine/peripheral/balloon0",
6046 property
=> "guest-stats-polling-interval",
6052 print "Resumed VM, removing state\n";
6053 if (my $vmstate = $conf->{vmstate
}) {
6054 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6055 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6057 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6058 PVE
::QemuConfig-
>write_config($vmid, $conf);
6061 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
6066 sub vm_commandline
{
6067 my ($storecfg, $vmid, $snapname) = @_;
6069 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6071 my ($forcemachine, $forcecpu);
6073 my $snapshot = $conf->{snapshots
}->{$snapname};
6074 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6076 # check for machine or CPU overrides in snapshot
6077 $forcemachine = $snapshot->{runningmachine
};
6078 $forcecpu = $snapshot->{runningcpu
};
6080 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
6085 my $defaults = load_defaults
();
6087 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6089 return PVE
::Tools
::cmd2string
($cmd);
6093 my ($vmid, $skiplock) = @_;
6095 PVE
::QemuConfig-
>lock_config($vmid, sub {
6097 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6099 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6101 mon_cmd
($vmid, "system_reset");
6105 sub get_vm_volumes
{
6109 foreach_volid
($conf, sub {
6110 my ($volid, $attr) = @_;
6112 return if $volid =~ m
|^/|;
6114 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6117 push @$vollist, $volid;
6123 sub cleanup_pci_devices
{
6124 my ($vmid, $conf) = @_;
6126 foreach my $key (keys %$conf) {
6127 next if $key !~ m/^hostpci(\d+)$/;
6128 my $hostpciindex = $1;
6129 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
6130 my $d = parse_hostpci
($conf->{$key});
6132 # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6133 # don't want to break ABI just for this two liner
6134 my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6135 PVE
::SysFSTools
::file_write
("$dev_sysfs_dir/remove", "1") if -e
$dev_sysfs_dir;
6138 PVE
::QemuServer
::PCI
::remove_pci_reservation
($vmid);
6141 sub vm_stop_cleanup
{
6142 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6147 my $vollist = get_vm_volumes
($conf);
6148 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6150 if (my $tpmdrive = $conf->{tpmstate0
}) {
6151 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
6152 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
6154 PVE
::Storage
::unmap_volume
($storecfg, $tpm->{file
});
6159 foreach my $ext (qw(mon qmp pid vnc qga)) {
6160 unlink "/var/run/qemu-server/${vmid}.$ext";
6163 if ($conf->{ivshmem
}) {
6164 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
6165 # just delete it for now, VMs which have this already open do not
6166 # are affected, but new VMs will get a separated one. If this
6167 # becomes an issue we either add some sort of ref-counting or just
6168 # add a "don't delete on stop" flag to the ivshmem format.
6169 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
6172 cleanup_pci_devices
($vmid, $conf);
6174 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
6176 warn $@ if $@; # avoid errors - just warn
6179 # call only in locked context
6181 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6183 my $pid = check_running
($vmid, $nocheck);
6188 $conf = PVE
::QemuConfig-
>load_config($vmid);
6189 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6190 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
6191 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
6192 $timeout = $opts->{down
} if $opts->{down
};
6194 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
6199 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
6200 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
6202 mon_cmd
($vmid, "system_powerdown");
6205 mon_cmd
($vmid, "quit");
6211 $timeout = 60 if !defined($timeout);
6214 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6219 if ($count >= $timeout) {
6221 warn "VM still running - terminating now with SIGTERM\n";
6224 die "VM quit/powerdown failed - got timeout\n";
6227 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6231 if (!check_running
($vmid, $nocheck)) {
6232 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6236 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6239 die "VM quit/powerdown failed\n";
6247 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6252 if ($count >= $timeout) {
6253 warn "VM still running - terminating now with SIGKILL\n";
6258 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6261 # Note: use $nocheck to skip tests if VM configuration file exists.
6262 # We need that when migration VMs to other nodes (files already moved)
6263 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6265 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6267 $force = 1 if !defined($force) && !$shutdown;
6270 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
6271 kill 15, $pid if $pid;
6272 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
6273 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
6277 PVE
::QemuConfig-
>lock_config($vmid, sub {
6278 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6283 my ($vmid, $timeout) = @_;
6285 PVE
::QemuConfig-
>lock_config($vmid, sub {
6288 # only reboot if running, as qmeventd starts it again on a stop event
6289 return if !check_running
($vmid);
6291 create_reboot_request
($vmid);
6293 my $storecfg = PVE
::Storage
::config
();
6294 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
6298 # avoid that the next normal shutdown will be confused for a reboot
6299 clear_reboot_request
($vmid);
6305 # note: if using the statestorage parameter, the caller has to check privileges
6307 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6314 PVE
::QemuConfig-
>lock_config($vmid, sub {
6316 $conf = PVE
::QemuConfig-
>load_config($vmid);
6318 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
6319 PVE
::QemuConfig-
>check_lock($conf)
6320 if !($skiplock || $is_backing_up);
6322 die "cannot suspend to disk during backup\n"
6323 if $is_backing_up && $includestate;
6325 if ($includestate) {
6326 $conf->{lock} = 'suspending';
6327 my $date = strftime
("%Y-%m-%d", localtime(time()));
6328 $storecfg = PVE
::Storage
::config
();
6329 if (!$statestorage) {
6330 $statestorage = find_vmstate_storage
($conf, $storecfg);
6331 # check permissions for the storage
6332 my $rpcenv = PVE
::RPCEnvironment
::get
();
6333 if ($rpcenv->{type
} ne 'cli') {
6334 my $authuser = $rpcenv->get_user();
6335 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6340 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
6341 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6342 $path = PVE
::Storage
::path
($storecfg, $vmstate);
6343 PVE
::QemuConfig-
>write_config($vmid, $conf);
6345 mon_cmd
($vmid, "stop");
6349 if ($includestate) {
6351 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
6354 set_migration_caps
($vmid, 1);
6355 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
6357 my $state = mon_cmd
($vmid, "query-savevm");
6358 if (!$state->{status
}) {
6359 die "savevm not active\n";
6360 } elsif ($state->{status
} eq 'active') {
6363 } elsif ($state->{status
} eq 'completed') {
6364 print "State saved, quitting\n";
6366 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
6367 die "query-savevm failed with error '$state->{error}'\n"
6369 die "query-savevm returned status '$state->{status}'\n";
6375 PVE
::QemuConfig-
>lock_config($vmid, sub {
6376 $conf = PVE
::QemuConfig-
>load_config($vmid);
6378 # cleanup, but leave suspending lock, to indicate something went wrong
6380 mon_cmd
($vmid, "savevm-end");
6381 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6382 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6383 delete $conf->@{qw(vmstate runningmachine runningcpu)};
6384 PVE
::QemuConfig-
>write_config($vmid, $conf);
6390 die "lock changed unexpectedly\n"
6391 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
6393 mon_cmd
($vmid, "quit");
6394 $conf->{lock} = 'suspended';
6395 PVE
::QemuConfig-
>write_config($vmid, $conf);
6400 # $nocheck is set when called as part of a migration - in this context the
6401 # location of the config file (source or target node) is not deterministic,
6402 # since migration cannot wait for pmxcfs to process the rename
6404 my ($vmid, $skiplock, $nocheck) = @_;
6406 PVE
::QemuConfig-
>lock_config($vmid, sub {
6407 my $res = mon_cmd
($vmid, 'query-status');
6408 my $resume_cmd = 'cont';
6412 $conf = eval { PVE
::QemuConfig-
>load_config($vmid) }; # try on target node
6414 my $vmlist = PVE
::Cluster
::get_vmlist
();
6415 if (exists($vmlist->{ids
}->{$vmid})) {
6416 my $node = $vmlist->{ids
}->{$vmid}->{node
};
6417 $conf = eval { PVE
::QemuConfig-
>load_config($vmid, $node) }; # try on source node
6420 PVE
::Cluster
::cfs_update
(); # vmlist was wrong, invalidate cache
6421 $conf = PVE
::QemuConfig-
>load_config($vmid); # last try on target node again
6425 $conf = PVE
::QemuConfig-
>load_config($vmid);
6428 if ($res->{status
}) {
6429 return if $res->{status
} eq 'running'; # job done, go home
6430 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
6431 $reset = 1 if $res->{status
} eq 'shutdown';
6435 PVE
::QemuConfig-
>check_lock($conf)
6436 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
6440 # required if a VM shuts down during a backup and we get a resume
6441 # request before the backup finishes for example
6442 mon_cmd
($vmid, "system_reset");
6445 add_nets_bridge_fdb
($conf, $vmid) if $resume_cmd eq 'cont';
6447 mon_cmd
($vmid, $resume_cmd);
6452 my ($vmid, $skiplock, $key) = @_;
6454 PVE
::QemuConfig-
>lock_config($vmid, sub {
6456 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6458 # there is no qmp command, so we use the human monitor command
6459 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
6460 die $res if $res ne '';
6464 # vzdump restore implementaion
6466 sub tar_archive_read_firstfile
{
6467 my $archive = shift;
6469 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
6471 # try to detect archive type first
6472 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6473 die "unable to open file '$archive'\n";
6474 my $firstfile = <$fh>;
6478 die "ERROR: archive contaions no data\n" if !$firstfile;
6484 sub tar_restore_cleanup
{
6485 my ($storecfg, $statfile) = @_;
6487 print STDERR
"starting cleanup\n";
6489 if (my $fd = IO
::File-
>new($statfile, "r")) {
6490 while (defined(my $line = <$fd>)) {
6491 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6494 if ($volid =~ m
|^/|) {
6495 unlink $volid || die 'unlink failed\n';
6497 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6499 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6501 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6503 print STDERR
"unable to parse line in statfile - $line";
6510 sub restore_file_archive
{
6511 my ($archive, $vmid, $user, $opts) = @_;
6513 return restore_vma_archive
($archive, $vmid, $user, $opts)
6516 my $info = PVE
::Storage
::archive_info
($archive);
6517 my $format = $opts->{format
} // $info->{format
};
6518 my $comp = $info->{compression
};
6520 # try to detect archive format
6521 if ($format eq 'tar') {
6522 return restore_tar_archive
($archive, $vmid, $user, $opts);
6524 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
6528 # hepler to remove disks that will not be used after restore
6529 my $restore_cleanup_oldconf = sub {
6530 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6532 my $kept_disks = {};
6534 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
6535 my ($ds, $drive) = @_;
6537 return if drive_is_cdrom
($drive, 1);
6539 my $volid = $drive->{file
};
6540 return if !$volid || $volid =~ m
|^/|;
6542 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
6543 return if !$path || !$owner || ($owner != $vmid);
6545 # Note: only delete disk we want to restore
6546 # other volumes will become unused
6547 if ($virtdev_hash->{$ds}) {
6548 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
6553 $kept_disks->{$volid} = 1;
6557 # after the restore we have no snapshots anymore
6558 for my $snapname (keys $oldconf->{snapshots
}->%*) {
6559 my $snap = $oldconf->{snapshots
}->{$snapname};
6560 if ($snap->{vmstate
}) {
6561 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
6567 for my $volid (keys $kept_disks->%*) {
6568 eval { PVE
::Storage
::volume_snapshot_delete
($storecfg, $volid, $snapname); };
6574 # Helper to parse vzdump backup device hints
6576 # $rpcenv: Environment, used to ckeck storage permissions
6577 # $user: User ID, to check storage permissions
6578 # $storecfg: Storage configuration
6579 # $fh: the file handle for reading the configuration
6580 # $devinfo: should contain device sizes for all backu-up'ed devices
6581 # $options: backup options (pool, default storage)
6583 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6584 my $parse_backup_hints = sub {
6585 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6587 my $check_storage = sub { # assert if an image can be allocate
6588 my ($storeid, $scfg) = @_;
6589 die "Content type 'images' is not available on storage '$storeid'\n"
6590 if !$scfg->{content
}->{images
};
6591 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6592 if $user ne 'root@pam';
6595 my $virtdev_hash = {};
6596 while (defined(my $line = <$fh>)) {
6597 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6598 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6599 die "archive does not contain data for drive '$virtdev'\n"
6600 if !$devinfo->{$devname};
6602 if (defined($options->{storage
})) {
6603 $storeid = $options->{storage
} || 'local';
6604 } elsif (!$storeid) {
6607 $format = 'raw' if !$format;
6608 $devinfo->{$devname}->{devname
} = $devname;
6609 $devinfo->{$devname}->{virtdev
} = $virtdev;
6610 $devinfo->{$devname}->{format
} = $format;
6611 $devinfo->{$devname}->{storeid
} = $storeid;
6613 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6614 $check_storage->($storeid, $scfg); # permission and content type check
6616 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6617 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6619 my $drive = parse_drive
($virtdev, $2);
6621 if (drive_is_cloudinit
($drive)) {
6622 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6623 $storeid = $options->{storage
} if defined ($options->{storage
});
6624 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6625 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6627 $check_storage->($storeid, $scfg); # permission and content type check
6629 $virtdev_hash->{$virtdev} = {
6631 storeid
=> $storeid,
6632 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6639 return $virtdev_hash;
6642 # Helper to allocate and activate all volumes required for a restore
6644 # $storecfg: Storage configuration
6645 # $virtdev_hash: as returned by parse_backup_hints()
6647 # Returns: { $virtdev => $volid }
6648 my $restore_allocate_devices = sub {
6649 my ($storecfg, $virtdev_hash, $vmid) = @_;
6652 foreach my $virtdev (sort keys %$virtdev_hash) {
6653 my $d = $virtdev_hash->{$virtdev};
6654 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6655 my $storeid = $d->{storeid
};
6656 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6658 # test if requested format is supported
6659 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6660 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6661 $d->{format
} = $defFormat if !$supported;
6664 if ($d->{is_cloudinit
}) {
6665 $name = "vm-$vmid-cloudinit";
6666 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6667 if ($scfg->{path
}) {
6668 $name .= ".$d->{format}";
6672 my $volid = PVE
::Storage
::vdisk_alloc
(
6673 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6675 print STDERR
"new volume ID is '$volid'\n";
6676 $d->{volid
} = $volid;
6678 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6680 $map->{$virtdev} = $volid;
6686 sub restore_update_config_line
{
6687 my ($cookie, $map, $line, $unique) = @_;
6689 return '' if $line =~ m/^\#qmdump\#/;
6690 return '' if $line =~ m/^\#vzdump\#/;
6691 return '' if $line =~ m/^lock:/;
6692 return '' if $line =~ m/^unused\d+:/;
6693 return '' if $line =~ m/^parent:/;
6697 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6698 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6699 # try to convert old 1.X settings
6700 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6701 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6702 my ($model, $macaddr) = split(/\=/, $devconfig);
6703 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6706 bridge
=> "vmbr$ind",
6707 macaddr
=> $macaddr,
6709 my $netstr = print_net
($net);
6711 $res .= "net$cookie->{netcount}: $netstr\n";
6712 $cookie->{netcount
}++;
6714 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6715 my ($id, $netstr) = ($1, $2);
6716 my $net = parse_net
($netstr);
6717 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6718 $netstr = print_net
($net);
6719 $res .= "$id: $netstr\n";
6720 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6723 my $di = parse_drive
($virtdev, $value);
6724 if (defined($di->{backup
}) && !$di->{backup
}) {
6726 } elsif ($map->{$virtdev}) {
6727 delete $di->{format
}; # format can change on restore
6728 $di->{file
} = $map->{$virtdev};
6729 $value = print_drive
($di);
6730 $res .= "$virtdev: $value\n";
6734 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6736 if ($vmgenid ne '0') {
6737 # always generate a new vmgenid if there was a valid one setup
6738 $vmgenid = generate_uuid
();
6740 $res .= "vmgenid: $vmgenid\n";
6741 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6742 my ($uuid, $uuid_str);
6743 UUID
::generate
($uuid);
6744 UUID
::unparse
($uuid, $uuid_str);
6745 my $smbios1 = parse_smbios1
($2);
6746 $smbios1->{uuid
} = $uuid_str;
6747 $res .= $1.print_smbios1
($smbios1)."\n";
6755 my $restore_deactivate_volumes = sub {
6756 my ($storecfg, $virtdev_hash) = @_;
6759 for my $dev (values $virtdev_hash->%*) {
6760 push $vollist->@*, $dev->{volid
} if $dev->{volid
};
6763 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
6764 print STDERR
$@ if $@;
6767 my $restore_destroy_volumes = sub {
6768 my ($storecfg, $virtdev_hash) = @_;
6770 for my $dev (values $virtdev_hash->%*) {
6771 my $volid = $dev->{volid
} or next;
6773 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6774 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6776 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6780 my $restore_merge_config = sub {
6781 my ($filename, $backup_conf_raw, $override_conf) = @_;
6783 my $backup_conf = parse_vm_config
($filename, $backup_conf_raw);
6784 for my $key (keys $override_conf->%*) {
6785 $backup_conf->{$key} = $override_conf->{$key};
6788 return $backup_conf;
6792 my ($cfg, $vmid) = @_;
6794 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6796 my $volid_hash = {};
6797 foreach my $storeid (keys %$info) {
6798 foreach my $item (@{$info->{$storeid}}) {
6799 next if !($item->{volid
} && $item->{size
});
6800 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6801 $volid_hash->{$item->{volid
}} = $item;
6808 sub update_disk_config
{
6809 my ($vmid, $conf, $volid_hash) = @_;
6812 my $prefix = "VM $vmid";
6814 # used and unused disks
6815 my $referenced = {};
6817 # Note: it is allowed to define multiple storages with same path (alias), so
6818 # we need to check both 'volid' and real 'path' (two different volid can point
6819 # to the same path).
6821 my $referencedpath = {};
6824 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6825 my ($opt, $drive) = @_;
6827 my $volid = $drive->{file
};
6829 my $volume = $volid_hash->{$volid};
6831 # mark volid as "in-use" for next step
6832 $referenced->{$volid} = 1;
6833 if ($volume && (my $path = $volume->{path
})) {
6834 $referencedpath->{$path} = 1;
6837 return if drive_is_cdrom
($drive);
6840 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6841 if (defined($updated)) {
6843 $conf->{$opt} = print_drive
($updated);
6844 print "$prefix ($opt): $msg\n";
6848 # remove 'unusedX' entry if volume is used
6849 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6850 my ($opt, $drive) = @_;
6852 my $volid = $drive->{file
};
6856 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6857 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6858 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6860 delete $conf->{$opt};
6863 $referenced->{$volid} = 1;
6864 $referencedpath->{$path} = 1 if $path;
6867 foreach my $volid (sort keys %$volid_hash) {
6868 next if $volid =~ m/vm-$vmid-state-/;
6869 next if $referenced->{$volid};
6870 my $path = $volid_hash->{$volid}->{path
};
6871 next if !$path; # just to be sure
6872 next if $referencedpath->{$path};
6874 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6875 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6876 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6883 my ($vmid, $nolock, $dryrun) = @_;
6885 my $cfg = PVE
::Storage
::config
();
6887 print "rescan volumes...\n";
6888 my $volid_hash = scan_volids
($cfg, $vmid);
6890 my $updatefn = sub {
6893 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6895 PVE
::QemuConfig-
>check_lock($conf);
6898 foreach my $volid (keys %$volid_hash) {
6899 my $info = $volid_hash->{$volid};
6900 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6903 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6905 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6908 if (defined($vmid)) {
6912 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6915 my $vmlist = config_list
();
6916 foreach my $vmid (keys %$vmlist) {
6920 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6926 sub restore_proxmox_backup_archive
{
6927 my ($archive, $vmid, $user, $options) = @_;
6929 my $storecfg = PVE
::Storage
::config
();
6931 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6932 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6934 my $fingerprint = $scfg->{fingerprint
};
6935 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6937 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6938 my $namespace = $scfg->{namespace
};
6940 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6941 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6942 local $ENV{PBS_PASSWORD
} = $password;
6943 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6945 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6946 PVE
::Storage
::parse_volname
($storecfg, $archive);
6948 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6950 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6952 my $tmpdir = "/var/tmp/vzdumptmp$$";
6956 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6957 # disable interrupts (always do cleanups)
6961 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6963 # Note: $oldconf is undef if VM does not exists
6964 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6965 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6966 my $new_conf_raw = '';
6968 my $rpcenv = PVE
::RPCEnvironment
::get
();
6969 my $devinfo = {}; # info about drives included in backup
6970 my $virtdev_hash = {}; # info about allocated drives
6978 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6980 my $cfgfn = "$tmpdir/qemu-server.conf";
6981 my $firewall_config_fn = "$tmpdir/fw.conf";
6982 my $index_fn = "$tmpdir/index.json";
6984 my $cmd = "restore";
6986 my $param = [$pbs_backup_name, "index.json", $index_fn];
6987 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6988 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6989 $index = decode_json
($index);
6991 foreach my $info (@{$index->{files
}}) {
6992 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6994 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6995 $devinfo->{$devname}->{size
} = $1;
6997 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7002 my $is_qemu_server_backup = scalar(
7003 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
7005 if (!$is_qemu_server_backup) {
7006 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7008 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
7010 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7011 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7013 if ($has_firewall_config) {
7014 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7015 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7017 my $pve_firewall_dir = '/etc/pve/firewall';
7018 mkdir $pve_firewall_dir; # make sure the dir exists
7019 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7022 my $fh = IO
::File-
>new($cfgfn, "r") ||
7023 die "unable to read qemu-server.conf - $!\n";
7025 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7027 # fixme: rate limit?
7029 # create empty/temp config
7030 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
7032 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7035 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7037 foreach my $virtdev (sort keys %$virtdev_hash) {
7038 my $d = $virtdev_hash->{$virtdev};
7039 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7041 # this fails if storage is unavailable
7042 my $volid = $d->{volid
};
7043 my $path = PVE
::Storage
::path
($storecfg, $volid);
7045 # for live-restore we only want to preload the efidisk and TPM state
7046 next if $options->{live
} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7049 if (defined(my $ns = $scfg->{namespace
})) {
7050 @ns_arg = ('--ns', $ns);
7053 my $pbs_restore_cmd = [
7054 '/usr/bin/pbs-restore',
7055 '--repository', $repo,
7058 "$d->{devname}.img.fidx",
7063 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
7064 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
7066 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
7067 push @$pbs_restore_cmd, '--skip-zero';
7070 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
7071 print "restore proxmox backup image: $dbg_cmdstring\n";
7072 run_command
($pbs_restore_cmd);
7075 $fh->seek(0, 0) || die "seek failed - $!\n";
7077 my $cookie = { netcount
=> 0 };
7078 while (defined(my $line = <$fh>)) {
7079 $new_conf_raw .= restore_update_config_line
(
7091 if ($err || !$options->{live
}) {
7092 $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7098 $restore_destroy_volumes->($storecfg, $virtdev_hash);
7102 if ($options->{live
}) {
7103 # keep lock during live-restore
7104 $new_conf_raw .= "\nlock: create";
7107 my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf
});
7108 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7110 eval { rescan
($vmid, 1); };
7113 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
7115 if ($options->{live
}) {
7121 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
7123 my $conf = PVE
::QemuConfig-
>load_config($vmid);
7124 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
7126 # these special drives are already restored before start
7127 delete $devinfo->{'drive-efidisk0'};
7128 delete $devinfo->{'drive-tpmstate0-backup'};
7132 keyfile
=> $keyfile,
7133 snapshot
=> $pbs_backup_name,
7134 namespace
=> $namespace,
7136 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7138 PVE
::QemuConfig-
>remove_lock($vmid, "create");
7142 sub pbs_live_restore
{
7143 my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7145 print "starting VM for live-restore\n";
7146 print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7148 my $pbs_backing = {};
7149 for my $ds (keys %$restored_disks) {
7150 $ds =~ m/^drive-(.*)$/;
7152 $pbs_backing->{$confname} = {
7153 repository
=> $opts->{repo
},
7154 snapshot
=> $opts->{snapshot
},
7155 archive
=> "$ds.img.fidx",
7157 $pbs_backing->{$confname}->{keyfile
} = $opts->{keyfile
} if -e
$opts->{keyfile
};
7158 $pbs_backing->{$confname}->{namespace
} = $opts->{namespace
} if defined($opts->{namespace
});
7160 my $drive = parse_drive
($confname, $conf->{$confname});
7161 print "restoring '$ds' to '$drive->{file}'\n";
7164 my $drives_streamed = 0;
7166 # make sure HA doesn't interrupt our restore by stopping the VM
7167 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
7168 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7171 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7172 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7173 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'pbs-backing' => $pbs_backing}, {});
7175 my $qmeventd_fd = register_qmeventd_handle
($vmid);
7177 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7178 # this will effectively collapse the backing image chain consisting of
7179 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7180 # removes itself once all backing images vanish with 'auto-remove=on')
7182 for my $ds (sort keys %$restored_disks) {
7183 my $job_id = "restore-$ds";
7184 mon_cmd
($vmid, 'block-stream',
7185 'job-id' => $job_id,
7188 $jobs->{$job_id} = {};
7191 mon_cmd
($vmid, 'cont');
7192 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
7194 print "restore-drive jobs finished successfully, removing all tracking block devices"
7195 ." to disconnect from Proxmox Backup Server\n";
7197 for my $ds (sort keys %$restored_disks) {
7198 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7201 close($qmeventd_fd);
7207 warn "An error occurred during live-restore: $err\n";
7208 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
7209 die "live-restore failed\n";
7213 sub restore_vma_archive
{
7214 my ($archive, $vmid, $user, $opts, $comp) = @_;
7216 my $readfrom = $archive;
7218 my $cfg = PVE
::Storage
::config
();
7220 my $bwlimit = $opts->{bwlimit
};
7222 my $dbg_cmdstring = '';
7223 my $add_pipe = sub {
7225 push @$commands, $cmd;
7226 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7227 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
7232 if ($archive eq '-') {
7235 # If we use a backup from a PVE defined storage we also consider that
7236 # storage's rate limit:
7237 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
7238 if (defined($volid)) {
7239 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
7240 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
7242 print STDERR
"applying read rate limit: $readlimit\n";
7243 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7244 $add_pipe->($cstream);
7250 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
7251 my $cmd = $info->{decompressor
};
7252 push @$cmd, $readfrom;
7256 my $tmpdir = "/var/tmp/vzdumptmp$$";
7259 # disable interrupts (always do cleanups)
7263 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
7265 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7266 POSIX
::mkfifo
($mapfifo, 0600);
7268 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7270 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7275 my $devinfo = {}; # info about drives included in backup
7276 my $virtdev_hash = {}; # info about allocated drives
7278 my $rpcenv = PVE
::RPCEnvironment
::get
();
7280 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7282 # Note: $oldconf is undef if VM does not exist
7283 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7284 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7285 my $new_conf_raw = '';
7289 my $print_devmap = sub {
7290 my $cfgfn = "$tmpdir/qemu-server.conf";
7292 # we can read the config - that is already extracted
7293 my $fh = IO
::File-
>new($cfgfn, "r") ||
7294 die "unable to read qemu-server.conf - $!\n";
7296 my $fwcfgfn = "$tmpdir/qemu-server.fw";
7298 my $pve_firewall_dir = '/etc/pve/firewall';
7299 mkdir $pve_firewall_dir; # make sure the dir exists
7300 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7303 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7305 foreach my $info (values %{$virtdev_hash}) {
7306 my $storeid = $info->{storeid
};
7307 next if defined($storage_limits{$storeid});
7309 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
7310 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7311 $storage_limits{$storeid} = $limit * 1024;
7314 foreach my $devname (keys %$devinfo) {
7315 die "found no device mapping information for device '$devname'\n"
7316 if !$devinfo->{$devname}->{virtdev
};
7319 # create empty/temp config
7321 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
7322 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7326 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7328 # print restore information to $fifofh
7329 foreach my $virtdev (sort keys %$virtdev_hash) {
7330 my $d = $virtdev_hash->{$virtdev};
7331 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7333 my $storeid = $d->{storeid
};
7334 my $volid = $d->{volid
};
7337 if (my $limit = $storage_limits{$storeid}) {
7338 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7341 my $write_zeros = 1;
7342 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
7346 my $path = PVE
::Storage
::path
($cfg, $volid);
7348 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7350 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7353 $fh->seek(0, 0) || die "seek failed - $!\n";
7355 my $cookie = { netcount
=> 0 };
7356 while (defined(my $line = <$fh>)) {
7357 $new_conf_raw .= restore_update_config_line
(
7374 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7375 local $SIG{ALRM
} = sub { die "got timeout\n"; };
7377 $oldtimeout = alarm($timeout);
7384 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7385 my ($dev_id, $size, $devname) = ($1, $2, $3);
7386 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
7387 } elsif ($line =~ m/^CTIME: /) {
7388 # we correctly received the vma config, so we can disable
7389 # the timeout now for disk allocation (set to 10 minutes, so
7390 # that we always timeout if something goes wrong)
7393 print $fifofh "done\n";
7394 my $tmp = $oldtimeout || 0;
7395 $oldtimeout = undef;
7402 print "restore vma archive: $dbg_cmdstring\n";
7403 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
7407 alarm($oldtimeout) if $oldtimeout;
7409 $restore_deactivate_volumes->($cfg, $virtdev_hash);
7411 close($fifofh) if $fifofh;
7416 $restore_destroy_volumes->($cfg, $virtdev_hash);
7420 my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf
});
7421 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7423 eval { rescan
($vmid, 1); };
7426 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
7429 sub restore_tar_archive
{
7430 my ($archive, $vmid, $user, $opts) = @_;
7432 if (scalar(keys $opts->{override_conf
}->%*) > 0) {
7433 my $keystring = join(' ', keys $opts->{override_conf
}->%*);
7434 die "cannot pass along options ($keystring) when restoring from tar archive\n";
7437 if ($archive ne '-') {
7438 my $firstfile = tar_archive_read_firstfile
($archive);
7439 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7440 if $firstfile ne 'qemu-server.conf';
7443 my $storecfg = PVE
::Storage
::config
();
7445 # avoid zombie disks when restoring over an existing VM -> cleanup first
7446 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7447 # skiplock=1 because qmrestore has set the 'create' lock itself already
7448 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
7449 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
7451 my $tocmd = "/usr/lib/qemu-server/qmextract";
7453 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
7454 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
7455 $tocmd .= ' --prealloc' if $opts->{prealloc
};
7456 $tocmd .= ' --info' if $opts->{info
};
7458 # tar option "xf" does not autodetect compression when read from STDIN,
7459 # so we pipe to zcat
7460 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
7461 PVE
::Tools
::shellquote
("--to-command=$tocmd");
7463 my $tmpdir = "/var/tmp/vzdumptmp$$";
7466 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
7467 local $ENV{VZDUMP_VMID
} = $vmid;
7468 local $ENV{VZDUMP_USER
} = $user;
7470 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7471 my $new_conf_raw = '';
7473 # disable interrupts (always do cleanups)
7477 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7485 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7487 if ($archive eq '-') {
7488 print "extracting archive from STDIN\n";
7489 run_command
($cmd, input
=> "<&STDIN");
7491 print "extracting archive '$archive'\n";
7495 return if $opts->{info
};
7499 my $statfile = "$tmpdir/qmrestore.stat";
7500 if (my $fd = IO
::File-
>new($statfile, "r")) {
7501 while (defined (my $line = <$fd>)) {
7502 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7503 $map->{$1} = $2 if $1;
7505 print STDERR
"unable to parse line in statfile - $line\n";
7511 my $confsrc = "$tmpdir/qemu-server.conf";
7513 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7515 my $cookie = { netcount
=> 0 };
7516 while (defined (my $line = <$srcfd>)) {
7517 $new_conf_raw .= restore_update_config_line
(
7528 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
7534 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7536 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7538 eval { rescan
($vmid, 1); };
7542 sub foreach_storage_used_by_vm
{
7543 my ($conf, $func) = @_;
7547 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7548 my ($ds, $drive) = @_;
7549 return if drive_is_cdrom
($drive);
7551 my $volid = $drive->{file
};
7553 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
7554 $sidhash->{$sid} = $sid if $sid;
7557 foreach my $sid (sort keys %$sidhash) {
7562 my $qemu_snap_storage = {
7565 sub do_snapshots_with_qemu
{
7566 my ($storecfg, $volid, $deviceid) = @_;
7568 return if $deviceid =~ m/tpmstate0/;
7570 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
7571 my $scfg = $storecfg->{ids
}->{$storage_name};
7572 die "could not find storage '$storage_name'\n" if !defined($scfg);
7574 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
7578 if ($volid =~ m/\.(qcow2|qed)$/){
7585 sub qga_check_running
{
7586 my ($vmid, $nowarn) = @_;
7588 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
7590 warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7596 sub template_create
{
7597 my ($vmid, $conf, $disk) = @_;
7599 my $storecfg = PVE
::Storage
::config
();
7601 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7602 my ($ds, $drive) = @_;
7604 return if drive_is_cdrom
($drive);
7605 return if $disk && $ds ne $disk;
7607 my $volid = $drive->{file
};
7608 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
7610 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
7611 $drive->{file
} = $voliddst;
7612 $conf->{$ds} = print_drive
($drive);
7613 PVE
::QemuConfig-
>write_config($vmid, $conf);
7617 sub convert_iscsi_path
{
7620 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
7625 my $initiator_name = get_initiator_name
();
7627 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7628 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7631 die "cannot convert iscsi path '$path', unkown format\n";
7634 sub qemu_img_convert
{
7635 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7637 my $storecfg = PVE
::Storage
::config
();
7638 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
7639 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
7641 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7645 my $src_is_iscsi = 0;
7649 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
7650 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7651 $src_format = qemu_img_format
($src_scfg, $src_volname);
7652 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
7653 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
7654 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
7655 } elsif (-f
$src_volid || -b
$src_volid) {
7656 $src_path = $src_volid;
7657 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7662 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7664 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7665 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7666 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7667 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7670 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7671 push @$cmd, '-l', "snapshot.name=$snapname"
7672 if $snapname && $src_format && $src_format eq "qcow2";
7673 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7674 push @$cmd, '-T', $cachemode if defined($cachemode);
7676 if ($src_is_iscsi) {
7677 push @$cmd, '--image-opts';
7678 $src_path = convert_iscsi_path
($src_path);
7679 } elsif ($src_format) {
7680 push @$cmd, '-f', $src_format;
7683 if ($dst_is_iscsi) {
7684 push @$cmd, '--target-image-opts';
7685 $dst_path = convert_iscsi_path
($dst_path);
7687 push @$cmd, '-O', $dst_format;
7690 push @$cmd, $src_path;
7692 if (!$dst_is_iscsi && $is_zero_initialized) {
7693 push @$cmd, "zeroinit:$dst_path";
7695 push @$cmd, $dst_path;
7700 if($line =~ m/\((\S+)\/100\
%\)/){
7702 my $transferred = int($size * $percent / 100);
7703 my $total_h = render_bytes
($size, 1);
7704 my $transferred_h = render_bytes
($transferred, 1);
7706 print "transferred $transferred_h of $total_h ($percent%)\n";
7711 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7713 die "copy failed: $err" if $err;
7716 sub qemu_img_format
{
7717 my ($scfg, $volname) = @_;
7719 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7726 sub qemu_drive_mirror
{
7727 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7729 $jobs = {} if !$jobs;
7733 $jobs->{"drive-$drive"} = {};
7735 if ($dst_volid =~ /^nbd:/) {
7736 $qemu_target = $dst_volid;
7739 my $storecfg = PVE
::Storage
::config
();
7740 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7742 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7744 $format = qemu_img_format
($dst_scfg, $dst_volname);
7746 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7748 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7751 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7752 $opts->{format
} = $format if $format;
7754 if (defined($src_bitmap)) {
7755 $opts->{sync
} = 'incremental';
7756 $opts->{bitmap
} = $src_bitmap;
7757 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7760 if (defined($bwlimit)) {
7761 $opts->{speed
} = $bwlimit * 1024;
7762 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7764 print "drive mirror is starting for drive-$drive\n";
7767 # if a job already runs for this device we get an error, catch it for cleanup
7768 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7770 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7772 die "mirroring error: $err\n";
7775 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7778 # $completion can be either
7779 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7780 # 'cancel': wait until all jobs are ready, block-job-cancel them
7781 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7782 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7783 sub qemu_drive_mirror_monitor
{
7784 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7786 $completion //= 'complete';
7790 my $err_complete = 0;
7792 my $starttime = time ();
7794 die "block job ('$op') timed out\n" if $err_complete > 300;
7796 my $stats = mon_cmd
($vmid, "query-block-jobs");
7799 my $running_jobs = {};
7800 for my $stat (@$stats) {
7801 next if $stat->{type
} ne $op;
7802 $running_jobs->{$stat->{device
}} = $stat;
7805 my $readycounter = 0;
7807 for my $job_id (sort keys %$jobs) {
7808 my $job = $running_jobs->{$job_id};
7810 my $vanished = !defined($job);
7811 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7812 if($complete || ($vanished && $completion eq 'auto')) {
7813 print "$job_id: $op-job finished\n";
7814 delete $jobs->{$job_id};
7818 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7820 my $busy = $job->{busy
};
7821 my $ready = $job->{ready
};
7822 if (my $total = $job->{len
}) {
7823 my $transferred = $job->{offset
} || 0;
7824 my $remaining = $total - $transferred;
7825 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7827 my $duration = $ctime - $starttime;
7828 my $total_h = render_bytes
($total, 1);
7829 my $transferred_h = render_bytes
($transferred, 1);
7831 my $status = sprintf(
7832 "transferred $transferred_h of $total_h ($percent%%) in %s",
7833 render_duration
($duration),
7838 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7840 $status .= ", ready";
7843 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
7844 $jobs->{$job_id}->{ready
} = $ready;
7847 $readycounter++ if $job->{ready
};
7850 last if scalar(keys %$jobs) == 0;
7852 if ($readycounter == scalar(keys %$jobs)) {
7853 print "all '$op' jobs are ready\n";
7855 # do the complete later (or has already been done)
7856 last if $completion eq 'skip' || $completion eq 'auto';
7858 if ($vmiddst && $vmiddst != $vmid) {
7859 my $agent_running = $qga && qga_check_running
($vmid);
7860 if ($agent_running) {
7861 print "freeze filesystem\n";
7862 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
7865 print "suspend vm\n";
7866 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
7870 # if we clone a disk for a new target vm, we don't switch the disk
7871 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
7873 if ($agent_running) {
7874 print "unfreeze filesystem\n";
7875 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
7878 print "resume vm\n";
7879 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
7886 for my $job_id (sort keys %$jobs) {
7887 # try to switch the disk if source and destination are on the same guest
7888 print "$job_id: Completing block job_id...\n";
7891 if ($completion eq 'complete') {
7892 $op = 'block-job-complete';
7893 } elsif ($completion eq 'cancel') {
7894 $op = 'block-job-cancel';
7896 die "invalid completion value: $completion\n";
7898 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
7899 if ($@ =~ m/cannot be completed/) {
7900 print "$job_id: block job cannot be completed, trying again.\n";
7903 print "$job_id: Completed successfully.\n";
7904 $jobs->{$job_id}->{complete
} = 1;
7915 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7916 die "block job ($op) error: $err";
7920 sub qemu_blockjobs_cancel
{
7921 my ($vmid, $jobs) = @_;
7923 foreach my $job (keys %$jobs) {
7924 print "$job: Cancelling block job\n";
7925 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
7926 $jobs->{$job}->{cancel
} = 1;
7930 my $stats = mon_cmd
($vmid, "query-block-jobs");
7932 my $running_jobs = {};
7933 foreach my $stat (@$stats) {
7934 $running_jobs->{$stat->{device
}} = $stat;
7937 foreach my $job (keys %$jobs) {
7939 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
7940 print "$job: Done.\n";
7941 delete $jobs->{$job};
7945 last if scalar(keys %$jobs) == 0;
7952 my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7954 my ($vmid, $running) = $source->@{qw(vmid running)};
7955 my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7957 my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7958 my ($storage, $format) = $dest->@{qw(storage format)};
7960 my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7962 if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7963 die "cloning from/to EFI disk requires EFI disk\n"
7964 if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7965 die "cloning from/to TPM state requires TPM state\n"
7966 if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7968 # This would lead to two device nodes in QEMU pointing to the same backing image!
7969 die "cannot change drive name when cloning disk from/to the same VM\n"
7970 if $use_drive_mirror && $vmid == $newvmid;
7973 die "cannot move TPM state while VM is running\n"
7974 if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7978 print "create " . ($full ?
'full' : 'linked') . " clone of drive ";
7979 print "$src_drivename " if $src_drivename;
7980 print "($drive->{file})\n";
7983 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
7984 push @$newvollist, $newvolid;
7987 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
7988 $storeid = $storage if $storage;
7990 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
7994 if (drive_is_cloudinit
($drive)) {
7995 $name = "vm-$newvmid-cloudinit";
7996 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7997 if ($scfg->{path
}) {
7998 $name .= ".$dst_format";
8001 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
8002 } elsif ($dst_drivename eq 'efidisk0') {
8003 $size = $efisize or die "internal error - need to specify EFI disk size\n";
8004 } elsif ($dst_drivename eq 'tpmstate0') {
8005 $dst_format = 'raw';
8006 $size = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8008 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
8010 $newvolid = PVE
::Storage
::vdisk_alloc
(
8011 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8013 push @$newvollist, $newvolid;
8015 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
8017 if (drive_is_cloudinit
($drive)) {
8018 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8019 # if this is the case, we have to complete any block-jobs still there from
8020 # previous drive-mirrors
8021 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8022 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
8027 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
8028 if ($use_drive_mirror) {
8029 qemu_drive_mirror
($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8030 $completion, $qga, $bwlimit);
8032 # TODO: handle bwlimits
8033 if ($dst_drivename eq 'efidisk0') {
8034 # the relevant data on the efidisk may be smaller than the source
8035 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8036 # that is given by the OVMF_VARS.fd
8037 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
8038 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
8040 my $src_format = (PVE
::Storage
::parse_volname
($storecfg, $drive->{file
}))[6];
8042 # better for Ceph if block size is not too small, see bug #3324
8045 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8047 if ($src_format eq 'qcow2' && $snapname) {
8048 die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8049 if !min_version
(kvm_user_version
(), 6, 2);
8050 push $cmd->@*, '-l', $snapname;
8052 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8055 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
8061 my ($size) = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
8063 my $disk = dclone
($drive);
8064 delete $disk->{format
};
8065 $disk->{file
} = $newvolid;
8066 $disk->{size
} = $size if defined($size);
8071 sub get_running_qemu_version
{
8073 my $res = mon_cmd
($vmid, "query-version");
8074 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8077 sub qemu_use_old_bios_files
{
8078 my ($machine_type) = @_;
8080 return if !$machine_type;
8082 my $use_old_bios_files = undef;
8084 if ($machine_type =~ m/^(\S+)\.pxe$/) {
8086 $use_old_bios_files = 1;
8088 my $version = extract_version
($machine_type, kvm_user_version
());
8089 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8090 # load new efi bios files on migration. So this hack is required to allow
8091 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8092 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8093 $use_old_bios_files = !min_version
($version, 2, 4);
8096 return ($use_old_bios_files, $machine_type);
8099 sub get_efivars_size
{
8100 my ($conf, $efidisk) = @_;
8102 my $arch = get_vm_arch
($conf);
8103 $efidisk //= $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
8104 my $smm = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
8105 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8106 return -s
$ovmf_vars;
8109 sub update_efidisk_size
{
8112 return if !defined($conf->{efidisk0
});
8114 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
8115 $disk->{size
} = get_efivars_size
($conf);
8116 $conf->{efidisk0
} = print_drive
($disk);
8121 sub update_tpmstate_size
{
8124 my $disk = PVE
::QemuServer
::parse_drive
('tpmstate0', $conf->{tpmstate0
});
8125 $disk->{size
} = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8126 $conf->{tpmstate0
} = print_drive
($disk);
8129 sub create_efidisk
($$$$$$$) {
8130 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8132 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8134 my $vars_size_b = -s
$ovmf_vars;
8135 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
8136 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8137 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
8139 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
8140 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
8142 return ($volid, $size/1024);
8145 sub vm_iothreads_list
{
8148 my $res = mon_cmd
($vmid, 'query-iothreads');
8151 foreach my $iothread (@$res) {
8152 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
8159 my ($conf, $drive) = @_;
8163 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
8165 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
8171 my $controller = int($drive->{index} / $maxdev);
8172 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
8176 return ($maxdev, $controller, $controller_prefix);
8179 sub resolve_dst_disk_format
{
8180 my ($storecfg, $storeid, $src_volname, $format) = @_;
8181 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
8184 # if no target format is specified, use the source disk format as hint
8186 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8187 $format = qemu_img_format
($scfg, $src_volname);
8193 # test if requested format is supported - else use default
8194 my $supported = grep { $_ eq $format } @$validFormats;
8195 $format = $defFormat if !$supported;
8199 # NOTE: if this logic changes, please update docs & possibly gui logic
8200 sub find_vmstate_storage
{
8201 my ($conf, $storecfg) = @_;
8203 # first, return storage from conf if set
8204 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
8206 my ($target, $shared, $local);
8208 foreach_storage_used_by_vm
($conf, sub {
8210 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
8211 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
8212 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
8215 # second, use shared storage where VM has at least one disk
8216 # third, use local storage where VM has at least one disk
8217 # fall back to local storage
8218 $target = $shared // $local // 'local';
8224 my ($uuid, $uuid_str);
8225 UUID
::generate
($uuid);
8226 UUID
::unparse
($uuid, $uuid_str);
8230 sub generate_smbios1_uuid
{
8231 return "uuid=".generate_uuid
();
8237 mon_cmd
($vmid, 'nbd-server-stop');
8240 sub create_reboot_request
{
8242 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8243 or die "failed to create reboot trigger file: $!\n";
8247 sub clear_reboot_request
{
8249 my $path = "/run/qemu-server/$vmid.reboot";
8252 $res = unlink($path);
8253 die "could not remove reboot request for $vmid: $!"
8254 if !$res && $! != POSIX
::ENOENT
;
8259 sub bootorder_from_legacy
{
8260 my ($conf, $bootcfg) = @_;
8262 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
8263 my $bootindex_hash = {};
8265 foreach my $o (split(//, $boot)) {
8266 $bootindex_hash->{$o} = $i*100;
8272 PVE
::QemuConfig-
>foreach_volume($conf, sub {
8273 my ($ds, $drive) = @_;
8275 if (drive_is_cdrom
($drive, 1)) {
8276 if ($bootindex_hash->{d
}) {
8277 $bootorder->{$ds} = $bootindex_hash->{d
};
8278 $bootindex_hash->{d
} += 1;
8280 } elsif ($bootindex_hash->{c
}) {
8281 $bootorder->{$ds} = $bootindex_hash->{c
}
8282 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
8283 $bootindex_hash->{c
} += 1;
8287 if ($bootindex_hash->{n
}) {
8288 for (my $i = 0; $i < $MAX_NETS; $i++) {
8289 my $netname = "net$i";
8290 next if !$conf->{$netname};
8291 $bootorder->{$netname} = $bootindex_hash->{n
};
8292 $bootindex_hash->{n
} += 1;
8299 # Generate default device list for 'boot: order=' property. Matches legacy
8300 # default boot order, but with explicit device names. This is important, since
8301 # the fallback for when neither 'order' nor the old format is specified relies
8302 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8303 sub get_default_bootdevices
{
8309 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
8310 push @ret, $first if $first;
8313 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
8314 push @ret, $first if $first;
8317 for (my $i = 0; $i < $MAX_NETS; $i++) {
8318 my $netname = "net$i";
8319 next if !$conf->{$netname};
8320 push @ret, $netname;
8327 sub device_bootorder
{
8330 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
8332 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
8335 if (!defined($boot) || $boot->{legacy
}) {
8336 $bootorder = bootorder_from_legacy
($conf, $boot);
8337 } elsif ($boot->{order
}) {
8338 my $i = 100; # start at 100 to allow user to insert devices before us with -args
8339 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
8340 $bootorder->{$dev} = $i++;
8347 sub register_qmeventd_handle
{
8351 my $peer = "/var/run/qmeventd.sock";
8356 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
8358 if ($! != EINTR
&& $! != EAGAIN
) {
8359 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8362 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8363 . "after $count retries\n";
8368 # send handshake to mark VM as backing up
8369 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
8371 # return handle to be closed later when inhibit is no longer required
8375 # bash completion helper
8377 sub complete_backup_archives
{
8378 my ($cmdname, $pname, $cvalue) = @_;
8380 my $cfg = PVE
::Storage
::config
();
8384 if ($cvalue =~ m/^([^:]+):/) {
8388 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
8391 foreach my $id (keys %$data) {
8392 foreach my $item (@{$data->{$id}}) {
8393 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8394 push @$res, $item->{volid
} if defined($item->{volid
});
8401 my $complete_vmid_full = sub {
8404 my $idlist = vmstatus
();
8408 foreach my $id (keys %$idlist) {
8409 my $d = $idlist->{$id};
8410 if (defined($running)) {
8411 next if $d->{template
};
8412 next if $running && $d->{status
} ne 'running';
8413 next if !$running && $d->{status
} eq 'running';
8422 return &$complete_vmid_full();
8425 sub complete_vmid_stopped
{
8426 return &$complete_vmid_full(0);
8429 sub complete_vmid_running
{
8430 return &$complete_vmid_full(1);
8433 sub complete_storage
{
8435 my $cfg = PVE
::Storage
::config
();
8436 my $ids = $cfg->{ids
};
8439 foreach my $sid (keys %$ids) {
8440 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
8441 next if !$ids->{$sid}->{content
}->{images
};
8448 sub complete_migration_storage
{
8449 my ($cmd, $param, $current_value, $all_args) = @_;
8451 my $targetnode = @$all_args[1];
8453 my $cfg = PVE
::Storage
::config
();
8454 my $ids = $cfg->{ids
};
8457 foreach my $sid (keys %$ids) {
8458 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
8459 next if !$ids->{$sid}->{content
}->{images
};
8468 my $qmpstatus = eval {
8469 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
8470 mon_cmd
($vmid, "query-status");
8473 return $qmpstatus && $qmpstatus->{status
} eq "paused";
8476 sub check_volume_storage_type
{
8477 my ($storecfg, $vol) = @_;
8479 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
8480 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8481 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
8483 die "storage '$storeid' does not support content-type '$vtype'\n"
8484 if !$scfg->{content
}->{$vtype};
8489 sub add_nets_bridge_fdb
{
8490 my ($conf, $vmid) = @_;
8492 for my $opt (keys %$conf) {
8493 next if $opt !~ m/^net(\d+)$/;
8494 my $iface = "tap${vmid}i$1";
8495 # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8496 my $net = parse_net
($conf->{$opt}, 1) or next;
8498 my $mac = $net->{macaddr
};
8500 log_warn
("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8501 if !file_read_firstline
("/sys/class/net/$iface/brport/learning");
8505 my $bridge = $net->{bridge
};
8507 PVE
::Network
::SDN
::Zones
::add_bridge_fdb
($iface, $mac, $bridge, $net->{firewall
});
8508 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8509 PVE
::Network
::add_bridge_fdb
($iface, $mac, $net->{firewall
});
8514 sub del_nets_bridge_fdb
{
8515 my ($conf, $vmid) = @_;
8517 for my $opt (keys %$conf) {
8518 next if $opt !~ m/^net(\d+)$/;
8519 my $iface = "tap${vmid}i$1";
8521 my $net = parse_net
($conf->{$opt}) or next;
8522 my $mac = $net->{macaddr
} or next;
8524 my $bridge = $net->{bridge
};
8526 PVE
::Network
::SDN
::Zones
::del_bridge_fdb
($iface, $mac, $bridge, $net->{firewall
});
8527 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8528 PVE
::Network
::del_bridge_fdb
($iface, $mac, $net->{firewall
});