1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday usleep);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
32 use PVE
::DataCenterConfig
;
33 use PVE
::Exception
qw(raise raise_param_exc);
34 use PVE
::Format
qw(render_duration render_bytes);
35 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
37 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
40 use PVE
::RESTEnvironment
qw(log_warn);
41 use PVE
::RPCEnvironment
;
45 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
49 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout windows_version);
50 use PVE
::QemuServer
::Cloudinit
;
51 use PVE
::QemuServer
::CGroup
;
52 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
53 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
54 use PVE
::QemuServer
::Machine
;
55 use PVE
::QemuServer
::Memory
;
56 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
57 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
58 use PVE
::QemuServer
::USB
qw(parse_usb_device);
62 require PVE
::Network
::SDN
::Zones
;
66 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
70 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
71 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
74 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
75 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
78 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
79 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
82 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
83 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
86 "$EDK2_FW_BASE/OVMF_CODE.fd",
87 "$EDK2_FW_BASE/OVMF_VARS.fd",
92 "$EDK2_FW_BASE/AAVMF_CODE.fd",
93 "$EDK2_FW_BASE/AAVMF_VARS.fd",
98 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
100 # Note about locking: we use flock on the config file protect against concurent actions.
101 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
102 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
103 # But you can ignore this kind of lock with the --skiplock flag.
111 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
112 description
=> "Some command save/restore state from this location.",
118 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
119 description
=> "Specifies the Qemu machine type.",
121 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
126 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
129 $nodename_cache //= PVE
::INotify
::nodename
();
130 return $nodename_cache;
137 enum
=> [qw(i6300esb ib700)],
138 description
=> "Watchdog type to emulate.",
139 default => 'i6300esb',
144 enum
=> [qw(reset shutdown poweroff pause debug none)],
145 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
149 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
153 description
=> "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
158 fstrim_cloned_disks
=> {
159 description
=> "Run fstrim after moving a disk or migrating the VM.",
165 description
=> "Select the agent type",
169 enum
=> [qw(virtio isa)],
175 description
=> "Select the VGA type.",
180 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
183 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
195 description
=> "The size of the file in MB.",
199 pattern
=> '[a-zA-Z0-9\-]+',
201 format_description
=> 'string',
202 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
209 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
210 description
=> "Configure an audio device."
214 enum
=> ['spice', 'none'],
217 description
=> "Driver backend for the audio device."
221 my $spice_enhancements_fmt = {
226 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
230 enum
=> ['off', 'all', 'filter'],
233 description
=> "Enable video streaming. Uses compression for detected video streams."
240 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
242 description
=> "The file on the host to gather entropy from. In most cases '/dev/urandom'"
243 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
244 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
245 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
246 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
247 ." a hardware RNG from the host.",
251 description
=> "Maximum bytes of entropy allowed to get injected into the guest every"
252 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
253 ." `0` to disable limiting (potentially dangerous!).",
256 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
257 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
258 # reading from /dev/urandom
263 description
=> "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
264 ." the guest to retrieve another 'max_bytes' of entropy.",
270 my $meta_info_fmt = {
273 description
=> "The guest creation timestamp as UNIX epoch time",
279 description
=> "The QEMU (machine) version from the time this VM was created.",
280 pattern
=> '\d+(\.\d+)+',
289 description
=> "Specifies whether a VM will be started during system bootup.",
295 description
=> "Automatic restart after crash (currently ignored).",
300 type
=> 'string', format
=> 'pve-hotplug-features',
301 description
=> "Selectively enable hotplug features. This is a comma separated list of"
302 ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
303 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
304 ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
306 default => 'network,disk,usb',
311 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
317 description
=> "Lock/unlock the VM.",
318 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
323 description
=> "Limit of CPU usage.",
324 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
325 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
333 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
334 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
335 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
336 ." weights of all the other running VMs.",
339 default => 'cgroup v1: 1024, cgroup v2: 100',
344 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when"
345 ." you use the balloon device.",
352 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
358 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the"
359 ." more memory this VM gets. Number is relative to weights of all other running VMs."
360 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
368 description
=> "Keyboard layout for VNC server. This option is generally not required and"
369 ." is often better handled from within the guest OS.",
370 enum
=> PVE
::Tools
::kvmkeymaplist
(),
375 type
=> 'string', format
=> 'dns-name',
376 description
=> "Set a name for the VM. Only used on the configuration web interface.",
381 description
=> "SCSI controller model",
382 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
388 description
=> "Description for the VM. Shown in the web-interface VM's summary."
389 ." This is saved as comment inside the configuration file.",
390 maxLength
=> 1024 * 8,
395 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
396 description
=> "Specify guest operating system.",
397 verbose_description
=> <<EODESC,
398 Specify guest operating system. This is used to enable special
399 optimization/features for specific operating systems:
402 other;; unspecified OS
403 wxp;; Microsoft Windows XP
404 w2k;; Microsoft Windows 2000
405 w2k3;; Microsoft Windows 2003
406 w2k8;; Microsoft Windows 2008
407 wvista;; Microsoft Windows Vista
408 win7;; Microsoft Windows 7
409 win8;; Microsoft Windows 8/2012/2012r2
410 win10;; Microsoft Windows 10/2016/2019
411 win11;; Microsoft Windows 11/2022
412 l24;; Linux 2.4 Kernel
413 l26;; Linux 2.6 - 5.X Kernel
414 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
419 type
=> 'string', format
=> 'pve-qm-boot',
420 description
=> "Specify guest boot order. Use the 'order=' sub-property as usage with no"
421 ." key or 'legacy=' is deprecated.",
425 type
=> 'string', format
=> 'pve-qm-bootdisk',
426 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
427 pattern
=> '(ide|sata|scsi|virtio)\d+',
432 description
=> "The number of CPUs. Please use option -sockets instead.",
439 description
=> "The number of CPU sockets.",
446 description
=> "The number of cores per socket.",
453 description
=> "Enable/disable NUMA.",
459 description
=> "Enable/disable hugepages memory.",
460 enum
=> [qw(any 2 1024)],
466 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
467 ." after VM shutdown and can be used for subsequent starts.",
472 description
=> "Number of hotplugged vcpus.",
479 description
=> "Enable/disable ACPI.",
484 description
=> "Enable/disable communication with the Qemu Guest Agent and its properties.",
486 format
=> $agent_fmt,
491 description
=> "Enable/disable KVM hardware virtualization.",
497 description
=> "Enable/disable time drift fix.",
503 description
=> "Set the real time clock (RTC) to local time. This is enabled by default if"
504 ." the `ostype` indicates a Microsoft Windows OS.",
509 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
513 type
=> 'string', format
=> $vga_fmt,
514 description
=> "Configure the VGA hardware.",
515 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
516 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
517 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
518 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
519 ." display server. For win* OS you can select how many independent displays you want,"
520 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
521 ." using a serial device as terminal.",
525 type
=> 'string', format
=> 'pve-qm-watchdog',
526 description
=> "Create a virtual hardware watchdog device.",
527 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
528 ." action), the watchdog must be periodically polled by an agent inside the guest or"
529 ." else the watchdog will reset the guest (or execute the respective action specified)",
534 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
535 description
=> "Set the initial date of the real time clock. Valid format for date are:"
536 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
537 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
540 startup
=> get_standard_option
('pve-startup-order'),
544 description
=> "Enable/disable Template.",
550 description
=> "Arbitrary arguments passed to kvm.",
551 verbose_description
=> <<EODESCR,
552 Arbitrary arguments passed to kvm, for example:
554 args: -no-reboot -no-hpet
556 NOTE: this option is for experts only.
563 description
=> "Enable/disable the USB tablet device.",
564 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
565 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
566 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
567 ." may consider disabling this to save some context switches. This is turned off by"
568 ." default if you use spice (`qm set <vmid> --vga qxl`).",
573 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
577 migrate_downtime
=> {
580 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
586 type
=> 'string', format
=> 'pve-qm-ide',
587 typetext
=> '<volume>',
588 description
=> "This is an alias for option -ide2",
592 description
=> "Emulated CPU type.",
594 format
=> 'pve-vm-cpu-conf',
596 parent
=> get_standard_option
('pve-snapshot-name', {
598 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
602 description
=> "Timestamp for snapshots.",
608 type
=> 'string', format
=> 'pve-volume-id',
609 description
=> "Reference to a volume which stores the VM state. This is used internally"
612 vmstatestorage
=> get_standard_option
('pve-storage-id', {
613 description
=> "Default storage for VM state volumes/files.",
616 runningmachine
=> get_standard_option
('pve-qemu-machine', {
617 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
621 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
622 ." internally for snapshots.",
625 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
626 format_description
=> 'QEMU -cpu parameter'
628 machine
=> get_standard_option
('pve-qemu-machine'),
630 description
=> "Virtual processor architecture. Defaults to the host.",
633 enum
=> [qw(x86_64 aarch64)],
636 description
=> "Specify SMBIOS type 1 fields.",
637 type
=> 'string', format
=> 'pve-qm-smbios1',
644 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
645 ." remove disk operations.",
651 enum
=> [ qw(seabios ovmf) ],
652 description
=> "Select BIOS implementation.",
653 default => 'seabios',
657 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
658 format_description
=> 'UUID',
659 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
660 ." to disable explicitly.",
661 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
662 ." value identifier to the guest OS. This allows to notify the guest operating system"
663 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
664 ." execution or creation from a template). The guest operating system notices the"
665 ." change, and is then able to react as appropriate by marking its copies of"
666 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
667 ."Note that auto-creation only works when done through API/CLI create or update methods"
668 .", but not when manually editing the config file.",
669 default => "1 (autogenerated)",
674 format
=> 'pve-volume-id',
676 description
=> "Script that will be executed during various steps in the vms lifetime.",
680 format
=> $ivshmem_fmt,
681 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
687 format
=> $audio_fmt,
688 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
691 spice_enhancements
=> {
693 format
=> $spice_enhancements_fmt,
694 description
=> "Configure additional enhancements for SPICE.",
698 type
=> 'string', format
=> 'pve-tag-list',
699 description
=> 'Tags of the VM. This is only meta information.',
705 description
=> "Configure a VirtIO-based Random Number Generator.",
710 format
=> $meta_info_fmt,
711 description
=> "Some (read-only) meta-information about this guest.",
715 type
=> 'string', format
=> 'pve-cpuset',
716 description
=> "List of host cores used to execute guest processes, for example: 0,5,8-11",
725 description
=> 'Specify a custom file containing all meta data passed to the VM via"
726 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
727 format
=> 'pve-volume-id',
728 format_description
=> 'volume',
733 description
=> 'Specify a custom file containing all network data passed to the VM via'
735 format
=> 'pve-volume-id',
736 format_description
=> 'volume',
741 description
=> 'Specify a custom file containing all user data passed to the VM via'
743 format
=> 'pve-volume-id',
744 format_description
=> 'volume',
749 description
=> 'Specify a custom file containing all vendor data passed to the VM via'
751 format
=> 'pve-volume-id',
752 format_description
=> 'volume',
755 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
757 my $confdesc_cloudinit = {
761 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
762 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
763 .' and `configdrive2` for windows.',
764 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
769 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
770 ." image's configured default user.",
775 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
776 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
777 .' support hashed passwords.',
782 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
784 format
=> 'pve-qm-cicustom',
789 description
=> 'cloud-init: Sets DNS search domains for a container. Create will'
790 .' automatically use the setting from the host if neither searchdomain nor nameserver'
795 type
=> 'string', format
=> 'address-list',
796 description
=> 'cloud-init: Sets DNS server IP address for a container. Create will'
797 .' automatically use the setting from the host if neither searchdomain nor nameserver'
803 format
=> 'urlencoded',
804 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
808 # what about other qemu settings ?
810 #machine => 'string',
823 ##soundhw => 'string',
825 while (my ($k, $v) = each %$confdesc) {
826 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
829 my $MAX_USB_DEVICES = 14;
831 my $MAX_SERIAL_PORTS = 4;
832 my $MAX_PARALLEL_PORTS = 3;
838 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
839 description
=> "CPUs accessing this NUMA node.",
840 format_description
=> "id[-id];...",
844 description
=> "Amount of memory this NUMA node provides.",
849 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
850 description
=> "Host NUMA nodes to use.",
851 format_description
=> "id[-id];...",
856 enum
=> [qw(preferred bind interleave)],
857 description
=> "NUMA allocation policy.",
861 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
864 type
=> 'string', format
=> $numa_fmt,
865 description
=> "NUMA topology.",
867 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
869 for (my $i = 0; $i < $MAX_NUMA; $i++) {
870 $confdesc->{"numa$i"} = $numadesc;
873 my $nic_model_list = [
889 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
891 my $net_fmt_bridge_descr = <<__EOD__;
892 Bridge to attach the network device to. The Proxmox VE standard bridge
895 If you do not specify a bridge, we create a kvm user (NATed) network
896 device, which provides DHCP and DNS services. The following addresses
903 The DHCP server assign addresses to the guest starting from 10.0.2.15.
907 macaddr
=> get_standard_option
('mac-addr', {
908 description
=> "MAC address. That address must be unique withing your network. This is"
909 ." automatically generated if not specified.",
913 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
914 ." very low CPU overhead. If your guest does not support this driver, it is usually"
915 ." best to use 'e1000'.",
916 enum
=> $nic_model_list,
919 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
920 bridge
=> get_standard_option
('pve-bridge-id', {
921 description
=> $net_fmt_bridge_descr,
926 minimum
=> 0, maximum
=> 64,
927 description
=> 'Number of packet queues to be used on the device.',
933 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
938 minimum
=> 1, maximum
=> 4094,
939 description
=> 'VLAN tag to apply to packets on this interface.',
944 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
945 description
=> 'VLAN trunks to pass through this interface.',
946 format_description
=> 'vlanid[;vlanid...]',
951 description
=> 'Whether this interface should be protected by the firewall.',
956 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
961 minimum
=> 1, maximum
=> 65520,
962 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
969 type
=> 'string', format
=> $net_fmt,
970 description
=> "Specify network devices.",
973 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
978 format
=> 'pve-ipv4-config',
979 format_description
=> 'IPv4Format/CIDR',
980 description
=> 'IPv4 address in CIDR format.',
987 format_description
=> 'GatewayIPv4',
988 description
=> 'Default gateway for IPv4 traffic.',
994 format
=> 'pve-ipv6-config',
995 format_description
=> 'IPv6Format/CIDR',
996 description
=> 'IPv6 address in CIDR format.',
1003 format_description
=> 'GatewayIPv6',
1004 description
=> 'Default gateway for IPv6 traffic.',
1009 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
1010 my $ipconfigdesc = {
1012 type
=> 'string', format
=> 'pve-qm-ipconfig',
1013 description
=> <<'EODESCR',
1014 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1016 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1018 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1019 gateway should be provided.
1020 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1021 cloud-init 19.4 or newer.
1023 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1027 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
1029 for (my $i = 0; $i < $MAX_NETS; $i++) {
1030 $confdesc->{"net$i"} = $netdesc;
1031 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1034 foreach my $key (keys %$confdesc_cloudinit) {
1035 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1038 PVE
::JSONSchema
::register_format
('pve-cpuset', \
&pve_verify_cpuset
);
1039 sub pve_verify_cpuset
{
1040 my ($set_text, $noerr) = @_;
1042 my ($count, $members) = eval { PVE
::CpuSet
::parse_cpuset
($set_text) };
1046 die "unable to parse cpuset option\n";
1049 return PVE
::CpuSet-
>new($members)->short_string();
1052 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1053 sub verify_volume_id_or_qm_path
{
1054 my ($volid, $noerr) = @_;
1056 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1058 return verify_volume_id_or_absolute_path
($volid, $noerr);
1061 PVE
::JSONSchema
::register_format
('pve-volume-id-or-absolute-path', \
&verify_volume_id_or_absolute_path
);
1062 sub verify_volume_id_or_absolute_path
{
1063 my ($volid, $noerr) = @_;
1065 return $volid if $volid =~ m
|^/|;
1067 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1078 type
=> 'string', format
=> 'pve-qm-usb-device',
1079 format_description
=> 'HOSTUSBDEVICE|spice',
1080 description
=> <<EODESCR,
1081 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1083 'bus-port(.port)*' (decimal numbers) or
1084 'vendor_id:product_id' (hexadeciaml numbers) or
1087 You can use the 'lsusb -t' command to list existing usb devices.
1089 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1090 machines - use with special care.
1092 The value 'spice' can be used to add a usb redirection devices for spice.
1098 description
=> "Specifies whether if given host option is a USB3 device or port."
1099 ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1100 ." is irrelevant (all devices are plugged into a xhci controller).",
1107 type
=> 'string', format
=> $usb_fmt,
1108 description
=> "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1109 ." l26 or windows > 7, n can be up to 14).",
1111 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1116 pattern
=> '(/dev/.+|socket)',
1117 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1118 verbose_description
=> <<EODESCR,
1119 Create a serial device inside the VM (n is 0 to 3), and pass through a
1120 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1121 host side (use 'qm terminal' to open a terminal connection).
1123 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1124 use with special care.
1126 CAUTION: Experimental! User reported problems with this option.
1133 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1134 description
=> "Map host parallel devices (n is 0 to 2).",
1135 verbose_description
=> <<EODESCR,
1136 Map host parallel devices (n is 0 to 2).
1138 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1139 machines - use with special care.
1141 CAUTION: Experimental! User reported problems with this option.
1145 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1146 $confdesc->{"parallel$i"} = $paralleldesc;
1149 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1150 $confdesc->{"serial$i"} = $serialdesc;
1153 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1154 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1157 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1158 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1161 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1162 $confdesc->{"usb$i"} = $usbdesc;
1170 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1171 . " Deprecated, use 'order=' instead.",
1172 pattern
=> '[acdn]{1,4}',
1173 format_description
=> "[acdn]{1,4}",
1175 # note: this is also the fallback if boot: is not given at all
1181 format
=> 'pve-qm-bootdev-list',
1182 format_description
=> "device[;device...]",
1183 description
=> <<EODESC,
1184 The guest will attempt to boot from devices in the order they appear here.
1186 Disks, optical drives and passed-through storage USB devices will be directly
1187 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1188 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1190 Note that only devices in this list will be marked as bootable and thus loaded
1191 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1192 (e.g. software-raid), you need to specify all of them here.
1194 Overrides the deprecated 'legacy=[acdn]*' value when given.
1198 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1200 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1201 sub verify_bootdev
{
1202 my ($dev, $noerr) = @_;
1204 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1205 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && !$special;
1209 return 0 if $dev !~ m/^$base\d+$/;
1210 return 0 if !$confdesc->{$dev};
1214 return $dev if $check->("net");
1215 return $dev if $check->("usb");
1216 return $dev if $check->("hostpci");
1219 die "invalid boot device '$dev'\n";
1222 sub print_bootorder
{
1224 return "" if !@$devs;
1225 my $data = { order
=> join(';', @$devs) };
1226 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1229 my $kvm_api_version = 0;
1232 return $kvm_api_version if $kvm_api_version;
1234 open my $fh, '<', '/dev/kvm' or return;
1236 # 0xae00 => KVM_GET_API_VERSION
1237 $kvm_api_version = ioctl($fh, 0xae00, 0);
1240 return $kvm_api_version;
1243 my $kvm_user_version = {};
1246 sub kvm_user_version
{
1249 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1250 my $st = stat($binary);
1252 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1253 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1254 $cachedmtime == $st->mtime;
1256 $kvm_user_version->{$binary} = 'unknown';
1257 $kvm_mtime->{$binary} = $st->mtime;
1261 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1262 $kvm_user_version->{$binary} = $2;
1266 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1269 return $kvm_user_version->{$binary};
1272 my sub extract_version
{
1273 my ($machine_type, $version) = @_;
1274 $version = kvm_user_version
() if !defined($version);
1275 return PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1278 sub kernel_has_vhost_net
{
1279 return -c
'/dev/vhost-net';
1284 return defined($confdesc->{$key});
1288 sub get_cdrom_path
{
1290 return $cdrom_path if $cdrom_path;
1292 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1293 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1294 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1298 my ($storecfg, $vmid, $cdrom) = @_;
1300 if ($cdrom eq 'cdrom') {
1301 return get_cdrom_path
();
1302 } elsif ($cdrom eq 'none') {
1304 } elsif ($cdrom =~ m
|^/|) {
1307 return PVE
::Storage
::path
($storecfg, $cdrom);
1311 # try to convert old style file names to volume IDs
1312 sub filename_to_volume_id
{
1313 my ($vmid, $file, $media) = @_;
1315 if (!($file eq 'none' || $file eq 'cdrom' ||
1316 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1318 return if $file =~ m
|/|;
1320 if ($media && $media eq 'cdrom') {
1321 $file = "local:iso/$file";
1323 $file = "local:$vmid/$file";
1330 sub verify_media_type
{
1331 my ($opt, $vtype, $media) = @_;
1336 if ($media eq 'disk') {
1338 } elsif ($media eq 'cdrom') {
1341 die "internal error";
1344 return if ($vtype eq $etype);
1346 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1349 sub cleanup_drive_path
{
1350 my ($opt, $storecfg, $drive) = @_;
1352 # try to convert filesystem paths to volume IDs
1354 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1355 ($drive->{file
} !~ m
|^/dev/.+|) &&
1356 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1357 ($drive->{file
} !~ m/^\d+$/)) {
1358 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1359 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1361 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1362 verify_media_type
($opt, $vtype, $drive->{media
});
1363 $drive->{file
} = $volid;
1366 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1369 sub parse_hotplug_features
{
1374 return $res if $data eq '0';
1376 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1378 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1379 if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1382 die "invalid hotplug feature '$feature'\n";
1388 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1389 sub pve_verify_hotplug_features
{
1390 my ($value, $noerr) = @_;
1392 return $value if parse_hotplug_features
($value);
1396 die "unable to parse hotplug option\n";
1400 my($fh, $noerr) = @_;
1403 my $SG_GET_VERSION_NUM = 0x2282;
1405 my $versionbuf = "\x00" x
8;
1406 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1408 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1411 my $version = unpack("I", $versionbuf);
1412 if ($version < 30000) {
1413 die "scsi generic interface too old\n" if !$noerr;
1417 my $buf = "\x00" x
36;
1418 my $sensebuf = "\x00" x
8;
1419 my $cmd = pack("C x3 C x1", 0x12, 36);
1421 # see /usr/include/scsi/sg.h
1422 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1425 $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1428 $ret = ioctl($fh, $SG_IO, $packet);
1430 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1434 my @res = unpack($sg_io_hdr_t, $packet);
1435 if ($res[17] || $res[18]) {
1436 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1441 $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1443 $res->{removable
} = $res->{removable
} & 128 ?
1 : 0;
1444 $res->{type
} &= 0x1F;
1452 my $fh = IO
::File-
>new("+<$path") || return;
1453 my $res = scsi_inquiry
($fh, 1);
1459 sub print_tabletdevice_full
{
1460 my ($conf, $arch) = @_;
1462 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1464 # we use uhci for old VMs because tablet driver was buggy in older qemu
1466 if ($q35 || $arch eq 'aarch64') {
1472 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1475 sub print_keyboarddevice_full
{
1476 my ($conf, $arch) = @_;
1478 return if $arch ne 'aarch64';
1480 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1483 my sub get_drive_id
{
1485 return "$drive->{interface}$drive->{index}";
1488 sub print_drivedevice_full
{
1489 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1494 my $drive_id = get_drive_id
($drive);
1495 if ($drive->{interface
} eq 'virtio') {
1496 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1497 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1498 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1499 } elsif ($drive->{interface
} eq 'scsi') {
1501 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1502 my $unit = $drive->{index} % $maxdev;
1503 my $devicetype = 'hd';
1505 if (drive_is_cdrom
($drive)) {
1508 if ($drive->{file
} =~ m
|^/|) {
1509 $path = $drive->{file
};
1510 if (my $info = path_is_scsi
($path)) {
1511 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1512 $devicetype = 'block';
1513 } elsif ($info->{type
} == 1) { # tape
1514 $devicetype = 'generic';
1518 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1521 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1522 my $version = extract_version
($machine_type, kvm_user_version
());
1523 if ($path =~ m/^iscsi\:\/\
// &&
1524 !min_version
($version, 4, 1)) {
1525 $devicetype = 'generic';
1529 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1530 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1532 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1533 .",lun=$drive->{index}";
1535 $device .= ",drive=drive-$drive_id,id=$drive_id";
1537 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1538 $device .= ",rotation_rate=1";
1540 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1542 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1543 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1544 my $controller = int($drive->{index} / $maxdev);
1545 my $unit = $drive->{index} % $maxdev;
1546 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1548 $device = "ide-$devicetype";
1549 if ($drive->{interface
} eq 'ide') {
1550 $device .= ",bus=ide.$controller,unit=$unit";
1552 $device .= ",bus=ahci$controller.$unit";
1554 $device .= ",drive=drive-$drive_id,id=$drive_id";
1556 if ($devicetype eq 'hd') {
1557 if (my $model = $drive->{model
}) {
1558 $model = URI
::Escape
::uri_unescape
($model);
1559 $device .= ",model=$model";
1561 if ($drive->{ssd
}) {
1562 $device .= ",rotation_rate=1";
1565 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1566 } elsif ($drive->{interface
} eq 'usb') {
1568 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1570 die "unsupported interface type";
1573 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1575 if (my $serial = $drive->{serial
}) {
1576 $serial = URI
::Escape
::uri_unescape
($serial);
1577 $device .= ",serial=$serial";
1584 sub get_initiator_name
{
1587 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1588 while (defined(my $line = <$fh>)) {
1589 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1598 sub print_drive_commandline_full
{
1599 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1602 my $volid = $drive->{file
};
1603 my $format = $drive->{format
};
1604 my $drive_id = get_drive_id
($drive);
1606 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1607 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1609 if (drive_is_cdrom
($drive)) {
1610 $path = get_iso_path
($storecfg, $vmid, $volid);
1611 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1614 $path = PVE
::Storage
::path
($storecfg, $volid);
1615 $format //= qemu_img_format
($scfg, $volname);
1622 my $is_rbd = $path =~ m/^rbd:/;
1625 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1626 foreach my $o (@qemu_drive_options) {
1627 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1630 # snapshot only accepts on|off
1631 if (defined($drive->{snapshot
})) {
1632 my $v = $drive->{snapshot
} ?
'on' : 'off';
1633 $opts .= ",snapshot=$v";
1636 if (defined($drive->{ro
})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1637 $opts .= ",readonly=" . ($drive->{ro
} ?
'on' : 'off');
1640 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1641 my ($dir, $qmpname) = @$type;
1642 if (my $v = $drive->{"mbps$dir"}) {
1643 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1645 if (my $v = $drive->{"mbps${dir}_max"}) {
1646 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1648 if (my $v = $drive->{"bps${dir}_max_length"}) {
1649 $opts .= ",throttling.bps$qmpname-max-length=$v";
1651 if (my $v = $drive->{"iops${dir}"}) {
1652 $opts .= ",throttling.iops$qmpname=$v";
1654 if (my $v = $drive->{"iops${dir}_max"}) {
1655 $opts .= ",throttling.iops$qmpname-max=$v";
1657 if (my $v = $drive->{"iops${dir}_max_length"}) {
1658 $opts .= ",throttling.iops$qmpname-max-length=$v";
1663 $format = "rbd" if $is_rbd;
1664 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1666 $opts .= ",format=alloc-track,file.driver=$format";
1668 $opts .= ",format=$format";
1671 my $cache_direct = 0;
1673 if (my $cache = $drive->{cache
}) {
1674 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1675 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1676 $opts .= ",cache=none";
1680 # io_uring with cache mode writeback or writethrough on krbd will hang...
1681 my $rbd_no_io_uring = $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1683 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1684 # sometimes, just plain disable...
1685 my $lvm_no_io_uring = $scfg && $scfg->{type
} eq 'lvm';
1687 # io_uring causes problems when used with CIFS since kernel 5.15
1688 # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1689 my $cifs_no_io_uring = $scfg && $scfg->{type
} eq 'cifs';
1691 if (!$drive->{aio
}) {
1692 if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1693 # io_uring supports all cache modes
1694 $opts .= ",aio=io_uring";
1696 # aio native works only with O_DIRECT
1698 $opts .= ",aio=native";
1700 $opts .= ",aio=threads";
1705 if (!drive_is_cdrom
($drive)) {
1707 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1708 $detectzeroes = 'off';
1709 } elsif ($drive->{discard
}) {
1710 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1712 # This used to be our default with discard not being specified:
1713 $detectzeroes = 'on';
1716 # note: 'detect-zeroes' works per blockdev and we want it to persist
1717 # after the alloc-track is removed, so put it on 'file' directly
1718 my $dz_param = $pbs_name ?
"file.detect-zeroes" : "detect-zeroes";
1719 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1723 $opts .= ",backing=$pbs_name";
1724 $opts .= ",auto-remove=on";
1727 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1728 my $file_param = "file";
1730 # non-rbd drivers require the underlying file to be a seperate block
1731 # node, so add a second .file indirection
1732 $file_param .= ".file" if !$is_rbd;
1733 $file_param .= ".filename";
1735 my $pathinfo = $path ?
"$file_param=$path," : '';
1737 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1740 sub print_pbs_blockdev
{
1741 my ($pbs_conf, $pbs_name) = @_;
1742 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1743 $blockdev .= ",repository=$pbs_conf->{repository}";
1744 $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace
};
1745 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1746 $blockdev .= ",archive=$pbs_conf->{archive}";
1747 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1751 sub print_netdevice_full
{
1752 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1754 my $device = $net->{model
};
1755 if ($net->{model
} eq 'virtio') {
1756 $device = 'virtio-net-pci';
1759 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1760 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1761 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1762 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1763 # and out of each queue plus one config interrupt and control vector queue
1764 my $vectors = $net->{queues
} * 2 + 2;
1765 $tmpstr .= ",vectors=$vectors,mq=on";
1766 if (min_version
($machine_version, 7, 1)) {
1767 $tmpstr .= ",packed=on";
1771 if (min_version
($machine_version, 7, 1) && $net->{model
} eq 'virtio'){
1772 $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1775 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1777 if (my $mtu = $net->{mtu
}) {
1778 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1779 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1782 } elsif ($mtu < 576) {
1783 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1784 } elsif ($mtu > $bridge_mtu) {
1785 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1787 $tmpstr .= ",host_mtu=$mtu";
1789 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1793 if ($use_old_bios_files) {
1795 if ($device eq 'virtio-net-pci') {
1796 $romfile = 'pxe-virtio.rom';
1797 } elsif ($device eq 'e1000') {
1798 $romfile = 'pxe-e1000.rom';
1799 } elsif ($device eq 'e1000e') {
1800 $romfile = 'pxe-e1000e.rom';
1801 } elsif ($device eq 'ne2k') {
1802 $romfile = 'pxe-ne2k_pci.rom';
1803 } elsif ($device eq 'pcnet') {
1804 $romfile = 'pxe-pcnet.rom';
1805 } elsif ($device eq 'rtl8139') {
1806 $romfile = 'pxe-rtl8139.rom';
1808 $tmpstr .= ",romfile=$romfile" if $romfile;
1814 sub print_netdev_full
{
1815 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1818 if ($netid =~ m/^net(\d+)$/) {
1822 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1824 my $ifname = "tap${vmid}i$i";
1826 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1827 die "interface name '$ifname' is too long (max 15 character)\n"
1828 if length($ifname) >= 16;
1830 my $vhostparam = '';
1831 if (is_native
($arch)) {
1832 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1835 my $vmname = $conf->{name
} || "vm$vmid";
1838 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1840 if ($net->{bridge
}) {
1841 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1842 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1844 $netdev = "type=user,id=$netid,hostname=$vmname";
1847 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1853 'cirrus' => 'cirrus-vga',
1855 'vmware' => 'vmware-svga',
1856 'virtio' => 'virtio-vga',
1857 'virtio-gl' => 'virtio-vga-gl',
1860 sub print_vga_device
{
1861 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1863 my $type = $vga_map->{$vga->{type
}};
1864 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1865 $type = 'virtio-gpu';
1867 my $vgamem_mb = $vga->{memory
};
1869 my $max_outputs = '';
1871 $type = $id ?
'qxl' : 'qxl-vga';
1873 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1874 # set max outputs so linux can have up to 4 qxl displays with one device
1875 if (min_version
($machine_version, 4, 1)) {
1876 $max_outputs = ",max_outputs=4";
1881 die "no devicetype for $vga->{type}\n" if !$type;
1885 if ($vga->{type
} =~ /^virtio/) {
1886 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1887 $memory = ",max_hostmem=$bytes";
1889 # from https://www.spice-space.org/multiple-monitors.html
1890 $memory = ",vgamem_mb=$vga->{memory}";
1891 my $ram = $vgamem_mb * 4;
1892 my $vram = $vgamem_mb * 2;
1893 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1895 $memory = ",vgamem_mb=$vga->{memory}";
1897 } elsif ($qxlnum && $id) {
1898 $memory = ",ram_size=67108864,vram_size=33554432";
1902 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1903 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1906 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1907 my $vgaid = "vga" . ($id // '');
1909 if ($q35 && $vgaid eq 'vga') {
1910 # the first display uses pcie.0 bus on q35 machines
1911 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1913 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1916 if ($vga->{type
} eq 'virtio-gl') {
1917 my $base = '/usr/lib/x86_64-linux-gnu/lib';
1918 die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1919 if !-e
"${base}EGL.so.1" || !-e
"${base}GL.so.1";
1921 die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1922 if !PVE
::Tools
::dir_glob_regex
('/dev/dri/', "renderD.*");
1925 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1928 sub parse_number_sets
{
1931 foreach my $part (split(/;/, $set)) {
1932 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1933 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1934 push @$res, [ $1, $2 ];
1936 die "invalid range: $part\n";
1945 my $res = parse_property_string
($numa_fmt, $data);
1946 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1947 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1951 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1953 my ($data, $disable_mac_autogen) = @_;
1955 my $res = eval { parse_property_string
($net_fmt, $data) };
1960 if (!defined($res->{macaddr
}) && !$disable_mac_autogen) {
1961 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1962 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1967 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1968 sub parse_ipconfig
{
1971 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1977 if ($res->{gw
} && !$res->{ip
}) {
1978 warn 'gateway specified without specifying an IP address';
1981 if ($res->{gw6
} && !$res->{ip6
}) {
1982 warn 'IPv6 gateway specified without specifying an IPv6 address';
1985 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1986 warn 'gateway specified together with DHCP';
1989 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1991 warn "IPv6 gateway specified together with $res->{ip6} address";
1995 if (!$res->{ip
} && !$res->{ip6
}) {
1996 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
2005 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
2008 sub add_random_macs
{
2009 my ($settings) = @_;
2011 foreach my $opt (keys %$settings) {
2012 next if $opt !~ m/^net(\d+)$/;
2013 my $net = parse_net
($settings->{$opt});
2015 $settings->{$opt} = print_net
($net);
2019 sub vm_is_volid_owner
{
2020 my ($storecfg, $vmid, $volid) = @_;
2022 if ($volid !~ m
|^/|) {
2024 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
2025 if ($owner && ($owner == $vmid)) {
2033 sub vmconfig_register_unused_drive
{
2034 my ($storecfg, $vmid, $conf, $drive) = @_;
2036 if (drive_is_cloudinit
($drive)) {
2037 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2039 delete $conf->{cloudinit
};
2040 } elsif (!drive_is_cdrom
($drive)) {
2041 my $volid = $drive->{file
};
2042 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2043 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2048 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2052 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2053 format_description
=> 'UUID',
2054 description
=> "Set SMBIOS1 UUID.",
2059 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2060 format_description
=> 'Base64 encoded string',
2061 description
=> "Set SMBIOS1 version.",
2066 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2067 format_description
=> 'Base64 encoded string',
2068 description
=> "Set SMBIOS1 serial number.",
2073 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2074 format_description
=> 'Base64 encoded string',
2075 description
=> "Set SMBIOS1 manufacturer.",
2080 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2081 format_description
=> 'Base64 encoded string',
2082 description
=> "Set SMBIOS1 product ID.",
2087 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2088 format_description
=> 'Base64 encoded string',
2089 description
=> "Set SMBIOS1 SKU string.",
2094 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2095 format_description
=> 'Base64 encoded string',
2096 description
=> "Set SMBIOS1 family string.",
2101 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2109 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2116 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2119 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2121 sub parse_watchdog
{
2126 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2131 sub parse_guest_agent
{
2134 return {} if !defined($conf->{agent
});
2136 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2139 # if the agent is disabled ignore the other potentially set properties
2140 return {} if !$res->{enabled
};
2145 my ($conf, $key) = @_;
2146 return undef if !defined($conf->{agent
});
2148 my $agent = parse_guest_agent
($conf);
2149 return $agent->{$key};
2155 return {} if !$value;
2156 my $res = eval { parse_property_string
($vga_fmt, $value) };
2166 my $res = eval { parse_property_string
($rng_fmt, $value) };
2171 sub parse_meta_info
{
2176 my $res = eval { parse_property_string
($meta_info_fmt, $value) };
2181 sub new_meta_info_string
{
2182 my () = @_; # for now do not allow to override any value
2184 return PVE
::JSONSchema
::print_property_string
(
2186 'creation-qemu' => kvm_user_version
(),
2187 ctime
=> "". int(time()),
2193 sub qemu_created_version_fixups
{
2194 my ($conf, $forcemachine, $kvmver) = @_;
2196 my $meta = parse_meta_info
($conf->{meta
}) // {};
2197 my $forced_vers = PVE
::QemuServer
::Machine
::extract_version
($forcemachine);
2199 # check if we need to apply some handling for VMs that always use the latest machine version but
2200 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2201 # would be required (we do not want to pin machine version for non-windows OS type)
2203 (!defined($conf->{machine
}) || $conf->{machine
} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2204 && (!defined($meta->{'creation-qemu'}) || !min_version
($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2205 && (!$forced_vers || min_version
($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2206 && min_version
($kvmver, 6, 1) # only need to apply the change since 6.1
2208 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2209 if ($q35 && $conf->{ostype
} && $conf->{ostype
} eq 'l26') {
2210 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2211 # and thus with the predictable interface naming of systemd
2212 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2218 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2219 sub verify_usb_device
{
2220 my ($value, $noerr) = @_;
2222 return $value if parse_usb_device
($value);
2226 die "unable to parse usb device\n";
2229 # add JSON properties for create and set function
2230 sub json_config_properties
{
2231 my ($prop, $with_disk_alloc) = @_;
2233 my $skip_json_config_opts = {
2237 runningmachine
=> 1,
2242 foreach my $opt (keys %$confdesc) {
2243 next if $skip_json_config_opts->{$opt};
2245 if ($with_disk_alloc && is_valid_drivename
($opt)) {
2246 $prop->{$opt} = $PVE::QemuServer
::Drive
::drivedesc_hash_with_alloc-
>{$opt};
2248 $prop->{$opt} = $confdesc->{$opt};
2255 # Properties that we can read from an OVF file
2256 sub json_ovf_properties
{
2259 for my $device (PVE
::QemuServer
::Drive
::valid_drive_names
()) {
2260 $prop->{$device} = {
2262 format
=> 'pve-volume-id-or-absolute-path',
2263 description
=> "Disk image that gets imported to $device",
2270 description
=> "The number of CPU cores.",
2275 description
=> "Amount of RAM for the VM in MB.",
2280 description
=> "Name of the VM.",
2287 # return copy of $confdesc_cloudinit to generate documentation
2288 sub cloudinit_config_properties
{
2290 return dclone
($confdesc_cloudinit);
2293 sub cloudinit_pending_properties
{
2295 map { $_ => 1 } keys $confdesc_cloudinit->%*,
2298 $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2303 my ($key, $value) = @_;
2305 die "unknown setting '$key'\n" if !$confdesc->{$key};
2307 my $type = $confdesc->{$key}->{type
};
2309 if (!defined($value)) {
2310 die "got undefined value\n";
2313 if ($value =~ m/[\n\r]/) {
2314 die "property contains a line feed\n";
2317 if ($type eq 'boolean') {
2318 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2319 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2320 die "type check ('boolean') failed - got '$value'\n";
2321 } elsif ($type eq 'integer') {
2322 return int($1) if $value =~ m/^(\d+)$/;
2323 die "type check ('integer') failed - got '$value'\n";
2324 } elsif ($type eq 'number') {
2325 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2326 die "type check ('number') failed - got '$value'\n";
2327 } elsif ($type eq 'string') {
2328 if (my $fmt = $confdesc->{$key}->{format
}) {
2329 PVE
::JSONSchema
::check_format
($fmt, $value);
2332 $value =~ s/^\"(.*)\"$/$1/;
2335 die "internal error"
2340 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2342 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2344 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2346 if ($conf->{template
}) {
2347 # check if any base image is still used by a linked clone
2348 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2349 my ($ds, $drive) = @_;
2350 return if drive_is_cdrom
($drive);
2352 my $volid = $drive->{file
};
2353 return if !$volid || $volid =~ m
|^/|;
2355 die "base volume '$volid' is still in use by linked cloned\n"
2356 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2362 my $remove_owned_drive = sub {
2363 my ($ds, $drive) = @_;
2364 return if drive_is_cdrom
($drive, 1);
2366 my $volid = $drive->{file
};
2367 return if !$volid || $volid =~ m
|^/|;
2368 return if $volids->{$volid};
2370 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2371 return if !$path || !$owner || ($owner != $vmid);
2373 $volids->{$volid} = 1;
2374 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2375 warn "Could not remove disk '$volid', check manually: $@" if $@;
2378 # only remove disks owned by this VM (referenced in the config)
2379 my $include_opts = {
2380 include_unused
=> 1,
2381 extra_keys
=> ['vmstate'],
2383 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2385 for my $snap (values %{$conf->{snapshots
}}) {
2386 next if !defined($snap->{vmstate
});
2387 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2388 next if !defined($drive);
2389 $remove_owned_drive->('vmstate', $drive);
2392 PVE
::QemuConfig-
>foreach_volume_full($conf->{pending
}, $include_opts, $remove_owned_drive);
2394 if ($purge_unreferenced) { # also remove unreferenced disk
2395 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2396 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2397 my ($volid, $sid, $volname, $d) = @_;
2398 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2403 if (defined $replacement_conf) {
2404 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2406 PVE
::QemuConfig-
>destroy_config($vmid);
2410 sub parse_vm_config
{
2411 my ($filename, $raw, $strict) = @_;
2413 return if !defined($raw);
2416 digest
=> Digest
::SHA
::sha1_hex
($raw),
2422 my $handle_error = sub {
2432 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2433 || die "got strange filename '$filename'";
2439 my $finish_description = sub {
2440 if (defined($descr)) {
2442 $conf->{description
} = $descr;
2448 my @lines = split(/\n/, $raw);
2449 foreach my $line (@lines) {
2450 next if $line =~ m/^\s*$/;
2452 if ($line =~ m/^\[PENDING\]\s*$/i) {
2453 $section = 'pending';
2454 $finish_description->();
2455 $conf = $res->{$section} = {};
2457 } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2458 $section = 'cloudinit';
2459 $finish_description->();
2460 $conf = $res->{$section} = {};
2463 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2465 $finish_description->();
2466 $conf = $res->{snapshots
}->{$section} = {};
2470 if ($line =~ m/^\#(.*)$/) {
2471 $descr = '' if !defined($descr);
2472 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2476 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2477 $descr = '' if !defined($descr);
2478 $descr .= PVE
::Tools
::decode_text
($2);
2479 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2480 $conf->{snapstate
} = $1;
2481 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2484 $conf->{$key} = $value;
2485 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2487 if ($section eq 'pending') {
2488 $conf->{delete} = $value; # we parse this later
2490 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2492 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2495 if ($section eq 'cloudinit') {
2496 # ignore validation only used for informative purpose
2497 $conf->{$key} = $value;
2500 eval { $value = check_type
($key, $value); };
2502 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2504 $key = 'ide2' if $key eq 'cdrom';
2505 my $fmt = $confdesc->{$key}->{format
};
2506 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2507 my $v = parse_drive
($key, $value);
2508 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2509 $v->{file
} = $volid;
2510 $value = print_drive
($v);
2512 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2517 $conf->{$key} = $value;
2520 $handle_error->("vm $vmid - unable to parse config: $line\n");
2524 $finish_description->();
2525 delete $res->{snapstate
}; # just to be sure
2530 sub write_vm_config
{
2531 my ($filename, $conf) = @_;
2533 delete $conf->{snapstate
}; # just to be sure
2535 if ($conf->{cdrom
}) {
2536 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2537 $conf->{ide2
} = $conf->{cdrom
};
2538 delete $conf->{cdrom
};
2541 # we do not use 'smp' any longer
2542 if ($conf->{sockets
}) {
2543 delete $conf->{smp
};
2544 } elsif ($conf->{smp
}) {
2545 $conf->{sockets
} = $conf->{smp
};
2546 delete $conf->{cores
};
2547 delete $conf->{smp
};
2550 my $used_volids = {};
2552 my $cleanup_config = sub {
2553 my ($cref, $pending, $snapname) = @_;
2555 foreach my $key (keys %$cref) {
2556 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2557 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2558 my $value = $cref->{$key};
2559 if ($key eq 'delete') {
2560 die "propertry 'delete' is only allowed in [PENDING]\n"
2562 # fixme: check syntax?
2565 eval { $value = check_type
($key, $value); };
2566 die "unable to parse value of '$key' - $@" if $@;
2568 $cref->{$key} = $value;
2570 if (!$snapname && is_valid_drivename
($key)) {
2571 my $drive = parse_drive
($key, $value);
2572 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2577 &$cleanup_config($conf);
2579 &$cleanup_config($conf->{pending
}, 1);
2581 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2582 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2583 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2586 # remove 'unusedX' settings if we re-add a volume
2587 foreach my $key (keys %$conf) {
2588 my $value = $conf->{$key};
2589 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2590 delete $conf->{$key};
2594 my $generate_raw_config = sub {
2595 my ($conf, $pending) = @_;
2599 # add description as comment to top of file
2600 if (defined(my $descr = $conf->{description
})) {
2602 foreach my $cl (split(/\n/, $descr)) {
2603 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2606 $raw .= "#\n" if $pending;
2610 foreach my $key (sort keys %$conf) {
2611 next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2612 $raw .= "$key: $conf->{$key}\n";
2617 my $raw = &$generate_raw_config($conf);
2619 if (scalar(keys %{$conf->{pending
}})){
2620 $raw .= "\n[PENDING]\n";
2621 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2624 if (scalar(keys %{$conf->{cloudinit
}}) && PVE
::QemuConfig-
>has_cloudinit($conf)){
2625 $raw .= "\n[special:cloudinit]\n";
2626 $raw .= &$generate_raw_config($conf->{cloudinit
});
2629 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2630 $raw .= "\n[$snapname]\n";
2631 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2641 # we use static defaults from our JSON schema configuration
2642 foreach my $key (keys %$confdesc) {
2643 if (defined(my $default = $confdesc->{$key}->{default})) {
2644 $res->{$key} = $default;
2652 my $vmlist = PVE
::Cluster
::get_vmlist
();
2654 return $res if !$vmlist || !$vmlist->{ids
};
2655 my $ids = $vmlist->{ids
};
2656 my $nodename = nodename
();
2658 foreach my $vmid (keys %$ids) {
2659 my $d = $ids->{$vmid};
2660 next if !$d->{node
} || $d->{node
} ne $nodename;
2661 next if !$d->{type
} || $d->{type
} ne 'qemu';
2662 $res->{$vmid}->{exists} = 1;
2667 # test if VM uses local resources (to prevent migration)
2668 sub check_local_resources
{
2669 my ($conf, $noerr) = @_;
2673 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2674 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2676 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2678 foreach my $k (keys %$conf) {
2679 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2680 # sockets are safe: they will recreated be on the target side post-migrate
2681 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2682 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2685 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2690 # check if used storages are available on all nodes (use by migrate)
2691 sub check_storage_availability
{
2692 my ($storecfg, $conf, $node) = @_;
2694 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2695 my ($ds, $drive) = @_;
2697 my $volid = $drive->{file
};
2700 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2703 # check if storage is available on both nodes
2704 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2705 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2707 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2709 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2710 if !$scfg->{content
}->{$vtype};
2714 # list nodes where all VM images are available (used by has_feature API)
2716 my ($conf, $storecfg) = @_;
2718 my $nodelist = PVE
::Cluster
::get_nodelist
();
2719 my $nodehash = { map { $_ => 1 } @$nodelist };
2720 my $nodename = nodename
();
2722 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2723 my ($ds, $drive) = @_;
2725 my $volid = $drive->{file
};
2728 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2730 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2731 if ($scfg->{disable
}) {
2733 } elsif (my $avail = $scfg->{nodes
}) {
2734 foreach my $node (keys %$nodehash) {
2735 delete $nodehash->{$node} if !$avail->{$node};
2737 } elsif (!$scfg->{shared
}) {
2738 foreach my $node (keys %$nodehash) {
2739 delete $nodehash->{$node} if $node ne $nodename
2748 sub check_local_storage_availability
{
2749 my ($conf, $storecfg) = @_;
2751 my $nodelist = PVE
::Cluster
::get_nodelist
();
2752 my $nodehash = { map { $_ => {} } @$nodelist };
2754 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2755 my ($ds, $drive) = @_;
2757 my $volid = $drive->{file
};
2760 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2762 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2764 if ($scfg->{disable
}) {
2765 foreach my $node (keys %$nodehash) {
2766 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2768 } elsif (my $avail = $scfg->{nodes
}) {
2769 foreach my $node (keys %$nodehash) {
2770 if (!$avail->{$node}) {
2771 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2778 foreach my $node (values %$nodehash) {
2779 if (my $unavail = $node->{unavailable_storages
}) {
2780 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2787 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2789 my ($vmid, $nocheck, $node) = @_;
2791 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2792 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2797 my $vzlist = config_list
();
2799 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2801 while (defined(my $de = $fd->read)) {
2802 next if $de !~ m/^(\d+)\.pid$/;
2804 next if !defined($vzlist->{$vmid});
2805 if (my $pid = check_running
($vmid)) {
2806 $vzlist->{$vmid}->{pid
} = $pid;
2813 our $vmstatus_return_properties = {
2814 vmid
=> get_standard_option
('pve-vmid'),
2816 description
=> "Qemu process status.",
2818 enum
=> ['stopped', 'running'],
2821 description
=> "Maximum memory in bytes.",
2824 renderer
=> 'bytes',
2827 description
=> "Root disk size in bytes.",
2830 renderer
=> 'bytes',
2833 description
=> "VM name.",
2838 description
=> "Qemu QMP agent status.",
2843 description
=> "PID of running qemu process.",
2848 description
=> "Uptime.",
2851 renderer
=> 'duration',
2854 description
=> "Maximum usable CPUs.",
2859 description
=> "The current config lock, if any.",
2864 description
=> "The current configured tags, if any",
2868 'running-machine' => {
2869 description
=> "The currently running machine type (if running).",
2874 description
=> "The currently running QEMU version (if running).",
2880 my $last_proc_pid_stat;
2882 # get VM status information
2883 # This must be fast and should not block ($full == false)
2884 # We only query KVM using QMP if $full == true (this can be slow)
2886 my ($opt_vmid, $full) = @_;
2890 my $storecfg = PVE
::Storage
::config
();
2892 my $list = vzlist
();
2893 my $defaults = load_defaults
();
2895 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2897 my $cpucount = $cpuinfo->{cpus
} || 1;
2899 foreach my $vmid (keys %$list) {
2900 next if $opt_vmid && ($vmid ne $opt_vmid);
2902 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2904 my $d = { vmid
=> int($vmid) };
2905 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2907 # fixme: better status?
2908 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2910 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2911 if (defined($size)) {
2912 $d->{disk
} = 0; # no info available
2913 $d->{maxdisk
} = $size;
2919 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2920 * ($conf->{cores
} || $defaults->{cores
});
2921 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2922 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2924 $d->{name
} = $conf->{name
} || "VM $vmid";
2925 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2926 : $defaults->{memory
}*(1024*1024);
2928 if ($conf->{balloon
}) {
2929 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2930 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2931 : $defaults->{shares
};
2942 $d->{diskwrite
} = 0;
2944 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2946 $d->{serial
} = 1 if conf_has_serial
($conf);
2947 $d->{lock} = $conf->{lock} if $conf->{lock};
2948 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2953 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2954 foreach my $dev (keys %$netdev) {
2955 next if $dev !~ m/^tap([1-9]\d*)i/;
2957 my $d = $res->{$vmid};
2960 $d->{netout
} += $netdev->{$dev}->{receive
};
2961 $d->{netin
} += $netdev->{$dev}->{transmit
};
2964 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2965 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2970 my $ctime = gettimeofday
;
2972 foreach my $vmid (keys %$list) {
2974 my $d = $res->{$vmid};
2975 my $pid = $d->{pid
};
2978 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2979 next if !$pstat; # not running
2981 my $used = $pstat->{utime} + $pstat->{stime
};
2983 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2985 if ($pstat->{vsize
}) {
2986 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2989 my $old = $last_proc_pid_stat->{$pid};
2991 $last_proc_pid_stat->{$pid} = {
2999 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
3001 if ($dtime > 1000) {
3002 my $dutime = $used - $old->{used
};
3004 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
3005 $last_proc_pid_stat->{$pid} = {
3011 $d->{cpu
} = $old->{cpu
};
3015 return $res if !$full;
3017 my $qmpclient = PVE
::QMPClient-
>new();
3019 my $ballooncb = sub {
3020 my ($vmid, $resp) = @_;
3022 my $info = $resp->{'return'};
3023 return if !$info->{max_mem
};
3025 my $d = $res->{$vmid};
3027 # use memory assigned to VM
3028 $d->{maxmem
} = $info->{max_mem
};
3029 $d->{balloon
} = $info->{actual
};
3031 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
3032 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
3033 $d->{freemem
} = $info->{free_mem
};
3036 $d->{ballooninfo
} = $info;
3039 my $blockstatscb = sub {
3040 my ($vmid, $resp) = @_;
3041 my $data = $resp->{'return'} || [];
3042 my $totalrdbytes = 0;
3043 my $totalwrbytes = 0;
3045 for my $blockstat (@$data) {
3046 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
3047 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
3049 $blockstat->{device
} =~ s/drive-//;
3050 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
3052 $res->{$vmid}->{diskread
} = $totalrdbytes;
3053 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
3056 my $machinecb = sub {
3057 my ($vmid, $resp) = @_;
3058 my $data = $resp->{'return'} || [];
3060 $res->{$vmid}->{'running-machine'} =
3061 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
3064 my $versioncb = sub {
3065 my ($vmid, $resp) = @_;
3066 my $data = $resp->{'return'} // {};
3067 my $version = 'unknown';
3069 if (my $v = $data->{qemu
}) {
3070 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
3073 $res->{$vmid}->{'running-qemu'} = $version;
3076 my $statuscb = sub {
3077 my ($vmid, $resp) = @_;
3079 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3080 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3081 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3082 # this fails if ballon driver is not loaded, so this must be
3083 # the last commnand (following command are aborted if this fails).
3084 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3086 my $status = 'unknown';
3087 if (!defined($status = $resp->{'return'}->{status
})) {
3088 warn "unable to get VM status\n";
3092 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3095 foreach my $vmid (keys %$list) {
3096 next if $opt_vmid && ($vmid ne $opt_vmid);
3097 next if !$res->{$vmid}->{pid
}; # not running
3098 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3101 $qmpclient->queue_execute(undef, 2);
3103 foreach my $vmid (keys %$list) {
3104 next if $opt_vmid && ($vmid ne $opt_vmid);
3105 next if !$res->{$vmid}->{pid
}; #not running
3107 # we can't use the $qmpclient since it might have already aborted on
3108 # 'query-balloon', but this might also fail for older versions...
3109 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
3110 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3113 foreach my $vmid (keys %$list) {
3114 next if $opt_vmid && ($vmid ne $opt_vmid);
3115 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3121 sub conf_has_serial
{
3124 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3125 if ($conf->{"serial$i"}) {
3133 sub conf_has_audio
{
3134 my ($conf, $id) = @_;
3137 my $audio = $conf->{"audio$id"};
3138 return if !defined($audio);
3140 my $audioproperties = parse_property_string
($audio_fmt, $audio);
3141 my $audiodriver = $audioproperties->{driver
} // 'spice';
3144 dev
=> $audioproperties->{device
},
3145 dev_id
=> "audiodev$id",
3146 backend
=> $audiodriver,
3147 backend_id
=> "$audiodriver-backend${id}",
3152 my ($audio, $audiopciaddr, $machine_version) = @_;
3156 my $id = $audio->{dev_id
};
3158 if (min_version
($machine_version, 4, 2)) {
3159 $audiodev = ",audiodev=$audio->{backend_id}";
3162 if ($audio->{dev
} eq 'AC97') {
3163 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3164 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3165 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3166 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3167 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3169 die "unkown audio device '$audio->{dev}', implement me!";
3172 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3180 socket => "/var/run/qemu-server/$vmid.swtpm",
3181 pid
=> "/var/run/qemu-server/$vmid.swtpm.pid",
3185 sub add_tpm_device
{
3186 my ($vmid, $devices, $conf) = @_;
3188 return if !$conf->{tpmstate0
};
3190 my $paths = get_tpm_paths
($vmid);
3192 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3193 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3194 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3198 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3200 return if !$tpmdrive;
3203 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
3204 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
3206 $state = PVE
::Storage
::map_volume
($storecfg, $tpm->{file
});
3208 $state = $tpm->{file
};
3211 my $paths = get_tpm_paths
($vmid);
3213 # during migration, we will get state from remote
3216 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3223 "--create-platform-cert",
3226 "/etc/swtpm_setup.conf", # do not use XDG configs
3228 "0", # force creation as root, error if not possible
3229 "--not-overwrite", # ignore existing state, do not modify
3232 push @$setup_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3233 # TPM 2.0 supports ECC crypto, use if possible
3234 push @$setup_cmd, "--ecc" if $tpm->{version
} eq 'v2.0';
3236 run_command
($setup_cmd, outfunc
=> sub {
3237 print "swtpm_setup: $1\n";
3241 my $emulator_cmd = [
3245 "backend-uri=file://$state,mode=0600",
3247 "type=unixio,path=$paths->{socket},mode=0600",
3249 "file=$paths->{pid}",
3250 "--terminate", # terminate on QEMU disconnect
3253 push @$emulator_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3254 run_command
($emulator_cmd, outfunc
=> sub { print $1; });
3256 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3257 while (! -e
$paths->{pid
}) {
3258 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3262 # return untainted PID of swtpm daemon so it can be killed on error
3263 file_read_firstline
($paths->{pid
}) =~ m/(\d+)/;
3267 sub vga_conf_has_spice
{
3270 my $vgaconf = parse_vga
($vga);
3271 my $vgatype = $vgaconf->{type
};
3272 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3279 return get_host_arch
() eq $arch;
3284 return $conf->{arch
} // get_host_arch
();
3287 my $default_machines = {
3292 sub get_installed_machine_version
{
3293 my ($kvmversion) = @_;
3294 $kvmversion = kvm_user_version
() if !defined($kvmversion);
3295 $kvmversion =~ m/^(\d+\.\d+)/;
3299 sub windows_get_pinned_machine_version
{
3300 my ($machine, $base_version, $kvmversion) = @_;
3302 my $pin_version = $base_version;
3303 if (!defined($base_version) ||
3304 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3306 $pin_version = get_installed_machine_version
($kvmversion);
3308 if (!$machine || $machine eq 'pc') {
3309 $machine = "pc-i440fx-$pin_version";
3310 } elsif ($machine eq 'q35') {
3311 $machine = "pc-q35-$pin_version";
3312 } elsif ($machine eq 'virt') {
3313 $machine = "virt-$pin_version";
3315 warn "unknown machine type '$machine', not touching that!\n";
3321 sub get_vm_machine
{
3322 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3324 my $machine = $forcemachine || $conf->{machine
};
3326 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3327 $kvmversion //= kvm_user_version
();
3328 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3329 # layout which confuses windows quite a bit and may result in various regressions..
3330 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3331 if (windows_version
($conf->{ostype
})) {
3332 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3335 $machine ||= $default_machines->{$arch};
3336 if ($add_pve_version) {
3337 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3338 $machine .= "+pve$pvever";
3342 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3343 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3344 $machine = $1 if $is_pxe;
3346 # for version-pinned machines that do not include a pve-version (e.g.
3347 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3348 $machine .= '+pve0';
3350 $machine .= '.pxe' if $is_pxe;
3356 sub get_ovmf_files
($$$) {
3357 my ($arch, $efidisk, $smm) = @_;
3359 my $types = $OVMF->{$arch}
3360 or die "no OVMF images known for architecture '$arch'\n";
3362 my $type = 'default';
3363 if (defined($efidisk->{efitype
}) && $efidisk->{efitype
} eq '4m') {
3364 $type = $smm ?
"4m" : "4m-no-smm";
3365 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3368 return $types->{$type}->@*;
3372 aarch64
=> '/usr/bin/qemu-system-aarch64',
3373 x86_64
=> '/usr/bin/qemu-system-x86_64',
3375 sub get_command_for_arch
($) {
3377 return '/usr/bin/kvm' if is_native
($arch);
3379 my $cmd = $Arch2Qemu->{$arch}
3380 or die "don't know how to emulate architecture '$arch'\n";
3384 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3385 # to use in a QEMU command line (-cpu element), first array_intersect the result
3386 # of query_supported_ with query_understood_. This is necessary because:
3388 # a) query_understood_ returns flags the host cannot use and
3389 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3390 # flags, but CPU settings - with most of them being flags. Those settings
3391 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3393 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3394 # expensive. If you need the value returned from this, you can get it much
3395 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3396 # $accel being 'kvm' or 'tcg'.
3398 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3399 # changes, automatically populating pmxcfs.
3401 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3402 # since kvm and tcg machines support different flags
3404 sub query_supported_cpu_flags
{
3407 $arch //= get_host_arch
();
3408 my $default_machine = $default_machines->{$arch};
3412 # FIXME: Once this is merged, the code below should work for ARM as well:
3413 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3414 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3417 my $kvm_supported = defined(kvm_version
());
3418 my $qemu_cmd = get_command_for_arch
($arch);
3420 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3422 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3423 my $query_supported_run_qemu = sub {
3429 '-machine', $default_machine,
3431 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3432 '-mon', 'chardev=qmp,mode=control',
3433 '-pidfile', $pidfile,
3438 push @$cmd, '-accel', 'tcg';
3441 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3442 die "QEMU flag querying VM exited with code " . $rc if $rc;
3445 my $cmd_result = mon_cmd
(
3447 'query-cpu-model-expansion',
3449 model
=> { name
=> 'host' }
3452 my $props = $cmd_result->{model
}->{props
};
3453 foreach my $prop (keys %$props) {
3454 next if $props->{$prop} ne '1';
3455 # QEMU returns some flags multiple times, with '_', '.' or '-'
3456 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3457 # We only keep those with underscores, to match /proc/cpuinfo
3458 $prop =~ s/\.|-/_/g;
3459 $flags->{$prop} = 1;
3464 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3465 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3469 return [ sort keys %$flags ];
3472 # We need to query QEMU twice, since KVM and TCG have different supported flags
3473 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3474 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3475 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3477 if ($kvm_supported) {
3478 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3479 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3486 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3487 my $understood_cpu_flag_dir = "/usr/share/kvm";
3488 sub query_understood_cpu_flags
{
3489 my $arch = get_host_arch
();
3490 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3492 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3495 my $raw = file_get_contents
($filepath);
3496 $raw =~ s/^\s+|\s+$//g;
3497 my @flags = split(/\s+/, $raw);
3502 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3503 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3504 my sub should_disable_smm
{
3505 my ($conf, $vga) = @_;
3507 return (!defined($conf->{bios
}) || $conf->{bios
} eq 'seabios') &&
3508 $vga->{type
} && $vga->{type
} =~ m/^(serial\d+|none)$/;
3511 sub config_to_command
{
3512 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3515 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3518 my $ostype = $conf->{ostype
};
3519 my $winversion = windows_version
($ostype);
3520 my $kvm = $conf->{kvm
};
3521 my $nodename = nodename
();
3523 my $arch = get_vm_arch
($conf);
3524 my $kvm_binary = get_command_for_arch
($arch);
3525 my $kvmver = kvm_user_version
($kvm_binary);
3527 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3528 $kvmver //= "undefined";
3529 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3532 my $add_pve_version = min_version
($kvmver, 4, 1);
3534 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3535 my $machine_version = extract_version
($machine_type, $kvmver);
3536 $kvm //= 1 if is_native
($arch);
3538 $machine_version =~ m/(\d+)\.(\d+)/;
3539 my ($machine_major, $machine_minor) = ($1, $2);
3541 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3542 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3543 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3544 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3545 ." please upgrade node '$nodename'\n"
3546 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3547 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3548 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3549 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3550 ." node '$nodename'\n";
3553 # if a specific +pve version is required for a feature, use $version_guard
3554 # instead of min_version to allow machines to be run with the minimum
3556 my $required_pve_version = 0;
3557 my $version_guard = sub {
3558 my ($major, $minor, $pve) = @_;
3559 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3560 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3561 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3562 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3566 if ($kvm && !defined kvm_version
()) {
3567 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3568 ." or enable in BIOS.\n";
3571 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3572 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3573 my $use_old_bios_files = undef;
3574 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3577 if ($conf->{affinity
}) {
3578 push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity
};
3581 push @$cmd, $kvm_binary;
3583 push @$cmd, '-id', $vmid;
3585 my $vmname = $conf->{name
} || "vm$vmid";
3587 push @$cmd, '-name', "$vmname,debug-threads=on";
3589 push @$cmd, '-no-shutdown';
3593 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3594 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3595 push @$cmd, '-mon', "chardev=qmp,mode=control";
3597 if (min_version
($machine_version, 2, 12)) {
3598 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3599 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3602 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3604 push @$cmd, '-daemonize';
3606 if ($conf->{smbios1
}) {
3607 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3608 if ($smbios_conf->{base64
}) {
3609 # Do not pass base64 flag to qemu
3610 delete $smbios_conf->{base64
};
3611 my $smbios_string = "";
3612 foreach my $key (keys %$smbios_conf) {
3614 if ($key eq "uuid") {
3615 $value = $smbios_conf->{uuid
}
3617 $value = decode_base64
($smbios_conf->{$key});
3619 # qemu accepts any binary data, only commas need escaping by double comma
3621 $smbios_string .= "," . $key . "=" . $value if $value;
3623 push @$cmd, '-smbios', "type=1" . $smbios_string;
3625 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3629 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3631 if (my $efidisk = $conf->{efidisk0
}) {
3632 $d = parse_drive
('efidisk0', $efidisk);
3635 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch, $d, $q35);
3636 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3638 my ($path, $format);
3639 my $read_only_str = '';
3641 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3642 $format = $d->{format
};
3644 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3645 if (!defined($format)) {
3646 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3647 $format = qemu_img_format
($scfg, $volname);
3651 die "efidisk format must be specified\n"
3652 if !defined($format);
3655 $read_only_str = ',readonly=on' if drive_is_read_only
($conf, $d);
3657 log_warn
("no efidisk configured! Using temporary efivars disk.");
3658 $path = "/tmp/$vmid-ovmf.fd";
3659 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3665 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3666 $size_str = ",size=" . (-s
$ovmf_vars);
3669 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3671 if ($path =~ m/^rbd:/) {
3672 $cache = ',cache=writeback';
3673 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3676 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3677 push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3680 if ($q35) { # tell QEMU to load q35 config early
3681 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3682 if (min_version
($machine_version, 4, 0)) {
3683 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3685 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3689 if (defined(my $fixups = qemu_created_version_fixups
($conf, $forcemachine, $kvmver))) {
3690 push @$cmd, $fixups->@*;
3693 if ($conf->{vmgenid
}) {
3694 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3697 # add usb controllers
3698 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3699 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES, $machine_version);
3700 push @$devices, @usbcontrollers if @usbcontrollers;
3701 my $vga = parse_vga
($conf->{vga
});
3703 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3704 $vga->{type
} = 'qxl' if $qxlnum;
3706 if (!$vga->{type
}) {
3707 if ($arch eq 'aarch64') {
3708 $vga->{type
} = 'virtio';
3709 } elsif (min_version
($machine_version, 2, 9)) {
3710 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3712 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3716 # enable absolute mouse coordinates (needed by vnc)
3717 my $tablet = $conf->{tablet
};
3718 if (!defined($tablet)) {
3719 $tablet = $defaults->{tablet
};
3720 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3721 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3725 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3726 my $kbd = print_keyboarddevice_full
($conf, $arch);
3727 push @$devices, '-device', $kbd if defined($kbd);
3730 my $bootorder = device_bootorder
($conf);
3732 # host pci device passthrough
3733 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3734 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3737 my $usb_dev_features = {};
3738 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3740 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3741 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3742 push @$devices, @usbdevices if @usbdevices;
3745 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3746 my $path = $conf->{"serial$i"} or next;
3747 if ($path eq 'socket') {
3748 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3749 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3750 # On aarch64, serial0 is the UART device. Qemu only allows
3751 # connecting UART devices via the '-serial' command line, as
3752 # the device has a fixed slot on the hardware...
3753 if ($arch eq 'aarch64' && $i == 0) {
3754 push @$devices, '-serial', "chardev:serial$i";
3756 push @$devices, '-device', "isa-serial,chardev=serial$i";
3759 die "no such serial device\n" if ! -c
$path;
3760 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3761 push @$devices, '-device', "isa-serial,chardev=serial$i";
3766 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3767 if (my $path = $conf->{"parallel$i"}) {
3768 die "no such parallel device\n" if ! -c
$path;
3769 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3770 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3771 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3775 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3776 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3777 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3778 push @$devices, @$audio_devs;
3781 add_tpm_device
($vmid, $devices, $conf);
3784 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3785 $sockets = $conf->{sockets
} if $conf->{sockets
};
3787 my $cores = $conf->{cores
} || 1;
3789 my $maxcpus = $sockets * $cores;
3791 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3793 my $allowed_vcpus = $cpuinfo->{cpus
};
3795 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3797 if ($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3798 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3799 for (my $i = 2; $i <= $vcpus; $i++) {
3800 my $cpustr = print_cpu_device
($conf,$i);
3801 push @$cmd, '-device', $cpustr;
3806 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3808 push @$cmd, '-nodefaults';
3810 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3812 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3814 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3816 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3817 push @$devices, '-device', print_vga_device
(
3818 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3820 push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type
} eq 'virtio-gl'; # VIRGL
3822 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3823 push @$cmd, '-vnc', "unix:$socket,password=on";
3825 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3826 push @$cmd, '-nographic';
3830 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3831 my $useLocaltime = $conf->{localtime};
3833 if ($winversion >= 5) { # windows
3834 $useLocaltime = 1 if !defined($conf->{localtime});
3836 # use time drift fix when acpi is enabled
3837 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3838 $tdf = 1 if !defined($conf->{tdf
});
3842 if ($winversion >= 6) {
3843 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3844 push @$cmd, '-no-hpet';
3847 push @$rtcFlags, 'driftfix=slew' if $tdf;
3849 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3850 push @$rtcFlags, "base=$conf->{startdate}";
3851 } elsif ($useLocaltime) {
3852 push @$rtcFlags, 'base=localtime';
3856 push @$cmd, '-cpu', $forcecpu;
3858 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3861 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3863 push @$cmd, '-S' if $conf->{freeze
};
3865 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3867 my $guest_agent = parse_guest_agent
($conf);
3869 if ($guest_agent->{enabled
}) {
3870 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3871 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3873 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3874 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3875 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3876 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3877 } elsif ($guest_agent->{type
} eq 'isa') {
3878 push @$devices, '-device', "isa-serial,chardev=qga0";
3882 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3883 if ($rng && $version_guard->(4, 1, 2)) {
3884 check_rng_source
($rng->{source
});
3886 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3887 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3888 my $limiter_str = "";
3890 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3893 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3894 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3895 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3900 if ($qxlnum || $vga->{type
} =~ /^virtio/) {
3903 for (my $i = 1; $i < $qxlnum; $i++){
3904 push @$devices, '-device', print_vga_device
(
3905 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3908 # assume other OS works like Linux
3909 my ($ram, $vram) = ("134217728", "67108864");
3910 if ($vga->{memory
}) {
3911 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3912 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3914 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3915 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3919 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3921 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3922 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3923 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3925 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3926 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3927 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3929 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3930 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3932 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3933 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3934 if ($spice_enhancement->{foldersharing
}) {
3935 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3936 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3939 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3940 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3941 if $spice_enhancement->{videostreaming
};
3943 push @$devices, '-spice', "$spice_opts";
3946 # enable balloon by default, unless explicitly disabled
3947 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3948 my $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3949 my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3950 $ballooncmd .= ",free-page-reporting=on" if min_version
($machine_version, 6, 2);
3951 push @$devices, '-device', $ballooncmd;
3954 if ($conf->{watchdog
}) {
3955 my $wdopts = parse_watchdog
($conf->{watchdog
});
3956 my $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3957 my $watchdog = $wdopts->{model
} || 'i6300esb';
3958 push @$devices, '-device', "$watchdog$pciaddr";
3959 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3963 my $scsicontroller = {};
3964 my $ahcicontroller = {};
3965 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3967 # Add iscsi initiator name if available
3968 if (my $initiator = get_initiator_name
()) {
3969 push @$devices, '-iscsi', "initiator-name=$initiator";
3972 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3973 my ($ds, $drive) = @_;
3975 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3976 check_volume_storage_type
($storecfg, $drive->{file
});
3977 push @$vollist, $drive->{file
};
3980 # ignore efidisk here, already added in bios/fw handling code above
3981 return if $drive->{interface
} eq 'efidisk';
3983 return if $drive->{interface
} eq 'tpmstate';
3985 $use_virtio = 1 if $ds =~ m/^virtio/;
3987 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3989 if ($drive->{interface
} eq 'virtio'){
3990 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3993 if ($drive->{interface
} eq 'scsi') {
3995 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3997 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3998 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4000 my $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
4001 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
4004 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
4005 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4006 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4007 } elsif ($drive->{iothread
}) {
4009 "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4014 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
4015 $queues = ",num_queues=$drive->{queues}";
4018 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4019 if !$scsicontroller->{$controller};
4020 $scsicontroller->{$controller}=1;
4023 if ($drive->{interface
} eq 'sata') {
4024 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
4025 my $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
4026 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4027 if !$ahcicontroller->{$controller};
4028 $ahcicontroller->{$controller}=1;
4031 my $pbs_conf = $pbs_backing->{$ds};
4032 my $pbs_name = undef;
4034 $pbs_name = "drive-$ds-pbs";
4035 push @$devices, '-blockdev', print_pbs_blockdev
($pbs_conf, $pbs_name);
4038 my $drive_cmd = print_drive_commandline_full
(
4039 $storecfg, $vmid, $drive, $pbs_name, min_version
($kvmver, 6, 0));
4041 # extra protection for templates, but SATA and IDE don't support it..
4042 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
4044 push @$devices, '-drive',$drive_cmd;
4045 push @$devices, '-device', print_drivedevice_full
(
4046 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4049 for (my $i = 0; $i < $MAX_NETS; $i++) {
4050 my $netname = "net$i";
4052 next if !$conf->{$netname};
4053 my $d = parse_net
($conf->{$netname});
4055 # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4057 $use_virtio = 1 if $d->{model
} eq 'virtio';
4059 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
4061 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
4062 push @$devices, '-netdev', $netdevfull;
4064 my $netdevicefull = print_netdevice_full
(
4065 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4067 push @$devices, '-device', $netdevicefull;
4070 if ($conf->{ivshmem
}) {
4071 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4075 $bus = print_pcie_addr
("ivshmem");
4077 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4080 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4081 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4083 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4084 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4085 .",size=$ivshmem->{size}M";
4088 # pci.4 is nested in pci.1
4089 $bridges->{1} = 1 if $bridges->{4};
4091 if (!$q35) { # add pci bridges
4092 if (min_version
($machine_version, 2, 3)) {
4096 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4099 for my $k (sort {$b cmp $a} keys %$bridges) {
4100 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4103 if ($k == 2 && $legacy_igd) {
4106 my $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
4107 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4109 if ($q35) { # add after -readconfig pve-q35.cfg
4110 splice @$devices, 2, 0, '-device', $devstr;
4112 unshift @$devices, '-device', $devstr if $k > 0;
4117 push @$machineFlags, 'accel=tcg';
4120 push @$machineFlags, 'smm=off' if should_disable_smm
($conf, $vga);
4122 my $machine_type_min = $machine_type;
4123 if ($add_pve_version) {
4124 $machine_type_min =~ s/\+pve\d+$//;
4125 $machine_type_min .= "+pve$required_pve_version";
4127 push @$machineFlags, "type=${machine_type_min}";
4129 push @$cmd, @$devices;
4130 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4131 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4132 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4134 if (my $vmstate = $conf->{vmstate
}) {
4135 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4136 push @$vollist, $vmstate;
4137 push @$cmd, '-loadstate', $statepath;
4138 print "activating and using '$vmstate' as vmstate\n";
4141 if (PVE
::QemuConfig-
>is_template($conf)) {
4142 # needed to workaround base volumes being read-only
4143 push @$cmd, '-snapshot';
4147 if ($conf->{args
}) {
4148 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4152 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
4155 sub check_rng_source
{
4158 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4159 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4162 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4163 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
4164 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4165 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4166 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4167 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4175 my $res = mon_cmd
($vmid, 'query-spice');
4177 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4180 sub vm_devices_list
{
4183 my $res = mon_cmd
($vmid, 'query-pci');
4184 my $devices_to_check = [];
4186 foreach my $pcibus (@$res) {
4187 push @$devices_to_check, @{$pcibus->{devices
}},
4190 while (@$devices_to_check) {
4192 for my $d (@$devices_to_check) {
4193 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4194 next if !$d->{'pci_bridge'};
4196 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4197 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4199 $devices_to_check = $to_check;
4202 my $resblock = mon_cmd
($vmid, 'query-block');
4203 foreach my $block (@$resblock) {
4204 if($block->{device
} =~ m/^drive-(\S+)/){
4209 my $resmice = mon_cmd
($vmid, 'query-mice');
4210 foreach my $mice (@$resmice) {
4211 if ($mice->{name
} eq 'QEMU HID Tablet') {
4212 $devices->{tablet
} = 1;
4217 # for usb devices there is no query-usb
4218 # but we can iterate over the entries in
4219 # qom-list path=/machine/peripheral
4220 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4221 foreach my $per (@$resperipheral) {
4222 if ($per->{name
} =~ m/^usb(?:redirdev)?\d+$/) {
4223 $devices->{$per->{name
}} = 1;
4231 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4233 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4235 my $devices_list = vm_devices_list
($vmid);
4236 return 1 if defined($devices_list->{$deviceid});
4238 # add PCI bridge if we need it for the device
4239 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4241 if ($deviceid eq 'tablet') {
4242 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4243 } elsif ($deviceid eq 'keyboard') {
4244 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4245 } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4247 qemu_spice_usbredir_chardev_add
($vmid, "usbredirchardev$id");
4248 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_spice_usbdevice
($id, "xhci", $id + 1));
4249 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4250 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device, {}, $1 + 1));
4251 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4252 qemu_iothread_add
($vmid, $deviceid, $device);
4254 qemu_driveadd
($storecfg, $vmid, $device);
4255 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4257 qemu_deviceadd
($vmid, $devicefull);
4258 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4260 eval { qemu_drivedel
($vmid, $deviceid); };
4264 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4265 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4266 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4267 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4269 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4271 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4272 qemu_iothread_add
($vmid, $deviceid, $device);
4273 $devicefull .= ",iothread=iothread-$deviceid";
4276 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4277 $devicefull .= ",num_queues=$device->{queues}";
4280 qemu_deviceadd
($vmid, $devicefull);
4281 qemu_deviceaddverify
($vmid, $deviceid);
4282 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4283 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4284 qemu_driveadd
($storecfg, $vmid, $device);
4286 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4287 eval { qemu_deviceadd
($vmid, $devicefull); };
4289 eval { qemu_drivedel
($vmid, $deviceid); };
4293 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4294 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4296 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4297 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type);
4298 my $use_old_bios_files = undef;
4299 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4301 my $netdevicefull = print_netdevice_full
(
4302 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4303 qemu_deviceadd
($vmid, $netdevicefull);
4305 qemu_deviceaddverify
($vmid, $deviceid);
4306 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4309 eval { qemu_netdevdel
($vmid, $deviceid); };
4313 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4315 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4316 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4318 qemu_deviceadd
($vmid, $devicefull);
4319 qemu_deviceaddverify
($vmid, $deviceid);
4321 die "can't hotplug device '$deviceid'\n";
4327 # fixme: this should raise exceptions on error!
4328 sub vm_deviceunplug
{
4329 my ($vmid, $conf, $deviceid) = @_;
4331 my $devices_list = vm_devices_list
($vmid);
4332 return 1 if !defined($devices_list->{$deviceid});
4334 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4335 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4337 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4338 qemu_devicedel
($vmid, $deviceid);
4339 } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4340 qemu_devicedel
($vmid, $deviceid);
4341 qemu_devicedelverify
($vmid, $deviceid);
4342 } elsif ($deviceid =~ m/^usb\d+$/) {
4343 qemu_devicedel
($vmid, $deviceid);
4344 qemu_devicedelverify
($vmid, $deviceid);
4345 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4346 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4348 qemu_devicedel
($vmid, $deviceid);
4349 qemu_devicedelverify
($vmid, $deviceid);
4350 qemu_drivedel
($vmid, $deviceid);
4351 qemu_iothread_del
($vmid, $deviceid, $device);
4352 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4353 qemu_devicedel
($vmid, $deviceid);
4354 qemu_devicedelverify
($vmid, $deviceid);
4355 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4356 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4358 qemu_devicedel
($vmid, $deviceid);
4359 qemu_devicedelverify
($vmid, $deviceid);
4360 qemu_drivedel
($vmid, $deviceid);
4361 qemu_deletescsihw
($conf, $vmid, $deviceid);
4363 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4364 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4365 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4366 qemu_devicedel
($vmid, $deviceid);
4367 qemu_devicedelverify
($vmid, $deviceid);
4368 qemu_netdevdel
($vmid, $deviceid);
4370 die "can't unplug device '$deviceid'\n";
4376 sub qemu_spice_usbredir_chardev_add
{
4377 my ($vmid, $id) = @_;
4379 mon_cmd
($vmid, "chardev-add" , (
4390 sub qemu_deviceadd
{
4391 my ($vmid, $devicefull) = @_;
4393 $devicefull = "driver=".$devicefull;
4394 my %options = split(/[=,]/, $devicefull);
4396 mon_cmd
($vmid, "device_add" , %options);
4399 sub qemu_devicedel
{
4400 my ($vmid, $deviceid) = @_;
4402 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4405 sub qemu_iothread_add
{
4406 my ($vmid, $deviceid, $device) = @_;
4408 if ($device->{iothread
}) {
4409 my $iothreads = vm_iothreads_list
($vmid);
4410 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4414 sub qemu_iothread_del
{
4415 my ($vmid, $deviceid, $device) = @_;
4417 if ($device->{iothread
}) {
4418 my $iothreads = vm_iothreads_list
($vmid);
4419 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4423 sub qemu_objectadd
{
4424 my ($vmid, $objectid, $qomtype) = @_;
4426 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4431 sub qemu_objectdel
{
4432 my ($vmid, $objectid) = @_;
4434 mon_cmd
($vmid, "object-del", id
=> $objectid);
4440 my ($storecfg, $vmid, $device) = @_;
4442 my $kvmver = get_running_qemu_version
($vmid);
4443 my $io_uring = min_version
($kvmver, 6, 0);
4444 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4445 $drive =~ s/\\/\\\\/g;
4446 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4448 # If the command succeeds qemu prints: "OK
"
4449 return 1 if $ret =~ m/OK/s;
4451 die "adding drive failed
: $ret\n";
4455 my ($vmid, $deviceid) = @_;
4457 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4460 return 1 if $ret eq "";
4462 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4463 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4465 die "deleting drive
$deviceid failed
: $ret\n";
4468 sub qemu_deviceaddverify {
4469 my ($vmid, $deviceid) = @_;
4471 for (my $i = 0; $i <= 5; $i++) {
4472 my $devices_list = vm_devices_list($vmid);
4473 return 1 if defined($devices_list->{$deviceid});
4477 die "error on hotplug device
'$deviceid'\n";
4481 sub qemu_devicedelverify {
4482 my ($vmid, $deviceid) = @_;
4484 # need to verify that the device is correctly removed as device_del
4485 # is async and empty return is not reliable
4487 for (my $i = 0; $i <= 5; $i++) {
4488 my $devices_list = vm_devices_list($vmid);
4489 return 1 if !defined($devices_list->{$deviceid});
4493 die "error on hot-unplugging device
'$deviceid'\n";
4496 sub qemu_findorcreatescsihw {
4497 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4499 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4501 my $scsihwid="$controller_prefix$controller";
4502 my $devices_list = vm_devices_list($vmid);
4504 if (!defined($devices_list->{$scsihwid})) {
4505 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4511 sub qemu_deletescsihw {
4512 my ($conf, $vmid, $opt) = @_;
4514 my $device = parse_drive($opt, $conf->{$opt});
4516 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4517 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4521 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4523 my $devices_list = vm_devices_list($vmid);
4524 foreach my $opt (keys %{$devices_list}) {
4525 if (is_valid_drivename($opt)) {
4526 my $drive = parse_drive($opt, $conf->{$opt});
4527 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4533 my $scsihwid="scsihw
$controller";
4535 vm_deviceunplug($vmid, $conf, $scsihwid);
4540 sub qemu_add_pci_bridge {
4541 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4547 print_pci_addr($device, $bridges, $arch, $machine_type);
4549 while (my ($k, $v) = each %$bridges) {
4552 return 1 if !defined($bridgeid) || $bridgeid < 1;
4554 my $bridge = "pci
.$bridgeid";
4555 my $devices_list = vm_devices_list($vmid);
4557 if (!defined($devices_list->{$bridge})) {
4558 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4564 sub qemu_set_link_status {
4565 my ($vmid, $device, $up) = @_;
4567 mon_cmd($vmid, "set_link
", name => $device,
4568 up => $up ? JSON::true : JSON::false);
4571 sub qemu_netdevadd {
4572 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4574 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4575 my %options = split(/[=,]/, $netdev);
4577 if (defined(my $vhost = $options{vhost})) {
4578 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4581 if (defined(my $queues = $options{queues})) {
4582 $options{queues} = $queues + 0;
4585 mon_cmd($vmid, "netdev_add
", %options);
4589 sub qemu_netdevdel {
4590 my ($vmid, $deviceid) = @_;
4592 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4595 sub qemu_usb_hotplug {
4596 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4600 # remove the old one first
4601 vm_deviceunplug($vmid, $conf, $deviceid);
4603 # check if xhci controller is necessary and available
4604 my $devicelist = vm_devices_list($vmid);
4606 if (!$devicelist->{xhci}) {
4607 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4608 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4611 # print_usbdevice_full expects the parsed device
4612 my $d = parse_usb_device($device->{host});
4613 $d->{usb3} = $device->{usb3};
4616 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4619 sub qemu_cpu_hotplug {
4620 my ($vmid, $conf, $vcpus) = @_;
4622 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4625 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4626 $sockets = $conf->{sockets} if $conf->{sockets};
4627 my $cores = $conf->{cores} || 1;
4628 my $maxcpus = $sockets * $cores;
4630 $vcpus = $maxcpus if !$vcpus;
4632 die "you can
't add more vcpus than maxcpus\n"
4633 if $vcpus > $maxcpus;
4635 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4637 if ($vcpus < $currentvcpus) {
4639 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4641 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4642 qemu_devicedel($vmid, "cpu$i");
4644 my $currentrunningvcpus = undef;
4646 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4647 last if scalar(@{$currentrunningvcpus}) == $i-1;
4648 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4652 #update conf after each succesfull cpu unplug
4653 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4654 PVE::QemuConfig->write_config($vmid, $conf);
4657 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4663 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4664 die "vcpus in running vm does not match its configuration\n"
4665 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4667 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4669 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4670 my $cpustr = print_cpu_device($conf, $i);
4671 qemu_deviceadd($vmid, $cpustr);
4674 my $currentrunningvcpus = undef;
4676 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4677 last if scalar(@{$currentrunningvcpus}) == $i;
4678 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4682 #update conf after each succesfull cpu hotplug
4683 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4684 PVE::QemuConfig->write_config($vmid, $conf);
4688 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4689 mon_cmd($vmid, "cpu-add", id => int($i));
4694 sub qemu_block_set_io_throttle {
4695 my ($vmid, $deviceid,
4696 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4697 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4698 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4699 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4701 return if !check_running($vmid) ;
4703 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4705 bps_rd => int($bps_rd),
4706 bps_wr => int($bps_wr),
4708 iops_rd => int($iops_rd),
4709 iops_wr => int($iops_wr),
4710 bps_max => int($bps_max),
4711 bps_rd_max => int($bps_rd_max),
4712 bps_wr_max => int($bps_wr_max),
4713 iops_max => int($iops_max),
4714 iops_rd_max => int($iops_rd_max),
4715 iops_wr_max => int($iops_wr_max),
4716 bps_max_length => int($bps_max_length),
4717 bps_rd_max_length => int($bps_rd_max_length),
4718 bps_wr_max_length => int($bps_wr_max_length),
4719 iops_max_length => int($iops_max_length),
4720 iops_rd_max_length => int($iops_rd_max_length),
4721 iops_wr_max_length => int($iops_wr_max_length),
4726 sub qemu_block_resize {
4727 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4729 my $running = check_running($vmid);
4731 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4733 return if !$running;
4735 my $padding = (1024 - $size % 1024) % 1024;
4736 $size = $size + $padding;
4741 device => $deviceid,
4747 sub qemu_volume_snapshot {
4748 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4750 my $running = check_running($vmid);
4752 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4753 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4755 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4759 sub qemu_volume_snapshot_delete {
4760 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4762 my $running = check_running($vmid);
4767 my $conf = PVE::QemuConfig->load_config($vmid);
4768 PVE::QemuConfig->foreach_volume($conf, sub {
4769 my ($ds, $drive) = @_;
4770 $running = 1 if $drive->{file} eq $volid;
4774 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4775 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4777 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4781 sub set_migration_caps {
4782 my ($vmid, $savevm) = @_;
4784 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4786 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4787 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4792 "auto-converge" => 1,
4794 "x-rdma-pin-all" => 0,
4797 "dirty-bitmaps" => $dirty_bitmaps,
4800 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4802 for my $supported_capability (@$supported_capabilities) {
4804 capability => $supported_capability->{capability},
4805 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4809 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4813 my ($conf, $func, @param) = @_;
4817 my $test_volid = sub {
4818 my ($key, $drive, $snapname) = @_;
4820 my $volid = $drive->{file};
4823 $volhash->{$volid}->{cdrom} //= 1;
4824 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4826 my $replicate = $drive->{replicate} // 1;
4827 $volhash->{$volid}->{replicate} //= 0;
4828 $volhash->{$volid}->{replicate} = 1 if $replicate;
4830 $volhash->{$volid}->{shared} //= 0;
4831 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4833 $volhash->{$volid}->{referenced_in_config} //= 0;
4834 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4836 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4837 if defined($snapname);
4839 my $size = $drive->{size};
4840 $volhash->{$volid}->{size} //= $size if $size;
4842 $volhash->{$volid}->{is_vmstate} //= 0;
4843 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4845 $volhash->{$volid}->{is_tpmstate} //= 0;
4846 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0
';
4848 $volhash->{$volid}->{is_unused} //= 0;
4849 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4851 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4854 my $include_opts = {
4855 extra_keys => ['vmstate
'],
4856 include_unused => 1,
4859 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4860 foreach my $snapname (keys %{$conf->{snapshots}}) {
4861 my $snap = $conf->{snapshots}->{$snapname};
4862 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4865 foreach my $volid (keys %$volhash) {
4866 &$func($volid, $volhash->{$volid}, @param);
4870 my $fast_plug_option = {
4878 'vmstatestorage
' => 1,
4883 for my $opt (keys %$confdesc_cloudinit) {
4884 $fast_plug_option->{$opt} = 1;
4887 # hotplug changes in [PENDING]
4888 # $selection hash can be used to only apply specified options, for
4889 # example: { cores => 1 } (only apply changed 'cores
')
4890 # $errors ref is used to return error messages
4891 sub vmconfig_hotplug_pending {
4892 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4894 my $defaults = load_defaults();
4895 my $arch = get_vm_arch($conf);
4896 my $machine_type = get_vm_machine($conf, undef, $arch);
4898 # commit values which do not have any impact on running VM first
4899 # Note: those option cannot raise errors, we we do not care about
4900 # $selection and always apply them.
4902 my $add_error = sub {
4903 my ($opt, $msg) = @_;
4904 $errors->{$opt} = "hotplug problem - $msg";
4907 my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4909 my $cloudinit_record_changed = sub {
4910 my ($conf, $opt, $old, $new) = @_;
4911 return if !$cloudinit_pending_properties->{$opt};
4913 my $ci = ($conf->{cloudinit} //= {});
4915 my $recorded = $ci->{$opt};
4916 my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4918 if (defined($new)) {
4919 if (defined($old)) {
4920 # an existing value is being modified
4921 if (defined($recorded)) {
4922 # the value was already not in sync
4923 if ($new eq $recorded) {
4924 # a value is being reverted to the cloud-init state:
4926 delete $added{$opt};
4928 # the value was changed multiple times, do nothing
4930 } elsif ($added{$opt}) {
4931 # the value had been marked as added and is being changed, do nothing
4933 # the value is new, record it:
4937 # a new value is being added
4938 if (defined($recorded)) {
4939 # it was already not in sync
4940 if ($new eq $recorded) {
4941 # a value is being reverted to the cloud-init state:
4943 delete $added{$opt};
4945 # the value had temporarily been removed, do nothing
4947 } elsif ($added{$opt}) {
4948 # the value had been marked as added already, do nothing
4950 # the value is new, add it
4954 } elsif (!defined($old)) {
4955 # a non-existent value is being removed? ignore...
4957 # a value is being deleted
4958 if (defined($recorded)) {
4959 # a value was already recorded, just keep it
4960 } elsif ($added{$opt}) {
4961 # the value was marked as added, remove it
4962 delete $added{$opt};
4964 # a previously unrecorded value is being removed, record the old value:
4969 my $added = join(',', sort keys %added);
4970 $ci->{added} = $added if length($added);
4974 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4975 if ($fast_plug_option->{$opt}) {
4976 my $new = delete $conf->{pending}->{$opt};
4977 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4978 $conf->{$opt} = $new;
4984 PVE::QemuConfig->write_config($vmid, $conf);
4987 my $ostype = $conf->{ostype};
4988 my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4989 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4990 my $usb_hotplug = $hotplug_features->{usb}
4991 && min_version($version, 7, 1)
4992 && defined($ostype) && ($ostype eq 'l26
' || windows_version($ostype) > 7);
4994 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4995 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4997 foreach my $opt (sort keys %$pending_delete_hash) {
4998 next if $selection && !$selection->{$opt};
4999 my $force = $pending_delete_hash->{$opt}->{force};
5001 if ($opt eq 'hotplug
') {
5002 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5003 } elsif ($opt eq 'tablet
') {
5004 die "skip\n" if !$hotplug_features->{usb};
5005 if ($defaults->{tablet}) {
5006 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
5007 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
5008 if $arch eq 'aarch64
';
5010 vm_deviceunplug($vmid, $conf, 'tablet
');
5011 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
5013 } elsif ($opt =~ m/^usb(\d+)$/) {
5015 die "skip\n" if !$usb_hotplug;
5016 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5017 vm_deviceunplug
($vmid, $conf, $opt);
5018 } elsif ($opt eq 'vcpus') {
5019 die "skip\n" if !$hotplug_features->{cpu
};
5020 qemu_cpu_hotplug
($vmid, $conf, undef);
5021 } elsif ($opt eq 'balloon') {
5022 # enable balloon device is not hotpluggable
5023 die "skip\n" if defined($conf->{balloon
}) && $conf->{balloon
} == 0;
5024 # here we reset the ballooning value to memory
5025 my $balloon = $conf->{memory
} || $defaults->{memory
};
5026 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5027 } elsif ($fast_plug_option->{$opt}) {
5029 } elsif ($opt =~ m/^net(\d+)$/) {
5030 die "skip\n" if !$hotplug_features->{network
};
5031 vm_deviceunplug
($vmid, $conf, $opt);
5032 } elsif (is_valid_drivename
($opt)) {
5033 die "skip\n" if !$hotplug_features->{disk
} || $opt =~ m/(ide|sata)(\d+)/;
5034 vm_deviceunplug
($vmid, $conf, $opt);
5035 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5036 } elsif ($opt =~ m/^memory$/) {
5037 die "skip\n" if !$hotplug_features->{memory
};
5038 PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $defaults, $opt);
5039 } elsif ($opt eq 'cpuunits') {
5040 $cgroup->change_cpu_shares(undef);
5041 } elsif ($opt eq 'cpulimit') {
5042 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5048 &$add_error($opt, $err) if $err ne "skip\n";
5050 my $old = delete $conf->{$opt};
5051 $cloudinit_record_changed->($conf, $opt, $old, undef);
5052 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5057 foreach my $opt (keys %{$conf->{pending
}}) {
5058 next if $selection && !$selection->{$opt};
5059 my $value = $conf->{pending
}->{$opt};
5061 if ($opt eq 'hotplug') {
5062 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug
} =~ /memory/);
5063 } elsif ($opt eq 'tablet') {
5064 die "skip\n" if !$hotplug_features->{usb
};
5066 vm_deviceplug
($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5067 vm_deviceplug
($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5068 if $arch eq 'aarch64';
5069 } elsif ($value == 0) {
5070 vm_deviceunplug
($vmid, $conf, 'tablet');
5071 vm_deviceunplug
($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5073 } elsif ($opt =~ m/^usb(\d+)$/) {
5075 die "skip\n" if !$usb_hotplug;
5076 my $d = eval { parse_property_string
($usbdesc->{format
}, $value) };
5078 if ($d->{host
} eq 'spice') {
5079 $id = "usbredirdev$index";
5081 qemu_usb_hotplug
($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5082 } elsif ($opt eq 'vcpus') {
5083 die "skip\n" if !$hotplug_features->{cpu
};
5084 qemu_cpu_hotplug
($vmid, $conf, $value);
5085 } elsif ($opt eq 'balloon') {
5086 # enable/disable balloning device is not hotpluggable
5087 my $old_balloon_enabled = !!(!defined($conf->{balloon
}) || $conf->{balloon
});
5088 my $new_balloon_enabled = !!(!defined($conf->{pending
}->{balloon
}) || $conf->{pending
}->{balloon
});
5089 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5091 # allow manual ballooning if shares is set to zero
5092 if ((defined($conf->{shares
}) && ($conf->{shares
} == 0))) {
5093 my $balloon = $conf->{pending
}->{balloon
} || $conf->{memory
} || $defaults->{memory
};
5094 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5096 } elsif ($opt =~ m/^net(\d+)$/) {
5097 # some changes can be done without hotplug
5098 vmconfig_update_net
($storecfg, $conf, $hotplug_features->{network
},
5099 $vmid, $opt, $value, $arch, $machine_type);
5100 } elsif (is_valid_drivename
($opt)) {
5101 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5102 # some changes can be done without hotplug
5103 my $drive = parse_drive
($opt, $value);
5104 if (drive_is_cloudinit
($drive)) {
5105 $cloudinit_opt = [$opt, $drive];
5106 # apply all the other changes first, then generate the cloudinit disk
5109 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5110 $vmid, $opt, $value, $arch, $machine_type);
5111 } elsif ($opt =~ m/^memory$/) { #dimms
5112 die "skip\n" if !$hotplug_features->{memory
};
5113 $value = PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $defaults, $opt, $value);
5114 } elsif ($opt eq 'cpuunits') {
5115 my $new_cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{pending
}->{$opt}); #clamp
5116 $cgroup->change_cpu_shares($new_cpuunits);
5117 } elsif ($opt eq 'cpulimit') {
5118 my $cpulimit = $conf->{pending
}->{$opt} == 0 ?
-1 : int($conf->{pending
}->{$opt} * 100000);
5119 $cgroup->change_cpu_quota($cpulimit, 100000);
5120 } elsif ($opt eq 'agent') {
5121 vmconfig_update_agent
($conf, $opt, $value);
5123 die "skip\n"; # skip non-hot-pluggable options
5127 &$add_error($opt, $err) if $err ne "skip\n";
5129 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5130 $conf->{$opt} = $value;
5131 delete $conf->{pending
}->{$opt};
5135 if (defined($cloudinit_opt)) {
5136 my ($opt, $drive) = @$cloudinit_opt;
5137 my $value = $conf->{pending
}->{$opt};
5139 my $temp = {%$conf, $opt => $value};
5140 PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($temp, $vmid);
5141 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5142 $vmid, $opt, $value, $arch, $machine_type);
5145 &$add_error($opt, $err) if $err ne "skip\n";
5147 $conf->{$opt} = $value;
5148 delete $conf->{pending
}->{$opt};
5152 # unplug xhci controller if no usb device is left
5155 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5156 next if !defined($conf->{"usb$i"});
5161 vm_deviceunplug
($vmid, $conf, 'xhci');
5165 PVE
::QemuConfig-
>write_config($vmid, $conf);
5167 if ($hotplug_features->{cloudinit
} && PVE
::QemuServer
::Cloudinit
::has_changes
($conf)) {
5168 PVE
::QemuServer
::vmconfig_update_cloudinit_drive
($storecfg, $conf, $vmid);
5172 sub try_deallocate_drive
{
5173 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5175 if (($force || $key =~ /^unused/) && !drive_is_cdrom
($drive, 1)) {
5176 my $volid = $drive->{file
};
5177 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
5178 my $sid = PVE
::Storage
::parse_volume_id
($volid);
5179 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5181 # check if the disk is really unused
5182 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5183 if PVE
::QemuServer
::Drive
::is_volume_in_use
($storecfg, $conf, $key, $volid);
5184 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5187 # If vm is not owner of this disk remove from config
5195 sub vmconfig_delete_or_detach_drive
{
5196 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5198 my $drive = parse_drive
($opt, $conf->{$opt});
5200 my $rpcenv = PVE
::RPCEnvironment
::get
();
5201 my $authuser = $rpcenv->get_user();
5204 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5205 try_deallocate_drive
($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5207 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $drive);
5213 sub vmconfig_apply_pending
{
5214 my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5216 return if !scalar(keys %{$conf->{pending
}});
5218 my $add_apply_error = sub {
5219 my ($opt, $msg) = @_;
5220 my $err_msg = "unable to apply pending change $opt : $msg";
5221 $errors->{$opt} = $err_msg;
5227 my $pending_delete_hash = PVE
::QemuConfig-
>parse_pending_delete($conf->{pending
}->{delete});
5228 foreach my $opt (sort keys %$pending_delete_hash) {
5229 my $force = $pending_delete_hash->{$opt}->{force
};
5231 if ($opt =~ m/^unused/) {
5232 die "internal error";
5233 } elsif (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5234 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5238 $add_apply_error->($opt, $err);
5240 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5241 delete $conf->{$opt};
5245 PVE
::QemuConfig-
>cleanup_pending($conf);
5247 my $generate_cloudinit = $skip_cloud_init ?
0 : undef;
5249 foreach my $opt (keys %{$conf->{pending
}}) { # add/change
5250 next if $opt eq 'delete'; # just to be sure
5252 if (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5253 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, parse_drive
($opt, $conf->{$opt}))
5257 $add_apply_error->($opt, $err);
5260 if (is_valid_drivename
($opt)) {
5261 my $drive = parse_drive
($opt, $conf->{pending
}->{$opt});
5262 $generate_cloudinit //= 1 if drive_is_cloudinit
($drive);
5265 $conf->{$opt} = delete $conf->{pending
}->{$opt};
5269 # write all changes at once to avoid unnecessary i/o
5270 PVE
::QemuConfig-
>write_config($vmid, $conf);
5271 if ($generate_cloudinit) {
5272 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5273 # After successful generation and if there were changes to be applied, update the
5274 # config to drop the {cloudinit} entry.
5275 PVE
::QemuConfig-
>write_config($vmid, $conf);
5280 sub vmconfig_update_net
{
5281 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5283 my $newnet = parse_net
($value);
5285 if ($conf->{$opt}) {
5286 my $oldnet = parse_net
($conf->{$opt});
5288 if (safe_string_ne
($oldnet->{model
}, $newnet->{model
}) ||
5289 safe_string_ne
($oldnet->{macaddr
}, $newnet->{macaddr
}) ||
5290 safe_num_ne
($oldnet->{queues
}, $newnet->{queues
}) ||
5291 !($newnet->{bridge
} && $oldnet->{bridge
})) { # bridge/nat mode change
5293 # for non online change, we try to hot-unplug
5294 die "skip\n" if !$hotplug;
5295 vm_deviceunplug
($vmid, $conf, $opt);
5298 die "internal error" if $opt !~ m/net(\d+)/;
5299 my $iface = "tap${vmid}i$1";
5301 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5302 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
}) ||
5303 safe_string_ne
($oldnet->{trunks
}, $newnet->{trunks
}) ||
5304 safe_num_ne
($oldnet->{firewall
}, $newnet->{firewall
})) {
5305 PVE
::Network
::tap_unplug
($iface);
5308 PVE
::Network
::SDN
::Zones
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5310 PVE
::Network
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5312 } elsif (safe_num_ne
($oldnet->{rate
}, $newnet->{rate
})) {
5313 # Rate can be applied on its own but any change above needs to
5314 # include the rate in tap_plug since OVS resets everything.
5315 PVE
::Network
::tap_rate_limit
($iface, $newnet->{rate
});
5318 if (safe_string_ne
($oldnet->{link_down
}, $newnet->{link_down
})) {
5319 qemu_set_link_status
($vmid, $opt, !$newnet->{link_down
});
5327 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5333 sub vmconfig_update_agent
{
5334 my ($conf, $opt, $value) = @_;
5336 die "skip\n" if !$conf->{$opt};
5338 my $hotplug_options = { fstrim_cloned_disks
=> 1 };
5340 my $old_agent = parse_guest_agent
($conf);
5341 my $agent = parse_guest_agent
({$opt => $value});
5343 for my $option (keys %$agent) { # added/changed options
5344 next if defined($hotplug_options->{$option});
5345 die "skip\n" if safe_string_ne
($agent->{$option}, $old_agent->{$option});
5348 for my $option (keys %$old_agent) { # removed options
5349 next if defined($hotplug_options->{$option});
5350 die "skip\n" if safe_string_ne
($old_agent->{$option}, $agent->{$option});
5353 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5356 sub vmconfig_update_disk
{
5357 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5359 my $drive = parse_drive
($opt, $value);
5361 if ($conf->{$opt} && (my $old_drive = parse_drive
($opt, $conf->{$opt}))) {
5362 my $media = $drive->{media
} || 'disk';
5363 my $oldmedia = $old_drive->{media
} || 'disk';
5364 die "unable to change media type\n" if $media ne $oldmedia;
5366 if (!drive_is_cdrom
($old_drive)) {
5368 if ($drive->{file
} ne $old_drive->{file
}) {
5370 die "skip\n" if !$hotplug;
5372 # unplug and register as unused
5373 vm_deviceunplug
($vmid, $conf, $opt);
5374 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive)
5377 # update existing disk
5379 # skip non hotpluggable value
5380 if (safe_string_ne
($drive->{discard
}, $old_drive->{discard
}) ||
5381 safe_string_ne
($drive->{iothread
}, $old_drive->{iothread
}) ||
5382 safe_string_ne
($drive->{queues
}, $old_drive->{queues
}) ||
5383 safe_string_ne
($drive->{cache
}, $old_drive->{cache
}) ||
5384 safe_string_ne
($drive->{ssd
}, $old_drive->{ssd
}) ||
5385 safe_string_ne
($drive->{ro
}, $old_drive->{ro
})) {
5390 if (safe_num_ne
($drive->{mbps
}, $old_drive->{mbps
}) ||
5391 safe_num_ne
($drive->{mbps_rd
}, $old_drive->{mbps_rd
}) ||
5392 safe_num_ne
($drive->{mbps_wr
}, $old_drive->{mbps_wr
}) ||
5393 safe_num_ne
($drive->{iops
}, $old_drive->{iops
}) ||
5394 safe_num_ne
($drive->{iops_rd
}, $old_drive->{iops_rd
}) ||
5395 safe_num_ne
($drive->{iops_wr
}, $old_drive->{iops_wr
}) ||
5396 safe_num_ne
($drive->{mbps_max
}, $old_drive->{mbps_max
}) ||
5397 safe_num_ne
($drive->{mbps_rd_max
}, $old_drive->{mbps_rd_max
}) ||
5398 safe_num_ne
($drive->{mbps_wr_max
}, $old_drive->{mbps_wr_max
}) ||
5399 safe_num_ne
($drive->{iops_max
}, $old_drive->{iops_max
}) ||
5400 safe_num_ne
($drive->{iops_rd_max
}, $old_drive->{iops_rd_max
}) ||
5401 safe_num_ne
($drive->{iops_wr_max
}, $old_drive->{iops_wr_max
}) ||
5402 safe_num_ne
($drive->{bps_max_length
}, $old_drive->{bps_max_length
}) ||
5403 safe_num_ne
($drive->{bps_rd_max_length
}, $old_drive->{bps_rd_max_length
}) ||
5404 safe_num_ne
($drive->{bps_wr_max_length
}, $old_drive->{bps_wr_max_length
}) ||
5405 safe_num_ne
($drive->{iops_max_length
}, $old_drive->{iops_max_length
}) ||
5406 safe_num_ne
($drive->{iops_rd_max_length
}, $old_drive->{iops_rd_max_length
}) ||
5407 safe_num_ne
($drive->{iops_wr_max_length
}, $old_drive->{iops_wr_max_length
})) {
5409 qemu_block_set_io_throttle
(
5411 ($drive->{mbps
} || 0)*1024*1024,
5412 ($drive->{mbps_rd
} || 0)*1024*1024,
5413 ($drive->{mbps_wr
} || 0)*1024*1024,
5414 $drive->{iops
} || 0,
5415 $drive->{iops_rd
} || 0,
5416 $drive->{iops_wr
} || 0,
5417 ($drive->{mbps_max
} || 0)*1024*1024,
5418 ($drive->{mbps_rd_max
} || 0)*1024*1024,
5419 ($drive->{mbps_wr_max
} || 0)*1024*1024,
5420 $drive->{iops_max
} || 0,
5421 $drive->{iops_rd_max
} || 0,
5422 $drive->{iops_wr_max
} || 0,
5423 $drive->{bps_max_length
} || 1,
5424 $drive->{bps_rd_max_length
} || 1,
5425 $drive->{bps_wr_max_length
} || 1,
5426 $drive->{iops_max_length
} || 1,
5427 $drive->{iops_rd_max_length
} || 1,
5428 $drive->{iops_wr_max_length
} || 1,
5438 if ($drive->{file
} eq 'none') {
5439 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5440 if (drive_is_cloudinit
($old_drive)) {
5441 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive);
5444 my $path = get_iso_path
($storecfg, $vmid, $drive->{file
});
5446 # force eject if locked
5447 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5450 mon_cmd
($vmid, "blockdev-change-medium",
5451 id
=> "$opt", filename
=> "$path");
5459 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5461 PVE
::Storage
::activate_volumes
($storecfg, [$drive->{file
}]) if $drive->{file
} !~ m
|^/dev/.+|;
5462 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5465 sub vmconfig_update_cloudinit_drive
{
5466 my ($storecfg, $conf, $vmid) = @_;
5468 my $cloudinit_ds = undef;
5469 my $cloudinit_drive = undef;
5471 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5472 my ($ds, $drive) = @_;
5473 if (PVE
::QemuServer
::drive_is_cloudinit
($drive)) {
5474 $cloudinit_ds = $ds;
5475 $cloudinit_drive = $drive;
5479 return if !$cloudinit_drive;
5481 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5482 PVE
::QemuConfig-
>write_config($vmid, $conf);
5485 my $running = PVE
::QemuServer
::check_running
($vmid);
5488 my $path = PVE
::Storage
::path
($storecfg, $cloudinit_drive->{file
});
5490 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$cloudinit_ds");
5491 mon_cmd
($vmid, "blockdev-change-medium", id
=> "$cloudinit_ds", filename
=> "$path");
5496 # called in locked context by incoming migration
5497 sub vm_migrate_get_nbd_disks
{
5498 my ($storecfg, $conf, $replicated_volumes) = @_;
5500 my $local_volumes = {};
5501 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5502 my ($ds, $drive) = @_;
5504 return if drive_is_cdrom
($drive);
5505 return if $ds eq 'tpmstate0';
5507 my $volid = $drive->{file
};
5511 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid);
5513 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5514 return if $scfg->{shared
};
5516 # replicated disks re-use existing state via bitmap
5517 my $use_existing = $replicated_volumes->{$volid} ?
1 : 0;
5518 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5520 return $local_volumes;
5523 # called in locked context by incoming migration
5524 sub vm_migrate_alloc_nbd_disks
{
5525 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5528 foreach my $opt (sort keys %$source_volumes) {
5529 my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5531 if ($use_existing) {
5532 $nbd->{$opt}->{drivestr
} = print_drive
($drive);
5533 $nbd->{$opt}->{volid
} = $volid;
5534 $nbd->{$opt}->{replicated
} = 1;
5538 # storage mapping + volname = regular migration
5539 # storage mapping + format = remote migration
5540 # order of precedence, filtered by whether storage supports it:
5541 # 1. explicit requested format
5542 # 2. format of current volume
5543 # 3. default format of storage
5544 if (!$storagemap->{identity
}) {
5545 $storeid = PVE
::JSONSchema
::map_id
($storagemap, $storeid);
5546 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5547 if (!$format || !grep { $format eq $_ } @$validFormats) {
5549 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5550 my $fileFormat = qemu_img_format
($scfg, $volname);
5551 $format = $fileFormat
5552 if grep { $fileFormat eq $_ } @$validFormats;
5554 $format //= $defFormat;
5557 # can't happen for remote migration, so $volname is always defined
5558 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5559 $format = qemu_img_format
($scfg, $volname);
5562 my $size = $drive->{size
} / 1024;
5563 my $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $format, undef, $size);
5564 my $newdrive = $drive;
5565 $newdrive->{format
} = $format;
5566 $newdrive->{file
} = $newvolid;
5567 my $drivestr = print_drive
($newdrive);
5568 $nbd->{$opt}->{drivestr
} = $drivestr;
5569 $nbd->{$opt}->{volid
} = $newvolid;
5575 # see vm_start_nolock for parameters, additionally:
5577 # storagemap = parsed storage map for allocating NBD disks
5579 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5581 return PVE
::QemuConfig-
>lock_config($vmid, sub {
5582 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migrate_opts->{migratedfrom
});
5584 die "you can't start a vm if it's a template\n"
5585 if !$params->{skiptemplate
} && PVE
::QemuConfig-
>is_template($conf);
5587 my $has_suspended_lock = PVE
::QemuConfig-
>has_lock($conf, 'suspended');
5588 my $has_backup_lock = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5590 my $running = check_running
($vmid, undef, $migrate_opts->{migratedfrom
});
5592 if ($has_backup_lock && $running) {
5593 # a backup is currently running, attempt to start the guest in the
5594 # existing QEMU instance
5595 return vm_resume
($vmid);
5598 PVE
::QemuConfig-
>check_lock($conf)
5599 if !($params->{skiplock
} || $has_suspended_lock);
5601 $params->{resume
} = $has_suspended_lock || defined($conf->{vmstate
});
5603 die "VM $vmid already running\n" if $running;
5605 if (my $storagemap = $migrate_opts->{storagemap
}) {
5606 my $replicated = $migrate_opts->{replicated_volumes
};
5607 my $disks = vm_migrate_get_nbd_disks
($storecfg, $conf, $replicated);
5608 $migrate_opts->{nbd
} = vm_migrate_alloc_nbd_disks
($storecfg, $vmid, $disks, $storagemap);
5610 foreach my $opt (keys %{$migrate_opts->{nbd
}}) {
5611 $conf->{$opt} = $migrate_opts->{nbd
}->{$opt}->{drivestr
};
5615 return vm_start_nolock
($storecfg, $vmid, $conf, $params, $migrate_opts);
5621 # statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5622 # skiplock => 0/1, skip checking for config lock
5623 # skiptemplate => 0/1, skip checking whether VM is template
5624 # forcemachine => to force Qemu machine (rollback/migration)
5625 # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5626 # timeout => in seconds
5627 # paused => start VM in paused state (backup)
5628 # resume => resume from hibernation
5639 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5640 # migratedfrom => source node
5641 # spice_ticket => used for spice migration, passed via tunnel/stdin
5642 # network => CIDR of migration network
5643 # type => secure/insecure - tunnel over encrypted connection or plain-text
5644 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5645 # replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5646 # offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5647 # contained in config
5648 sub vm_start_nolock
{
5649 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5651 my $statefile = $params->{statefile
};
5652 my $resume = $params->{resume
};
5654 my $migratedfrom = $migrate_opts->{migratedfrom
};
5655 my $migration_type = $migrate_opts->{type
};
5659 # clean up leftover reboot request files
5660 eval { clear_reboot_request
($vmid); };
5663 if (!$statefile && scalar(keys %{$conf->{pending
}})) {
5664 vmconfig_apply_pending
($vmid, $conf, $storecfg);
5665 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5668 # don't regenerate the ISO if the VM is started as part of a live migration
5669 # this way we can reuse the old ISO with the correct config
5670 if (!$migratedfrom) {
5671 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5672 # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5673 # $conf->{cloudinit}, so we could just not do this?
5674 # But we do it above, so for now let's be consistent.
5675 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5679 # override offline migrated volumes, conf is out of date still
5680 if (my $offline_volumes = $migrate_opts->{offline_volumes
}) {
5681 for my $key (sort keys $offline_volumes->%*) {
5682 my $parsed = parse_drive
($key, $conf->{$key});
5683 $parsed->{file
} = $offline_volumes->{$key};
5684 $conf->{$key} = print_drive
($parsed);
5688 my $defaults = load_defaults
();
5690 # set environment variable useful inside network script
5691 # for remote migration the config is available on the target node!
5692 if (!$migrate_opts->{remote_node
}) {
5693 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom;
5696 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5698 my $forcemachine = $params->{forcemachine
};
5699 my $forcecpu = $params->{forcecpu
};
5701 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5702 $forcemachine = $conf->{runningmachine
};
5703 $forcecpu = $conf->{runningcpu
};
5704 print "Resuming suspended VM\n";
5707 my ($cmd, $vollist, $spice_port) = config_to_command
($storecfg, $vmid,
5708 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5711 my $get_migration_ip = sub {
5712 my ($nodename) = @_;
5714 return $migration_ip if defined($migration_ip);
5716 my $cidr = $migrate_opts->{network
};
5718 if (!defined($cidr)) {
5719 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5720 $cidr = $dc_conf->{migration
}->{network
};
5723 if (defined($cidr)) {
5724 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5726 die "could not get IP: no address configured on local " .
5727 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5729 die "could not get IP: multiple addresses configured on local " .
5730 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5732 $migration_ip = @$ips[0];
5735 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5736 if !defined($migration_ip);
5738 return $migration_ip;
5742 if ($statefile eq 'tcp') {
5743 my $migrate = $res->{migrate
} = { proto
=> 'tcp' };
5744 $migrate->{addr
} = "localhost";
5745 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5746 my $nodename = nodename
();
5748 if (!defined($migration_type)) {
5749 if (defined($datacenterconf->{migration
}->{type
})) {
5750 $migration_type = $datacenterconf->{migration
}->{type
};
5752 $migration_type = 'secure';
5756 if ($migration_type eq 'insecure') {
5757 $migrate->{addr
} = $get_migration_ip->($nodename);
5758 $migrate->{addr
} = "[$migrate->{addr}]" if Net
::IP
::ip_is_ipv6
($migrate->{addr
});
5761 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5762 $migrate->{port
} = PVE
::Tools
::next_migrate_port
($pfamily);
5763 $migrate->{uri
} = "tcp:$migrate->{addr}:$migrate->{port}";
5764 push @$cmd, '-incoming', $migrate->{uri
};
5767 } elsif ($statefile eq 'unix') {
5768 # should be default for secure migrations as a ssh TCP forward
5769 # tunnel is not deterministic reliable ready and fails regurarly
5770 # to set up in time, so use UNIX socket forwards
5771 my $migrate = $res->{migrate
} = { proto
=> 'unix' };
5772 $migrate->{addr
} = "/run/qemu-server/$vmid.migrate";
5773 unlink $migrate->{addr
};
5775 $migrate->{uri
} = "unix:$migrate->{addr}";
5776 push @$cmd, '-incoming', $migrate->{uri
};
5779 } elsif (-e
$statefile) {
5780 push @$cmd, '-loadstate', $statefile;
5782 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5783 push @$vollist, $statefile;
5784 push @$cmd, '-loadstate', $statepath;
5786 } elsif ($params->{paused
}) {
5790 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $resume);
5792 my $pci_devices = {}; # host pci devices
5793 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
5794 my $dev = $conf->{"hostpci$i"} or next;
5795 $pci_devices->{$i} = parse_hostpci
($dev);
5798 # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5799 my $real_pci_devices = [ grep { !(defined($_->{mdev
}) && scalar($_->{pciid
}->@*) == 1) } values $pci_devices->%* ];
5801 # map to a flat list of pci ids
5802 my $pci_id_list = [ map { $_->{id
} } map { $_->{pciid
}->@* } $real_pci_devices->@* ];
5804 # reserve all PCI IDs before actually doing anything with them
5805 PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_id_list, $vmid, $start_timeout);
5809 for my $id (sort keys %$pci_devices) {
5810 my $d = $pci_devices->{$id};
5811 for my $dev ($d->{pciid
}->@*) {
5812 my $info = PVE
::QemuServer
::PCI
::prepare_pci_device
($vmid, $dev->{id
}, $id, $d->{mdev
});
5814 # nvidia grid needs the uuid of the mdev as qemu parameter
5815 if ($d->{mdev
} && !defined($uuid) && $info->{vendor
} eq '10de') {
5816 $uuid = PVE
::QemuServer
::PCI
::generate_mdev_uuid
($vmid, $id);
5820 push @$cmd, '-uuid', $uuid if defined($uuid);
5823 eval { cleanup_pci_devices
($vmid, $conf) };
5828 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5831 run_command
(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc
=> sub{}, errfunc
=> sub{});
5833 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5834 # timeout should be more than enough here...
5835 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 20);
5837 my $cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{cpuunits
});
5840 timeout
=> $statefile ?
undef : $start_timeout,
5845 # when migrating, prefix QEMU output so other side can pick up any
5846 # errors that might occur and show the user
5847 if ($migratedfrom) {
5848 $run_params{quiet
} = 1;
5849 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5852 my %systemd_properties = (
5853 Slice
=> 'qemu.slice',
5854 KillMode
=> 'process',
5856 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5859 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5860 $systemd_properties{CPUWeight
} = $cpuunits;
5862 $systemd_properties{CPUShares
} = $cpuunits;
5865 if (my $cpulimit = $conf->{cpulimit
}) {
5866 $systemd_properties{CPUQuota
} = int($cpulimit * 100);
5868 $systemd_properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5870 my $run_qemu = sub {
5871 PVE
::Tools
::run_fork
sub {
5872 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5875 if (my $tpm = $conf->{tpmstate0
}) {
5876 # start the TPM emulator so QEMU can connect on start
5877 $tpmpid = start_swtpm
($storecfg, $vmid, $tpm, $migratedfrom);
5880 my $exitcode = run_command
($cmd, %run_params);
5883 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5884 kill 'TERM', $tpmpid;
5886 die "QEMU exited with code $exitcode\n";
5891 if ($conf->{hugepages
}) {
5894 my $hugepages_topology = PVE
::QemuServer
::Memory
::hugepages_topology
($conf);
5895 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5897 PVE
::QemuServer
::Memory
::hugepages_mount
();
5898 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5900 eval { $run_qemu->() };
5902 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5903 if !$conf->{keephugepages
};
5907 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5908 if !$conf->{keephugepages
};
5910 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5913 eval { $run_qemu->() };
5917 # deactivate volumes if start fails
5918 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5920 eval { cleanup_pci_devices
($vmid, $conf) };
5923 die "start failed: $err";
5926 # re-reserve all PCI IDs now that we can know the actual VM PID
5927 my $pid = PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
5928 eval { PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_id_list, $vmid, undef, $pid) };
5931 if (defined($res->{migrate
})) {
5932 print "migration listens on $res->{migrate}->{uri}\n";
5933 } elsif ($statefile) {
5934 eval { mon_cmd
($vmid, "cont"); };
5938 #start nbd server for storage migration
5939 if (my $nbd = $migrate_opts->{nbd
}) {
5940 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5942 my $migrate_storage_uri;
5943 # nbd_protocol_version > 0 for unix socket support
5944 if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5945 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5946 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
5947 $migrate_storage_uri = "nbd:unix:$socket_path";
5948 $res->{migrate
}->{unix_sockets
} = [$socket_path];
5950 my $nodename = nodename
();
5951 my $localip = $get_migration_ip->($nodename);
5952 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5953 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5955 mon_cmd
($vmid, "nbd-server-start", addr
=> {
5958 host
=> "${localip}",
5959 port
=> "${storage_migrate_port}",
5962 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5963 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5966 foreach my $opt (sort keys %$nbd) {
5967 my $drivestr = $nbd->{$opt}->{drivestr
};
5968 my $volid = $nbd->{$opt}->{volid
};
5969 mon_cmd
($vmid, "nbd-server-add", device
=> "drive-$opt", writable
=> JSON
::true
);
5970 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5971 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5972 print "re-using replicated volume: $opt - $volid\n"
5973 if $nbd->{$opt}->{replicated
};
5975 $res->{drives
}->{$opt} = $nbd->{$opt};
5976 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
5980 if ($migratedfrom) {
5982 set_migration_caps
($vmid);
5987 print "spice listens on port $spice_port\n";
5988 $res->{spice_port
} = $spice_port;
5989 if ($migrate_opts->{spice_ticket
}) {
5990 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
5991 $migrate_opts->{spice_ticket
});
5992 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
5997 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
5998 if !$statefile && $conf->{balloon
};
6000 foreach my $opt (keys %$conf) {
6001 next if $opt !~ m/^net\d+$/;
6002 my $nicconf = parse_net
($conf->{$opt});
6003 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
6005 add_nets_bridge_fdb
($conf, $vmid);
6008 mon_cmd
($vmid, 'qom-set',
6009 path
=> "machine/peripheral/balloon0",
6010 property
=> "guest-stats-polling-interval",
6011 value
=> 2) if (!defined($conf->{balloon
}) || $conf->{balloon
});
6014 print "Resumed VM, removing state\n";
6015 if (my $vmstate = $conf->{vmstate
}) {
6016 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6017 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6019 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6020 PVE
::QemuConfig-
>write_config($vmid, $conf);
6023 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
6028 sub vm_commandline
{
6029 my ($storecfg, $vmid, $snapname) = @_;
6031 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6033 my ($forcemachine, $forcecpu);
6035 my $snapshot = $conf->{snapshots
}->{$snapname};
6036 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6038 # check for machine or CPU overrides in snapshot
6039 $forcemachine = $snapshot->{runningmachine
};
6040 $forcecpu = $snapshot->{runningcpu
};
6042 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
6047 my $defaults = load_defaults
();
6049 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6051 return PVE
::Tools
::cmd2string
($cmd);
6055 my ($vmid, $skiplock) = @_;
6057 PVE
::QemuConfig-
>lock_config($vmid, sub {
6059 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6061 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6063 mon_cmd
($vmid, "system_reset");
6067 sub get_vm_volumes
{
6071 foreach_volid
($conf, sub {
6072 my ($volid, $attr) = @_;
6074 return if $volid =~ m
|^/|;
6076 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6079 push @$vollist, $volid;
6085 sub cleanup_pci_devices
{
6086 my ($vmid, $conf) = @_;
6088 foreach my $key (keys %$conf) {
6089 next if $key !~ m/^hostpci(\d+)$/;
6090 my $hostpciindex = $1;
6091 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
6092 my $d = parse_hostpci
($conf->{$key});
6094 # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6095 # don't want to break ABI just for this two liner
6096 my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6097 PVE
::SysFSTools
::file_write
("$dev_sysfs_dir/remove", "1") if -e
$dev_sysfs_dir;
6100 PVE
::QemuServer
::PCI
::remove_pci_reservation
($vmid);
6103 sub vm_stop_cleanup
{
6104 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6109 my $vollist = get_vm_volumes
($conf);
6110 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6112 if (my $tpmdrive = $conf->{tpmstate0
}) {
6113 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
6114 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
6116 PVE
::Storage
::unmap_volume
($storecfg, $tpm->{file
});
6121 foreach my $ext (qw(mon qmp pid vnc qga)) {
6122 unlink "/var/run/qemu-server/${vmid}.$ext";
6125 if ($conf->{ivshmem
}) {
6126 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
6127 # just delete it for now, VMs which have this already open do not
6128 # are affected, but new VMs will get a separated one. If this
6129 # becomes an issue we either add some sort of ref-counting or just
6130 # add a "don't delete on stop" flag to the ivshmem format.
6131 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
6134 cleanup_pci_devices
($vmid, $conf);
6136 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
6138 warn $@ if $@; # avoid errors - just warn
6141 # call only in locked context
6143 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6145 my $pid = check_running
($vmid, $nocheck);
6150 $conf = PVE
::QemuConfig-
>load_config($vmid);
6151 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6152 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
6153 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
6154 $timeout = $opts->{down
} if $opts->{down
};
6156 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
6161 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
6162 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
6164 mon_cmd
($vmid, "system_powerdown");
6167 mon_cmd
($vmid, "quit");
6173 $timeout = 60 if !defined($timeout);
6176 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6181 if ($count >= $timeout) {
6183 warn "VM still running - terminating now with SIGTERM\n";
6186 die "VM quit/powerdown failed - got timeout\n";
6189 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6193 if (!check_running
($vmid, $nocheck)) {
6194 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6198 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6201 die "VM quit/powerdown failed\n";
6209 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6214 if ($count >= $timeout) {
6215 warn "VM still running - terminating now with SIGKILL\n";
6220 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6223 # Note: use $nocheck to skip tests if VM configuration file exists.
6224 # We need that when migration VMs to other nodes (files already moved)
6225 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6227 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6229 $force = 1 if !defined($force) && !$shutdown;
6232 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
6233 kill 15, $pid if $pid;
6234 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
6235 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
6239 PVE
::QemuConfig-
>lock_config($vmid, sub {
6240 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6245 my ($vmid, $timeout) = @_;
6247 PVE
::QemuConfig-
>lock_config($vmid, sub {
6250 # only reboot if running, as qmeventd starts it again on a stop event
6251 return if !check_running
($vmid);
6253 create_reboot_request
($vmid);
6255 my $storecfg = PVE
::Storage
::config
();
6256 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
6260 # avoid that the next normal shutdown will be confused for a reboot
6261 clear_reboot_request
($vmid);
6267 # note: if using the statestorage parameter, the caller has to check privileges
6269 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6276 PVE
::QemuConfig-
>lock_config($vmid, sub {
6278 $conf = PVE
::QemuConfig-
>load_config($vmid);
6280 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
6281 PVE
::QemuConfig-
>check_lock($conf)
6282 if !($skiplock || $is_backing_up);
6284 die "cannot suspend to disk during backup\n"
6285 if $is_backing_up && $includestate;
6287 if ($includestate) {
6288 $conf->{lock} = 'suspending';
6289 my $date = strftime
("%Y-%m-%d", localtime(time()));
6290 $storecfg = PVE
::Storage
::config
();
6291 if (!$statestorage) {
6292 $statestorage = find_vmstate_storage
($conf, $storecfg);
6293 # check permissions for the storage
6294 my $rpcenv = PVE
::RPCEnvironment
::get
();
6295 if ($rpcenv->{type
} ne 'cli') {
6296 my $authuser = $rpcenv->get_user();
6297 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6302 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
6303 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6304 $path = PVE
::Storage
::path
($storecfg, $vmstate);
6305 PVE
::QemuConfig-
>write_config($vmid, $conf);
6307 mon_cmd
($vmid, "stop");
6311 if ($includestate) {
6313 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
6316 set_migration_caps
($vmid, 1);
6317 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
6319 my $state = mon_cmd
($vmid, "query-savevm");
6320 if (!$state->{status
}) {
6321 die "savevm not active\n";
6322 } elsif ($state->{status
} eq 'active') {
6325 } elsif ($state->{status
} eq 'completed') {
6326 print "State saved, quitting\n";
6328 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
6329 die "query-savevm failed with error '$state->{error}'\n"
6331 die "query-savevm returned status '$state->{status}'\n";
6337 PVE
::QemuConfig-
>lock_config($vmid, sub {
6338 $conf = PVE
::QemuConfig-
>load_config($vmid);
6340 # cleanup, but leave suspending lock, to indicate something went wrong
6342 mon_cmd
($vmid, "savevm-end");
6343 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6344 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6345 delete $conf->@{qw(vmstate runningmachine runningcpu)};
6346 PVE
::QemuConfig-
>write_config($vmid, $conf);
6352 die "lock changed unexpectedly\n"
6353 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
6355 mon_cmd
($vmid, "quit");
6356 $conf->{lock} = 'suspended';
6357 PVE
::QemuConfig-
>write_config($vmid, $conf);
6363 my ($vmid, $skiplock, $nocheck) = @_;
6365 PVE
::QemuConfig-
>lock_config($vmid, sub {
6366 my $res = mon_cmd
($vmid, 'query-status');
6367 my $resume_cmd = 'cont';
6371 my $vmlist = PVE
::Cluster
::get_vmlist
();
6373 if (exists($vmlist->{ids
}->{$vmid})) {
6374 $node = $vmlist->{ids
}->{$vmid}->{node
};
6376 $conf = PVE
::QemuConfig-
>load_config($vmid, $node);
6378 $conf = PVE
::QemuConfig-
>load_config($vmid);
6381 if ($res->{status
}) {
6382 return if $res->{status
} eq 'running'; # job done, go home
6383 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
6384 $reset = 1 if $res->{status
} eq 'shutdown';
6388 PVE
::QemuConfig-
>check_lock($conf)
6389 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
6393 # required if a VM shuts down during a backup and we get a resume
6394 # request before the backup finishes for example
6395 mon_cmd
($vmid, "system_reset");
6398 add_nets_bridge_fdb
($conf, $vmid) if $resume_cmd eq 'cont';
6400 mon_cmd
($vmid, $resume_cmd);
6405 my ($vmid, $skiplock, $key) = @_;
6407 PVE
::QemuConfig-
>lock_config($vmid, sub {
6409 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6411 # there is no qmp command, so we use the human monitor command
6412 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
6413 die $res if $res ne '';
6417 # vzdump restore implementaion
6419 sub tar_archive_read_firstfile
{
6420 my $archive = shift;
6422 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
6424 # try to detect archive type first
6425 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6426 die "unable to open file '$archive'\n";
6427 my $firstfile = <$fh>;
6431 die "ERROR: archive contaions no data\n" if !$firstfile;
6437 sub tar_restore_cleanup
{
6438 my ($storecfg, $statfile) = @_;
6440 print STDERR
"starting cleanup\n";
6442 if (my $fd = IO
::File-
>new($statfile, "r")) {
6443 while (defined(my $line = <$fd>)) {
6444 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6447 if ($volid =~ m
|^/|) {
6448 unlink $volid || die 'unlink failed\n';
6450 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6452 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6454 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6456 print STDERR
"unable to parse line in statfile - $line";
6463 sub restore_file_archive
{
6464 my ($archive, $vmid, $user, $opts) = @_;
6466 return restore_vma_archive
($archive, $vmid, $user, $opts)
6469 my $info = PVE
::Storage
::archive_info
($archive);
6470 my $format = $opts->{format
} // $info->{format
};
6471 my $comp = $info->{compression
};
6473 # try to detect archive format
6474 if ($format eq 'tar') {
6475 return restore_tar_archive
($archive, $vmid, $user, $opts);
6477 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
6481 # hepler to remove disks that will not be used after restore
6482 my $restore_cleanup_oldconf = sub {
6483 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6485 my $kept_disks = {};
6487 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
6488 my ($ds, $drive) = @_;
6490 return if drive_is_cdrom
($drive, 1);
6492 my $volid = $drive->{file
};
6493 return if !$volid || $volid =~ m
|^/|;
6495 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
6496 return if !$path || !$owner || ($owner != $vmid);
6498 # Note: only delete disk we want to restore
6499 # other volumes will become unused
6500 if ($virtdev_hash->{$ds}) {
6501 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
6506 $kept_disks->{$volid} = 1;
6510 # after the restore we have no snapshots anymore
6511 for my $snapname (keys $oldconf->{snapshots
}->%*) {
6512 my $snap = $oldconf->{snapshots
}->{$snapname};
6513 if ($snap->{vmstate
}) {
6514 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
6520 for my $volid (keys $kept_disks->%*) {
6521 eval { PVE
::Storage
::volume_snapshot_delete
($storecfg, $volid, $snapname); };
6527 # Helper to parse vzdump backup device hints
6529 # $rpcenv: Environment, used to ckeck storage permissions
6530 # $user: User ID, to check storage permissions
6531 # $storecfg: Storage configuration
6532 # $fh: the file handle for reading the configuration
6533 # $devinfo: should contain device sizes for all backu-up'ed devices
6534 # $options: backup options (pool, default storage)
6536 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6537 my $parse_backup_hints = sub {
6538 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6540 my $check_storage = sub { # assert if an image can be allocate
6541 my ($storeid, $scfg) = @_;
6542 die "Content type 'images' is not available on storage '$storeid'\n"
6543 if !$scfg->{content
}->{images
};
6544 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6545 if $user ne 'root@pam';
6548 my $virtdev_hash = {};
6549 while (defined(my $line = <$fh>)) {
6550 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6551 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6552 die "archive does not contain data for drive '$virtdev'\n"
6553 if !$devinfo->{$devname};
6555 if (defined($options->{storage
})) {
6556 $storeid = $options->{storage
} || 'local';
6557 } elsif (!$storeid) {
6560 $format = 'raw' if !$format;
6561 $devinfo->{$devname}->{devname
} = $devname;
6562 $devinfo->{$devname}->{virtdev
} = $virtdev;
6563 $devinfo->{$devname}->{format
} = $format;
6564 $devinfo->{$devname}->{storeid
} = $storeid;
6566 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6567 $check_storage->($storeid, $scfg); # permission and content type check
6569 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6570 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6572 my $drive = parse_drive
($virtdev, $2);
6574 if (drive_is_cloudinit
($drive)) {
6575 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6576 $storeid = $options->{storage
} if defined ($options->{storage
});
6577 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6578 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6580 $check_storage->($storeid, $scfg); # permission and content type check
6582 $virtdev_hash->{$virtdev} = {
6584 storeid
=> $storeid,
6585 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6592 return $virtdev_hash;
6595 # Helper to allocate and activate all volumes required for a restore
6597 # $storecfg: Storage configuration
6598 # $virtdev_hash: as returned by parse_backup_hints()
6600 # Returns: { $virtdev => $volid }
6601 my $restore_allocate_devices = sub {
6602 my ($storecfg, $virtdev_hash, $vmid) = @_;
6605 foreach my $virtdev (sort keys %$virtdev_hash) {
6606 my $d = $virtdev_hash->{$virtdev};
6607 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6608 my $storeid = $d->{storeid
};
6609 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6611 # test if requested format is supported
6612 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6613 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6614 $d->{format
} = $defFormat if !$supported;
6617 if ($d->{is_cloudinit
}) {
6618 $name = "vm-$vmid-cloudinit";
6619 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6620 if ($scfg->{path
}) {
6621 $name .= ".$d->{format}";
6625 my $volid = PVE
::Storage
::vdisk_alloc
(
6626 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6628 print STDERR
"new volume ID is '$volid'\n";
6629 $d->{volid
} = $volid;
6631 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6633 $map->{$virtdev} = $volid;
6639 sub restore_update_config_line
{
6640 my ($cookie, $map, $line, $unique) = @_;
6642 return '' if $line =~ m/^\#qmdump\#/;
6643 return '' if $line =~ m/^\#vzdump\#/;
6644 return '' if $line =~ m/^lock:/;
6645 return '' if $line =~ m/^unused\d+:/;
6646 return '' if $line =~ m/^parent:/;
6650 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6651 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6652 # try to convert old 1.X settings
6653 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6654 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6655 my ($model, $macaddr) = split(/\=/, $devconfig);
6656 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6659 bridge
=> "vmbr$ind",
6660 macaddr
=> $macaddr,
6662 my $netstr = print_net
($net);
6664 $res .= "net$cookie->{netcount}: $netstr\n";
6665 $cookie->{netcount
}++;
6667 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6668 my ($id, $netstr) = ($1, $2);
6669 my $net = parse_net
($netstr);
6670 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6671 $netstr = print_net
($net);
6672 $res .= "$id: $netstr\n";
6673 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6676 my $di = parse_drive
($virtdev, $value);
6677 if (defined($di->{backup
}) && !$di->{backup
}) {
6679 } elsif ($map->{$virtdev}) {
6680 delete $di->{format
}; # format can change on restore
6681 $di->{file
} = $map->{$virtdev};
6682 $value = print_drive
($di);
6683 $res .= "$virtdev: $value\n";
6687 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6689 if ($vmgenid ne '0') {
6690 # always generate a new vmgenid if there was a valid one setup
6691 $vmgenid = generate_uuid
();
6693 $res .= "vmgenid: $vmgenid\n";
6694 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6695 my ($uuid, $uuid_str);
6696 UUID
::generate
($uuid);
6697 UUID
::unparse
($uuid, $uuid_str);
6698 my $smbios1 = parse_smbios1
($2);
6699 $smbios1->{uuid
} = $uuid_str;
6700 $res .= $1.print_smbios1
($smbios1)."\n";
6708 my $restore_deactivate_volumes = sub {
6709 my ($storecfg, $virtdev_hash) = @_;
6712 for my $dev (values $virtdev_hash->%*) {
6713 push $vollist->@*, $dev->{volid
} if $dev->{volid
};
6716 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
6717 print STDERR
$@ if $@;
6720 my $restore_destroy_volumes = sub {
6721 my ($storecfg, $virtdev_hash) = @_;
6723 for my $dev (values $virtdev_hash->%*) {
6724 my $volid = $dev->{volid
} or next;
6726 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6727 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6729 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6733 my $restore_merge_config = sub {
6734 my ($filename, $backup_conf_raw, $override_conf) = @_;
6736 my $backup_conf = parse_vm_config
($filename, $backup_conf_raw);
6737 for my $key (keys $override_conf->%*) {
6738 $backup_conf->{$key} = $override_conf->{$key};
6741 return $backup_conf;
6745 my ($cfg, $vmid) = @_;
6747 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6749 my $volid_hash = {};
6750 foreach my $storeid (keys %$info) {
6751 foreach my $item (@{$info->{$storeid}}) {
6752 next if !($item->{volid
} && $item->{size
});
6753 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6754 $volid_hash->{$item->{volid
}} = $item;
6761 sub update_disk_config
{
6762 my ($vmid, $conf, $volid_hash) = @_;
6765 my $prefix = "VM $vmid";
6767 # used and unused disks
6768 my $referenced = {};
6770 # Note: it is allowed to define multiple storages with same path (alias), so
6771 # we need to check both 'volid' and real 'path' (two different volid can point
6772 # to the same path).
6774 my $referencedpath = {};
6777 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6778 my ($opt, $drive) = @_;
6780 my $volid = $drive->{file
};
6782 my $volume = $volid_hash->{$volid};
6784 # mark volid as "in-use" for next step
6785 $referenced->{$volid} = 1;
6786 if ($volume && (my $path = $volume->{path
})) {
6787 $referencedpath->{$path} = 1;
6790 return if drive_is_cdrom
($drive);
6793 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6794 if (defined($updated)) {
6796 $conf->{$opt} = print_drive
($updated);
6797 print "$prefix ($opt): $msg\n";
6801 # remove 'unusedX' entry if volume is used
6802 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6803 my ($opt, $drive) = @_;
6805 my $volid = $drive->{file
};
6809 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6810 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6811 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6813 delete $conf->{$opt};
6816 $referenced->{$volid} = 1;
6817 $referencedpath->{$path} = 1 if $path;
6820 foreach my $volid (sort keys %$volid_hash) {
6821 next if $volid =~ m/vm-$vmid-state-/;
6822 next if $referenced->{$volid};
6823 my $path = $volid_hash->{$volid}->{path
};
6824 next if !$path; # just to be sure
6825 next if $referencedpath->{$path};
6827 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6828 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6829 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6836 my ($vmid, $nolock, $dryrun) = @_;
6838 my $cfg = PVE
::Storage
::config
();
6840 print "rescan volumes...\n";
6841 my $volid_hash = scan_volids
($cfg, $vmid);
6843 my $updatefn = sub {
6846 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6848 PVE
::QemuConfig-
>check_lock($conf);
6851 foreach my $volid (keys %$volid_hash) {
6852 my $info = $volid_hash->{$volid};
6853 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6856 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6858 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6861 if (defined($vmid)) {
6865 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6868 my $vmlist = config_list
();
6869 foreach my $vmid (keys %$vmlist) {
6873 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6879 sub restore_proxmox_backup_archive
{
6880 my ($archive, $vmid, $user, $options) = @_;
6882 my $storecfg = PVE
::Storage
::config
();
6884 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6885 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6887 my $fingerprint = $scfg->{fingerprint
};
6888 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6890 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6891 my $namespace = $scfg->{namespace
};
6893 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6894 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6895 local $ENV{PBS_PASSWORD
} = $password;
6896 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6898 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6899 PVE
::Storage
::parse_volname
($storecfg, $archive);
6901 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6903 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6905 my $tmpdir = "/var/tmp/vzdumptmp$$";
6909 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6910 # disable interrupts (always do cleanups)
6914 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6916 # Note: $oldconf is undef if VM does not exists
6917 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6918 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6919 my $new_conf_raw = '';
6921 my $rpcenv = PVE
::RPCEnvironment
::get
();
6922 my $devinfo = {}; # info about drives included in backup
6923 my $virtdev_hash = {}; # info about allocated drives
6931 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6933 my $cfgfn = "$tmpdir/qemu-server.conf";
6934 my $firewall_config_fn = "$tmpdir/fw.conf";
6935 my $index_fn = "$tmpdir/index.json";
6937 my $cmd = "restore";
6939 my $param = [$pbs_backup_name, "index.json", $index_fn];
6940 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6941 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6942 $index = decode_json
($index);
6944 foreach my $info (@{$index->{files
}}) {
6945 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6947 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6948 $devinfo->{$devname}->{size
} = $1;
6950 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6955 my $is_qemu_server_backup = scalar(
6956 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6958 if (!$is_qemu_server_backup) {
6959 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6961 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6963 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6964 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6966 if ($has_firewall_config) {
6967 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6968 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6970 my $pve_firewall_dir = '/etc/pve/firewall';
6971 mkdir $pve_firewall_dir; # make sure the dir exists
6972 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6975 my $fh = IO
::File-
>new($cfgfn, "r") ||
6976 die "unable to read qemu-server.conf - $!\n";
6978 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6980 # fixme: rate limit?
6982 # create empty/temp config
6983 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6985 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6988 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6990 foreach my $virtdev (sort keys %$virtdev_hash) {
6991 my $d = $virtdev_hash->{$virtdev};
6992 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6994 # this fails if storage is unavailable
6995 my $volid = $d->{volid
};
6996 my $path = PVE
::Storage
::path
($storecfg, $volid);
6998 # for live-restore we only want to preload the efidisk and TPM state
6999 next if $options->{live
} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7002 if (defined(my $ns = $scfg->{namespace
})) {
7003 @ns_arg = ('--ns', $ns);
7006 my $pbs_restore_cmd = [
7007 '/usr/bin/pbs-restore',
7008 '--repository', $repo,
7011 "$d->{devname}.img.fidx",
7016 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
7017 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
7019 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
7020 push @$pbs_restore_cmd, '--skip-zero';
7023 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
7024 print "restore proxmox backup image: $dbg_cmdstring\n";
7025 run_command
($pbs_restore_cmd);
7028 $fh->seek(0, 0) || die "seek failed - $!\n";
7030 my $cookie = { netcount
=> 0 };
7031 while (defined(my $line = <$fh>)) {
7032 $new_conf_raw .= restore_update_config_line
(
7044 if ($err || !$options->{live
}) {
7045 $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7051 $restore_destroy_volumes->($storecfg, $virtdev_hash);
7055 if ($options->{live
}) {
7056 # keep lock during live-restore
7057 $new_conf_raw .= "\nlock: create";
7060 my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf
});
7061 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7063 eval { rescan
($vmid, 1); };
7066 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
7068 if ($options->{live
}) {
7074 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
7076 my $conf = PVE
::QemuConfig-
>load_config($vmid);
7077 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
7079 # these special drives are already restored before start
7080 delete $devinfo->{'drive-efidisk0'};
7081 delete $devinfo->{'drive-tpmstate0-backup'};
7085 keyfile
=> $keyfile,
7086 snapshot
=> $pbs_backup_name,
7087 namespace
=> $namespace,
7089 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7091 PVE
::QemuConfig-
>remove_lock($vmid, "create");
7095 sub pbs_live_restore
{
7096 my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7098 print "starting VM for live-restore\n";
7099 print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7101 my $pbs_backing = {};
7102 for my $ds (keys %$restored_disks) {
7103 $ds =~ m/^drive-(.*)$/;
7105 $pbs_backing->{$confname} = {
7106 repository
=> $opts->{repo
},
7107 snapshot
=> $opts->{snapshot
},
7108 archive
=> "$ds.img.fidx",
7110 $pbs_backing->{$confname}->{keyfile
} = $opts->{keyfile
} if -e
$opts->{keyfile
};
7111 $pbs_backing->{$confname}->{namespace
} = $opts->{namespace
} if defined($opts->{namespace
});
7113 my $drive = parse_drive
($confname, $conf->{$confname});
7114 print "restoring '$ds' to '$drive->{file}'\n";
7117 my $drives_streamed = 0;
7119 # make sure HA doesn't interrupt our restore by stopping the VM
7120 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
7121 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7124 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7125 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7126 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'pbs-backing' => $pbs_backing}, {});
7128 my $qmeventd_fd = register_qmeventd_handle
($vmid);
7130 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7131 # this will effectively collapse the backing image chain consisting of
7132 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7133 # removes itself once all backing images vanish with 'auto-remove=on')
7135 for my $ds (sort keys %$restored_disks) {
7136 my $job_id = "restore-$ds";
7137 mon_cmd
($vmid, 'block-stream',
7138 'job-id' => $job_id,
7141 $jobs->{$job_id} = {};
7144 mon_cmd
($vmid, 'cont');
7145 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
7147 print "restore-drive jobs finished successfully, removing all tracking block devices"
7148 ." to disconnect from Proxmox Backup Server\n";
7150 for my $ds (sort keys %$restored_disks) {
7151 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7154 close($qmeventd_fd);
7160 warn "An error occurred during live-restore: $err\n";
7161 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
7162 die "live-restore failed\n";
7166 sub restore_vma_archive
{
7167 my ($archive, $vmid, $user, $opts, $comp) = @_;
7169 my $readfrom = $archive;
7171 my $cfg = PVE
::Storage
::config
();
7173 my $bwlimit = $opts->{bwlimit
};
7175 my $dbg_cmdstring = '';
7176 my $add_pipe = sub {
7178 push @$commands, $cmd;
7179 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7180 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
7185 if ($archive eq '-') {
7188 # If we use a backup from a PVE defined storage we also consider that
7189 # storage's rate limit:
7190 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
7191 if (defined($volid)) {
7192 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
7193 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
7195 print STDERR
"applying read rate limit: $readlimit\n";
7196 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7197 $add_pipe->($cstream);
7203 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
7204 my $cmd = $info->{decompressor
};
7205 push @$cmd, $readfrom;
7209 my $tmpdir = "/var/tmp/vzdumptmp$$";
7212 # disable interrupts (always do cleanups)
7216 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
7218 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7219 POSIX
::mkfifo
($mapfifo, 0600);
7221 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7223 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7228 my $devinfo = {}; # info about drives included in backup
7229 my $virtdev_hash = {}; # info about allocated drives
7231 my $rpcenv = PVE
::RPCEnvironment
::get
();
7233 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7235 # Note: $oldconf is undef if VM does not exist
7236 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7237 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7238 my $new_conf_raw = '';
7242 my $print_devmap = sub {
7243 my $cfgfn = "$tmpdir/qemu-server.conf";
7245 # we can read the config - that is already extracted
7246 my $fh = IO
::File-
>new($cfgfn, "r") ||
7247 die "unable to read qemu-server.conf - $!\n";
7249 my $fwcfgfn = "$tmpdir/qemu-server.fw";
7251 my $pve_firewall_dir = '/etc/pve/firewall';
7252 mkdir $pve_firewall_dir; # make sure the dir exists
7253 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7256 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7258 foreach my $info (values %{$virtdev_hash}) {
7259 my $storeid = $info->{storeid
};
7260 next if defined($storage_limits{$storeid});
7262 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
7263 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7264 $storage_limits{$storeid} = $limit * 1024;
7267 foreach my $devname (keys %$devinfo) {
7268 die "found no device mapping information for device '$devname'\n"
7269 if !$devinfo->{$devname}->{virtdev
};
7272 # create empty/temp config
7274 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
7275 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7279 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7281 # print restore information to $fifofh
7282 foreach my $virtdev (sort keys %$virtdev_hash) {
7283 my $d = $virtdev_hash->{$virtdev};
7284 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7286 my $storeid = $d->{storeid
};
7287 my $volid = $d->{volid
};
7290 if (my $limit = $storage_limits{$storeid}) {
7291 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7294 my $write_zeros = 1;
7295 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
7299 my $path = PVE
::Storage
::path
($cfg, $volid);
7301 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7303 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7306 $fh->seek(0, 0) || die "seek failed - $!\n";
7308 my $cookie = { netcount
=> 0 };
7309 while (defined(my $line = <$fh>)) {
7310 $new_conf_raw .= restore_update_config_line
(
7327 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7328 local $SIG{ALRM
} = sub { die "got timeout\n"; };
7330 $oldtimeout = alarm($timeout);
7337 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7338 my ($dev_id, $size, $devname) = ($1, $2, $3);
7339 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
7340 } elsif ($line =~ m/^CTIME: /) {
7341 # we correctly received the vma config, so we can disable
7342 # the timeout now for disk allocation (set to 10 minutes, so
7343 # that we always timeout if something goes wrong)
7346 print $fifofh "done\n";
7347 my $tmp = $oldtimeout || 0;
7348 $oldtimeout = undef;
7355 print "restore vma archive: $dbg_cmdstring\n";
7356 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
7360 alarm($oldtimeout) if $oldtimeout;
7362 $restore_deactivate_volumes->($cfg, $virtdev_hash);
7364 close($fifofh) if $fifofh;
7369 $restore_destroy_volumes->($cfg, $virtdev_hash);
7373 my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf
});
7374 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7376 eval { rescan
($vmid, 1); };
7379 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
7382 sub restore_tar_archive
{
7383 my ($archive, $vmid, $user, $opts) = @_;
7385 if (scalar(keys $opts->{override_conf
}->%*) > 0) {
7386 my $keystring = join(' ', keys $opts->{override_conf
}->%*);
7387 die "cannot pass along options ($keystring) when restoring from tar archive\n";
7390 if ($archive ne '-') {
7391 my $firstfile = tar_archive_read_firstfile
($archive);
7392 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7393 if $firstfile ne 'qemu-server.conf';
7396 my $storecfg = PVE
::Storage
::config
();
7398 # avoid zombie disks when restoring over an existing VM -> cleanup first
7399 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7400 # skiplock=1 because qmrestore has set the 'create' lock itself already
7401 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
7402 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
7404 my $tocmd = "/usr/lib/qemu-server/qmextract";
7406 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
7407 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
7408 $tocmd .= ' --prealloc' if $opts->{prealloc
};
7409 $tocmd .= ' --info' if $opts->{info
};
7411 # tar option "xf" does not autodetect compression when read from STDIN,
7412 # so we pipe to zcat
7413 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
7414 PVE
::Tools
::shellquote
("--to-command=$tocmd");
7416 my $tmpdir = "/var/tmp/vzdumptmp$$";
7419 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
7420 local $ENV{VZDUMP_VMID
} = $vmid;
7421 local $ENV{VZDUMP_USER
} = $user;
7423 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7424 my $new_conf_raw = '';
7426 # disable interrupts (always do cleanups)
7430 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7438 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7440 if ($archive eq '-') {
7441 print "extracting archive from STDIN\n";
7442 run_command
($cmd, input
=> "<&STDIN");
7444 print "extracting archive '$archive'\n";
7448 return if $opts->{info
};
7452 my $statfile = "$tmpdir/qmrestore.stat";
7453 if (my $fd = IO
::File-
>new($statfile, "r")) {
7454 while (defined (my $line = <$fd>)) {
7455 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7456 $map->{$1} = $2 if $1;
7458 print STDERR
"unable to parse line in statfile - $line\n";
7464 my $confsrc = "$tmpdir/qemu-server.conf";
7466 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7468 my $cookie = { netcount
=> 0 };
7469 while (defined (my $line = <$srcfd>)) {
7470 $new_conf_raw .= restore_update_config_line
(
7481 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
7487 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7489 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7491 eval { rescan
($vmid, 1); };
7495 sub foreach_storage_used_by_vm
{
7496 my ($conf, $func) = @_;
7500 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7501 my ($ds, $drive) = @_;
7502 return if drive_is_cdrom
($drive);
7504 my $volid = $drive->{file
};
7506 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
7507 $sidhash->{$sid} = $sid if $sid;
7510 foreach my $sid (sort keys %$sidhash) {
7515 my $qemu_snap_storage = {
7518 sub do_snapshots_with_qemu
{
7519 my ($storecfg, $volid, $deviceid) = @_;
7521 return if $deviceid =~ m/tpmstate0/;
7523 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
7524 my $scfg = $storecfg->{ids
}->{$storage_name};
7525 die "could not find storage '$storage_name'\n" if !defined($scfg);
7527 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
7531 if ($volid =~ m/\.(qcow2|qed)$/){
7538 sub qga_check_running
{
7539 my ($vmid, $nowarn) = @_;
7541 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
7543 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7549 sub template_create
{
7550 my ($vmid, $conf, $disk) = @_;
7552 my $storecfg = PVE
::Storage
::config
();
7554 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7555 my ($ds, $drive) = @_;
7557 return if drive_is_cdrom
($drive);
7558 return if $disk && $ds ne $disk;
7560 my $volid = $drive->{file
};
7561 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
7563 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
7564 $drive->{file
} = $voliddst;
7565 $conf->{$ds} = print_drive
($drive);
7566 PVE
::QemuConfig-
>write_config($vmid, $conf);
7570 sub convert_iscsi_path
{
7573 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
7578 my $initiator_name = get_initiator_name
();
7580 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7581 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7584 die "cannot convert iscsi path '$path', unkown format\n";
7587 sub qemu_img_convert
{
7588 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7590 my $storecfg = PVE
::Storage
::config
();
7591 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
7592 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
7594 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7598 my $src_is_iscsi = 0;
7602 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
7603 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7604 $src_format = qemu_img_format
($src_scfg, $src_volname);
7605 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
7606 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
7607 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
7608 } elsif (-f
$src_volid || -b
$src_volid) {
7609 $src_path = $src_volid;
7610 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7615 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7617 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7618 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7619 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7620 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7623 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7624 push @$cmd, '-l', "snapshot.name=$snapname"
7625 if $snapname && $src_format && $src_format eq "qcow2";
7626 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7627 push @$cmd, '-T', $cachemode if defined($cachemode);
7629 if ($src_is_iscsi) {
7630 push @$cmd, '--image-opts';
7631 $src_path = convert_iscsi_path
($src_path);
7632 } elsif ($src_format) {
7633 push @$cmd, '-f', $src_format;
7636 if ($dst_is_iscsi) {
7637 push @$cmd, '--target-image-opts';
7638 $dst_path = convert_iscsi_path
($dst_path);
7640 push @$cmd, '-O', $dst_format;
7643 push @$cmd, $src_path;
7645 if (!$dst_is_iscsi && $is_zero_initialized) {
7646 push @$cmd, "zeroinit:$dst_path";
7648 push @$cmd, $dst_path;
7653 if($line =~ m/\((\S+)\/100\
%\)/){
7655 my $transferred = int($size * $percent / 100);
7656 my $total_h = render_bytes
($size, 1);
7657 my $transferred_h = render_bytes
($transferred, 1);
7659 print "transferred $transferred_h of $total_h ($percent%)\n";
7664 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7666 die "copy failed: $err" if $err;
7669 sub qemu_img_format
{
7670 my ($scfg, $volname) = @_;
7672 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7679 sub qemu_drive_mirror
{
7680 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7682 $jobs = {} if !$jobs;
7686 $jobs->{"drive-$drive"} = {};
7688 if ($dst_volid =~ /^nbd:/) {
7689 $qemu_target = $dst_volid;
7692 my $storecfg = PVE
::Storage
::config
();
7693 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7695 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7697 $format = qemu_img_format
($dst_scfg, $dst_volname);
7699 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7701 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7704 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7705 $opts->{format
} = $format if $format;
7707 if (defined($src_bitmap)) {
7708 $opts->{sync
} = 'incremental';
7709 $opts->{bitmap
} = $src_bitmap;
7710 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7713 if (defined($bwlimit)) {
7714 $opts->{speed
} = $bwlimit * 1024;
7715 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7717 print "drive mirror is starting for drive-$drive\n";
7720 # if a job already runs for this device we get an error, catch it for cleanup
7721 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7723 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7725 die "mirroring error: $err\n";
7728 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7731 # $completion can be either
7732 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7733 # 'cancel': wait until all jobs are ready, block-job-cancel them
7734 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7735 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7736 sub qemu_drive_mirror_monitor
{
7737 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7739 $completion //= 'complete';
7743 my $err_complete = 0;
7745 my $starttime = time ();
7747 die "block job ('$op') timed out\n" if $err_complete > 300;
7749 my $stats = mon_cmd
($vmid, "query-block-jobs");
7752 my $running_jobs = {};
7753 for my $stat (@$stats) {
7754 next if $stat->{type
} ne $op;
7755 $running_jobs->{$stat->{device
}} = $stat;
7758 my $readycounter = 0;
7760 for my $job_id (sort keys %$jobs) {
7761 my $job = $running_jobs->{$job_id};
7763 my $vanished = !defined($job);
7764 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7765 if($complete || ($vanished && $completion eq 'auto')) {
7766 print "$job_id: $op-job finished\n";
7767 delete $jobs->{$job_id};
7771 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7773 my $busy = $job->{busy
};
7774 my $ready = $job->{ready
};
7775 if (my $total = $job->{len
}) {
7776 my $transferred = $job->{offset
} || 0;
7777 my $remaining = $total - $transferred;
7778 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7780 my $duration = $ctime - $starttime;
7781 my $total_h = render_bytes
($total, 1);
7782 my $transferred_h = render_bytes
($transferred, 1);
7784 my $status = sprintf(
7785 "transferred $transferred_h of $total_h ($percent%%) in %s",
7786 render_duration
($duration),
7791 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7793 $status .= ", ready";
7796 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
7797 $jobs->{$job_id}->{ready
} = $ready;
7800 $readycounter++ if $job->{ready
};
7803 last if scalar(keys %$jobs) == 0;
7805 if ($readycounter == scalar(keys %$jobs)) {
7806 print "all '$op' jobs are ready\n";
7808 # do the complete later (or has already been done)
7809 last if $completion eq 'skip' || $completion eq 'auto';
7811 if ($vmiddst && $vmiddst != $vmid) {
7812 my $agent_running = $qga && qga_check_running
($vmid);
7813 if ($agent_running) {
7814 print "freeze filesystem\n";
7815 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
7818 print "suspend vm\n";
7819 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
7823 # if we clone a disk for a new target vm, we don't switch the disk
7824 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
7826 if ($agent_running) {
7827 print "unfreeze filesystem\n";
7828 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
7831 print "resume vm\n";
7832 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
7839 for my $job_id (sort keys %$jobs) {
7840 # try to switch the disk if source and destination are on the same guest
7841 print "$job_id: Completing block job_id...\n";
7844 if ($completion eq 'complete') {
7845 $op = 'block-job-complete';
7846 } elsif ($completion eq 'cancel') {
7847 $op = 'block-job-cancel';
7849 die "invalid completion value: $completion\n";
7851 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
7852 if ($@ =~ m/cannot be completed/) {
7853 print "$job_id: block job cannot be completed, trying again.\n";
7856 print "$job_id: Completed successfully.\n";
7857 $jobs->{$job_id}->{complete
} = 1;
7868 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7869 die "block job ($op) error: $err";
7873 sub qemu_blockjobs_cancel
{
7874 my ($vmid, $jobs) = @_;
7876 foreach my $job (keys %$jobs) {
7877 print "$job: Cancelling block job\n";
7878 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
7879 $jobs->{$job}->{cancel
} = 1;
7883 my $stats = mon_cmd
($vmid, "query-block-jobs");
7885 my $running_jobs = {};
7886 foreach my $stat (@$stats) {
7887 $running_jobs->{$stat->{device
}} = $stat;
7890 foreach my $job (keys %$jobs) {
7892 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
7893 print "$job: Done.\n";
7894 delete $jobs->{$job};
7898 last if scalar(keys %$jobs) == 0;
7905 my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7907 my ($vmid, $running) = $source->@{qw(vmid running)};
7908 my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7910 my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7911 my ($storage, $format) = $dest->@{qw(storage format)};
7913 my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7915 if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7916 die "cloning from/to EFI disk requires EFI disk\n"
7917 if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7918 die "cloning from/to TPM state requires TPM state\n"
7919 if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7921 # This would lead to two device nodes in QEMU pointing to the same backing image!
7922 die "cannot change drive name when cloning disk from/to the same VM\n"
7923 if $use_drive_mirror && $vmid == $newvmid;
7926 die "cannot move TPM state while VM is running\n"
7927 if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7931 print "create " . ($full ?
'full' : 'linked') . " clone of drive ";
7932 print "$src_drivename " if $src_drivename;
7933 print "($drive->{file})\n";
7936 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
7937 push @$newvollist, $newvolid;
7940 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
7941 $storeid = $storage if $storage;
7943 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
7947 if (drive_is_cloudinit
($drive)) {
7948 $name = "vm-$newvmid-cloudinit";
7949 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7950 if ($scfg->{path
}) {
7951 $name .= ".$dst_format";
7954 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
7955 } elsif ($dst_drivename eq 'efidisk0') {
7956 $size = $efisize or die "internal error - need to specify EFI disk size\n";
7957 } elsif ($dst_drivename eq 'tpmstate0') {
7958 $dst_format = 'raw';
7959 $size = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
7961 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
7963 $newvolid = PVE
::Storage
::vdisk_alloc
(
7964 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7966 push @$newvollist, $newvolid;
7968 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
7970 if (drive_is_cloudinit
($drive)) {
7971 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7972 # if this is the case, we have to complete any block-jobs still there from
7973 # previous drive-mirrors
7974 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7975 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
7980 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
7981 if ($use_drive_mirror) {
7982 qemu_drive_mirror
($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7983 $completion, $qga, $bwlimit);
7985 # TODO: handle bwlimits
7986 if ($dst_drivename eq 'efidisk0') {
7987 # the relevant data on the efidisk may be smaller than the source
7988 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7989 # that is given by the OVMF_VARS.fd
7990 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
7991 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
7993 my $src_format = (PVE
::Storage
::parse_volname
($storecfg, $drive->{file
}))[6];
7995 # better for Ceph if block size is not too small, see bug #3324
7998 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8000 if ($src_format eq 'qcow2' && $snapname) {
8001 die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8002 if !min_version
(kvm_user_version
(), 6, 2);
8003 push $cmd->@*, '-l', $snapname;
8005 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8008 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
8014 my ($size) = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
8016 my $disk = dclone
($drive);
8017 delete $disk->{format
};
8018 $disk->{file
} = $newvolid;
8019 $disk->{size
} = $size if defined($size);
8024 sub get_running_qemu_version
{
8026 my $res = mon_cmd
($vmid, "query-version");
8027 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8030 sub qemu_use_old_bios_files
{
8031 my ($machine_type) = @_;
8033 return if !$machine_type;
8035 my $use_old_bios_files = undef;
8037 if ($machine_type =~ m/^(\S+)\.pxe$/) {
8039 $use_old_bios_files = 1;
8041 my $version = extract_version
($machine_type, kvm_user_version
());
8042 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8043 # load new efi bios files on migration. So this hack is required to allow
8044 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8045 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8046 $use_old_bios_files = !min_version
($version, 2, 4);
8049 return ($use_old_bios_files, $machine_type);
8052 sub get_efivars_size
{
8053 my ($conf, $efidisk) = @_;
8055 my $arch = get_vm_arch
($conf);
8056 $efidisk //= $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
8057 my $smm = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
8058 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8059 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
8060 return -s
$ovmf_vars;
8063 sub update_efidisk_size
{
8066 return if !defined($conf->{efidisk0
});
8068 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
8069 $disk->{size
} = get_efivars_size
($conf);
8070 $conf->{efidisk0
} = print_drive
($disk);
8075 sub update_tpmstate_size
{
8078 my $disk = PVE
::QemuServer
::parse_drive
('tpmstate0', $conf->{tpmstate0
});
8079 $disk->{size
} = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8080 $conf->{tpmstate0
} = print_drive
($disk);
8083 sub create_efidisk
($$$$$$$) {
8084 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8086 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8087 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
8089 my $vars_size_b = -s
$ovmf_vars;
8090 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
8091 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8092 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
8094 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
8095 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
8097 return ($volid, $size/1024);
8100 sub vm_iothreads_list
{
8103 my $res = mon_cmd
($vmid, 'query-iothreads');
8106 foreach my $iothread (@$res) {
8107 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
8114 my ($conf, $drive) = @_;
8118 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
8120 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
8126 my $controller = int($drive->{index} / $maxdev);
8127 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
8131 return ($maxdev, $controller, $controller_prefix);
8134 sub resolve_dst_disk_format
{
8135 my ($storecfg, $storeid, $src_volname, $format) = @_;
8136 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
8139 # if no target format is specified, use the source disk format as hint
8141 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8142 $format = qemu_img_format
($scfg, $src_volname);
8148 # test if requested format is supported - else use default
8149 my $supported = grep { $_ eq $format } @$validFormats;
8150 $format = $defFormat if !$supported;
8154 # NOTE: if this logic changes, please update docs & possibly gui logic
8155 sub find_vmstate_storage
{
8156 my ($conf, $storecfg) = @_;
8158 # first, return storage from conf if set
8159 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
8161 my ($target, $shared, $local);
8163 foreach_storage_used_by_vm
($conf, sub {
8165 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
8166 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
8167 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
8170 # second, use shared storage where VM has at least one disk
8171 # third, use local storage where VM has at least one disk
8172 # fall back to local storage
8173 $target = $shared // $local // 'local';
8179 my ($uuid, $uuid_str);
8180 UUID
::generate
($uuid);
8181 UUID
::unparse
($uuid, $uuid_str);
8185 sub generate_smbios1_uuid
{
8186 return "uuid=".generate_uuid
();
8192 mon_cmd
($vmid, 'nbd-server-stop');
8195 sub create_reboot_request
{
8197 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8198 or die "failed to create reboot trigger file: $!\n";
8202 sub clear_reboot_request
{
8204 my $path = "/run/qemu-server/$vmid.reboot";
8207 $res = unlink($path);
8208 die "could not remove reboot request for $vmid: $!"
8209 if !$res && $! != POSIX
::ENOENT
;
8214 sub bootorder_from_legacy
{
8215 my ($conf, $bootcfg) = @_;
8217 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
8218 my $bootindex_hash = {};
8220 foreach my $o (split(//, $boot)) {
8221 $bootindex_hash->{$o} = $i*100;
8227 PVE
::QemuConfig-
>foreach_volume($conf, sub {
8228 my ($ds, $drive) = @_;
8230 if (drive_is_cdrom
($drive, 1)) {
8231 if ($bootindex_hash->{d
}) {
8232 $bootorder->{$ds} = $bootindex_hash->{d
};
8233 $bootindex_hash->{d
} += 1;
8235 } elsif ($bootindex_hash->{c
}) {
8236 $bootorder->{$ds} = $bootindex_hash->{c
}
8237 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
8238 $bootindex_hash->{c
} += 1;
8242 if ($bootindex_hash->{n
}) {
8243 for (my $i = 0; $i < $MAX_NETS; $i++) {
8244 my $netname = "net$i";
8245 next if !$conf->{$netname};
8246 $bootorder->{$netname} = $bootindex_hash->{n
};
8247 $bootindex_hash->{n
} += 1;
8254 # Generate default device list for 'boot: order=' property. Matches legacy
8255 # default boot order, but with explicit device names. This is important, since
8256 # the fallback for when neither 'order' nor the old format is specified relies
8257 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8258 sub get_default_bootdevices
{
8264 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
8265 push @ret, $first if $first;
8268 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
8269 push @ret, $first if $first;
8272 for (my $i = 0; $i < $MAX_NETS; $i++) {
8273 my $netname = "net$i";
8274 next if !$conf->{$netname};
8275 push @ret, $netname;
8282 sub device_bootorder
{
8285 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
8287 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
8290 if (!defined($boot) || $boot->{legacy
}) {
8291 $bootorder = bootorder_from_legacy
($conf, $boot);
8292 } elsif ($boot->{order
}) {
8293 my $i = 100; # start at 100 to allow user to insert devices before us with -args
8294 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
8295 $bootorder->{$dev} = $i++;
8302 sub register_qmeventd_handle
{
8306 my $peer = "/var/run/qmeventd.sock";
8311 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
8313 if ($! != EINTR
&& $! != EAGAIN
) {
8314 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8317 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8318 . "after $count retries\n";
8323 # send handshake to mark VM as backing up
8324 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
8326 # return handle to be closed later when inhibit is no longer required
8330 # bash completion helper
8332 sub complete_backup_archives
{
8333 my ($cmdname, $pname, $cvalue) = @_;
8335 my $cfg = PVE
::Storage
::config
();
8339 if ($cvalue =~ m/^([^:]+):/) {
8343 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
8346 foreach my $id (keys %$data) {
8347 foreach my $item (@{$data->{$id}}) {
8348 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8349 push @$res, $item->{volid
} if defined($item->{volid
});
8356 my $complete_vmid_full = sub {
8359 my $idlist = vmstatus
();
8363 foreach my $id (keys %$idlist) {
8364 my $d = $idlist->{$id};
8365 if (defined($running)) {
8366 next if $d->{template
};
8367 next if $running && $d->{status
} ne 'running';
8368 next if !$running && $d->{status
} eq 'running';
8377 return &$complete_vmid_full();
8380 sub complete_vmid_stopped
{
8381 return &$complete_vmid_full(0);
8384 sub complete_vmid_running
{
8385 return &$complete_vmid_full(1);
8388 sub complete_storage
{
8390 my $cfg = PVE
::Storage
::config
();
8391 my $ids = $cfg->{ids
};
8394 foreach my $sid (keys %$ids) {
8395 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
8396 next if !$ids->{$sid}->{content
}->{images
};
8403 sub complete_migration_storage
{
8404 my ($cmd, $param, $current_value, $all_args) = @_;
8406 my $targetnode = @$all_args[1];
8408 my $cfg = PVE
::Storage
::config
();
8409 my $ids = $cfg->{ids
};
8412 foreach my $sid (keys %$ids) {
8413 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
8414 next if !$ids->{$sid}->{content
}->{images
};
8423 my $qmpstatus = eval {
8424 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
8425 mon_cmd
($vmid, "query-status");
8428 return $qmpstatus && $qmpstatus->{status
} eq "paused";
8431 sub check_volume_storage_type
{
8432 my ($storecfg, $vol) = @_;
8434 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
8435 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8436 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
8438 die "storage '$storeid' does not support content-type '$vtype'\n"
8439 if !$scfg->{content
}->{$vtype};
8444 sub add_nets_bridge_fdb
{
8445 my ($conf, $vmid) = @_;
8447 for my $opt (keys %$conf) {
8448 next if $opt !~ m/^net(\d+)$/;
8449 my $iface = "tap${vmid}i$1";
8450 # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8451 my $net = parse_net
($conf->{$opt}, 1) or next;
8453 my $mac = $net->{macaddr
};
8455 log_warn
("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8456 if !file_read_firstline
("/sys/class/net/$iface/brport/learning");
8460 my $bridge = $net->{bridge
};
8462 PVE
::Network
::SDN
::Zones
::add_bridge_fdb
($iface, $mac, $bridge, $net->{firewall
});
8463 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8464 PVE
::Network
::add_bridge_fdb
($iface, $mac, $net->{firewall
});
8469 sub del_nets_bridge_fdb
{
8470 my ($conf, $vmid) = @_;
8472 for my $opt (keys %$conf) {
8473 next if $opt !~ m/^net(\d+)$/;
8474 my $iface = "tap${vmid}i$1";
8476 my $net = parse_net
($conf->{$opt}) or next;
8477 my $mac = $net->{macaddr
} or next;
8479 my $bridge = $net->{bridge
};
8481 PVE
::Network
::SDN
::Zones
::del_bridge_fdb
($iface, $mac, $bridge, $net->{firewall
});
8482 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8483 PVE
::Network
::del_bridge_fdb
($iface, $mac, $net->{firewall
});