1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
22 use List
::Util
qw(first);
25 use Storable
qw(dclone);
26 use Time
::HiRes
qw(gettimeofday usleep);
30 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
33 use PVE
::DataCenterConfig
;
34 use PVE
::Exception
qw(raise raise_param_exc);
35 use PVE
::Format
qw(render_duration render_bytes);
36 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
37 use PVE
::Mapping
::PCI
;
38 use PVE
::Mapping
::USB
;
40 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
43 use PVE
::RESTEnvironment
qw(log_warn);
44 use PVE
::RPCEnvironment
;
48 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
52 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout windows_version);
53 use PVE
::QemuServer
::Cloudinit
;
54 use PVE
::QemuServer
::CGroup
;
55 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
56 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
57 use PVE
::QemuServer
::Machine
;
58 use PVE
::QemuServer
::Memory
;
59 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
60 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
61 use PVE
::QemuServer
::USB
;
65 require PVE
::Network
::SDN
::Zones
;
69 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
73 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
74 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
77 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
78 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
81 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
82 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
85 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
86 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
89 "$EDK2_FW_BASE/OVMF_CODE.fd",
90 "$EDK2_FW_BASE/OVMF_VARS.fd",
95 "$EDK2_FW_BASE/AAVMF_CODE.fd",
96 "$EDK2_FW_BASE/AAVMF_VARS.fd",
101 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
103 # Note about locking: we use flock on the config file protect against concurent actions.
104 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
105 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
106 # But you can ignore this kind of lock with the --skiplock flag.
114 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
115 description
=> "Some command save/restore state from this location.",
121 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
122 description
=> "Specifies the QEMU machine type.",
124 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
129 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
132 $nodename_cache //= PVE
::INotify
::nodename
();
133 return $nodename_cache;
140 enum
=> [qw(i6300esb ib700)],
141 description
=> "Watchdog type to emulate.",
142 default => 'i6300esb',
147 enum
=> [qw(reset shutdown poweroff pause debug none)],
148 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
152 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
156 description
=> "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
161 fstrim_cloned_disks
=> {
162 description
=> "Run fstrim after moving a disk or migrating the VM.",
167 'freeze-fs-on-backup' => {
168 description
=> "Freeze/thaw guest filesystems on backup for consistency.",
174 description
=> "Select the agent type",
178 enum
=> [qw(virtio isa)],
184 description
=> "Select the VGA type.",
189 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
192 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
204 description
=> "The size of the file in MB.",
208 pattern
=> '[a-zA-Z0-9\-]+',
210 format_description
=> 'string',
211 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
218 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
219 description
=> "Configure an audio device."
223 enum
=> ['spice', 'none'],
226 description
=> "Driver backend for the audio device."
230 my $spice_enhancements_fmt = {
235 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
239 enum
=> ['off', 'all', 'filter'],
242 description
=> "Enable video streaming. Uses compression for detected video streams."
249 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
251 description
=> "The file on the host to gather entropy from. In most cases '/dev/urandom'"
252 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
253 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
254 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
255 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
256 ." a hardware RNG from the host.",
260 description
=> "Maximum bytes of entropy allowed to get injected into the guest every"
261 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
262 ." `0` to disable limiting (potentially dangerous!).",
265 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
266 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
267 # reading from /dev/urandom
272 description
=> "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
273 ." the guest to retrieve another 'max_bytes' of entropy.",
279 my $meta_info_fmt = {
282 description
=> "The guest creation timestamp as UNIX epoch time",
288 description
=> "The QEMU (machine) version from the time this VM was created.",
289 pattern
=> '\d+(\.\d+)+',
298 description
=> "Specifies whether a VM will be started during system bootup.",
304 description
=> "Automatic restart after crash (currently ignored).",
309 type
=> 'string', format
=> 'pve-hotplug-features',
310 description
=> "Selectively enable hotplug features. This is a comma separated list of"
311 ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
312 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
313 ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
315 default => 'network,disk,usb',
320 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
326 description
=> "Lock/unlock the VM.",
327 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
332 description
=> "Limit of CPU usage.",
333 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
334 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
342 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
343 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
344 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
345 ." weights of all the other running VMs.",
348 default => 'cgroup v1: 1024, cgroup v2: 100',
353 description
=> "Amount of RAM for the VM in MiB. This is the maximum available memory when"
354 ." you use the balloon device.",
361 description
=> "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
367 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the"
368 ." more memory this VM gets. Number is relative to weights of all other running VMs."
369 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
377 description
=> "Keyboard layout for VNC server. This option is generally not required and"
378 ." is often better handled from within the guest OS.",
379 enum
=> PVE
::Tools
::kvmkeymaplist
(),
384 type
=> 'string', format
=> 'dns-name',
385 description
=> "Set a name for the VM. Only used on the configuration web interface.",
390 description
=> "SCSI controller model",
391 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
397 description
=> "Description for the VM. Shown in the web-interface VM's summary."
398 ." This is saved as comment inside the configuration file.",
399 maxLength
=> 1024 * 8,
404 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
405 description
=> "Specify guest operating system.",
406 verbose_description
=> <<EODESC,
407 Specify guest operating system. This is used to enable special
408 optimization/features for specific operating systems:
411 other;; unspecified OS
412 wxp;; Microsoft Windows XP
413 w2k;; Microsoft Windows 2000
414 w2k3;; Microsoft Windows 2003
415 w2k8;; Microsoft Windows 2008
416 wvista;; Microsoft Windows Vista
417 win7;; Microsoft Windows 7
418 win8;; Microsoft Windows 8/2012/2012r2
419 win10;; Microsoft Windows 10/2016/2019
420 win11;; Microsoft Windows 11/2022
421 l24;; Linux 2.4 Kernel
422 l26;; Linux 2.6 - 6.X Kernel
423 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
428 type
=> 'string', format
=> 'pve-qm-boot',
429 description
=> "Specify guest boot order. Use the 'order=' sub-property as usage with no"
430 ." key or 'legacy=' is deprecated.",
434 type
=> 'string', format
=> 'pve-qm-bootdisk',
435 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
436 pattern
=> '(ide|sata|scsi|virtio)\d+',
441 description
=> "The number of CPUs. Please use option -sockets instead.",
448 description
=> "The number of CPU sockets.",
455 description
=> "The number of cores per socket.",
462 description
=> "Enable/disable NUMA.",
468 description
=> "Enable/disable hugepages memory.",
469 enum
=> [qw(any 2 1024)],
475 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
476 ." after VM shutdown and can be used for subsequent starts.",
481 description
=> "Number of hotplugged vcpus.",
488 description
=> "Enable/disable ACPI.",
493 description
=> "Enable/disable communication with the QEMU Guest Agent and its properties.",
495 format
=> $agent_fmt,
500 description
=> "Enable/disable KVM hardware virtualization.",
506 description
=> "Enable/disable time drift fix.",
512 description
=> "Set the real time clock (RTC) to local time. This is enabled by default if"
513 ." the `ostype` indicates a Microsoft Windows OS.",
518 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
522 type
=> 'string', format
=> $vga_fmt,
523 description
=> "Configure the VGA hardware.",
524 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
525 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
526 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
527 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
528 ." display server. For win* OS you can select how many independent displays you want,"
529 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
530 ." using a serial device as terminal.",
534 type
=> 'string', format
=> 'pve-qm-watchdog',
535 description
=> "Create a virtual hardware watchdog device.",
536 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
537 ." action), the watchdog must be periodically polled by an agent inside the guest or"
538 ." else the watchdog will reset the guest (or execute the respective action specified)",
543 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
544 description
=> "Set the initial date of the real time clock. Valid format for date are:"
545 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
546 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
549 startup
=> get_standard_option
('pve-startup-order'),
553 description
=> "Enable/disable Template.",
559 description
=> "Arbitrary arguments passed to kvm.",
560 verbose_description
=> <<EODESCR,
561 Arbitrary arguments passed to kvm, for example:
563 args: -no-reboot -smbios 'type=0,vendor=FOO'
565 NOTE: this option is for experts only.
572 description
=> "Enable/disable the USB tablet device.",
573 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
574 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
575 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
576 ." may consider disabling this to save some context switches. This is turned off by"
577 ." default if you use spice (`qm set <vmid> --vga qxl`).",
582 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
586 migrate_downtime
=> {
589 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
595 type
=> 'string', format
=> 'pve-qm-ide',
596 typetext
=> '<volume>',
597 description
=> "This is an alias for option -ide2",
601 description
=> "Emulated CPU type.",
603 format
=> 'pve-vm-cpu-conf',
605 parent
=> get_standard_option
('pve-snapshot-name', {
607 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
611 description
=> "Timestamp for snapshots.",
617 type
=> 'string', format
=> 'pve-volume-id',
618 description
=> "Reference to a volume which stores the VM state. This is used internally"
621 vmstatestorage
=> get_standard_option
('pve-storage-id', {
622 description
=> "Default storage for VM state volumes/files.",
625 runningmachine
=> get_standard_option
('pve-qemu-machine', {
626 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
630 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
631 ." internally for snapshots.",
634 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
635 format_description
=> 'QEMU -cpu parameter'
637 machine
=> get_standard_option
('pve-qemu-machine'),
639 description
=> "Virtual processor architecture. Defaults to the host.",
642 enum
=> [qw(x86_64 aarch64)],
645 description
=> "Specify SMBIOS type 1 fields.",
646 type
=> 'string', format
=> 'pve-qm-smbios1',
653 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
654 ." remove disk operations.",
660 enum
=> [ qw(seabios ovmf) ],
661 description
=> "Select BIOS implementation.",
662 default => 'seabios',
666 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
667 format_description
=> 'UUID',
668 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
669 ." to disable explicitly.",
670 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
671 ." value identifier to the guest OS. This allows to notify the guest operating system"
672 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
673 ." execution or creation from a template). The guest operating system notices the"
674 ." change, and is then able to react as appropriate by marking its copies of"
675 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
676 ."Note that auto-creation only works when done through API/CLI create or update methods"
677 .", but not when manually editing the config file.",
678 default => "1 (autogenerated)",
683 format
=> 'pve-volume-id',
685 description
=> "Script that will be executed during various steps in the vms lifetime.",
689 format
=> $ivshmem_fmt,
690 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
696 format
=> $audio_fmt,
697 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
700 spice_enhancements
=> {
702 format
=> $spice_enhancements_fmt,
703 description
=> "Configure additional enhancements for SPICE.",
707 type
=> 'string', format
=> 'pve-tag-list',
708 description
=> 'Tags of the VM. This is only meta information.',
714 description
=> "Configure a VirtIO-based Random Number Generator.",
719 format
=> $meta_info_fmt,
720 description
=> "Some (read-only) meta-information about this guest.",
724 type
=> 'string', format
=> 'pve-cpuset',
725 description
=> "List of host cores used to execute guest processes, for example: 0,5,8-11",
734 description
=> 'Specify a custom file containing all meta data passed to the VM via"
735 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
736 format
=> 'pve-volume-id',
737 format_description
=> 'volume',
742 description
=> 'To pass a custom file containing all network data to the VM via cloud-init.',
743 format
=> 'pve-volume-id',
744 format_description
=> 'volume',
749 description
=> 'To pass a custom file containing all user data to the VM via cloud-init.',
750 format
=> 'pve-volume-id',
751 format_description
=> 'volume',
756 description
=> 'To pass a custom file containing all vendor data to the VM via cloud-init.',
757 format
=> 'pve-volume-id',
758 format_description
=> 'volume',
761 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
763 my $confdesc_cloudinit = {
767 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
768 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
769 .' and `configdrive2` for windows.',
770 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
775 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
776 ." image's configured default user.",
781 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
782 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
783 .' support hashed passwords.',
788 description
=> 'cloud-init: do an automatic package upgrade after the first boot.',
794 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
796 format
=> 'pve-qm-cicustom',
801 description
=> 'cloud-init: Sets DNS search domains for a container. Create will'
802 .' automatically use the setting from the host if neither searchdomain nor nameserver'
807 type
=> 'string', format
=> 'address-list',
808 description
=> 'cloud-init: Sets DNS server IP address for a container. Create will'
809 .' automatically use the setting from the host if neither searchdomain nor nameserver'
815 format
=> 'urlencoded',
816 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
820 # what about other qemu settings ?
822 #machine => 'string',
835 ##soundhw => 'string',
837 while (my ($k, $v) = each %$confdesc) {
838 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
842 my $MAX_SERIAL_PORTS = 4;
843 my $MAX_PARALLEL_PORTS = 3;
849 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
850 description
=> "CPUs accessing this NUMA node.",
851 format_description
=> "id[-id];...",
855 description
=> "Amount of memory this NUMA node provides.",
860 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
861 description
=> "Host NUMA nodes to use.",
862 format_description
=> "id[-id];...",
867 enum
=> [qw(preferred bind interleave)],
868 description
=> "NUMA allocation policy.",
872 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
875 type
=> 'string', format
=> $numa_fmt,
876 description
=> "NUMA topology.",
878 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
880 for (my $i = 0; $i < $MAX_NUMA; $i++) {
881 $confdesc->{"numa$i"} = $numadesc;
884 my $nic_model_list = [
900 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
902 my $net_fmt_bridge_descr = <<__EOD__;
903 Bridge to attach the network device to. The Proxmox VE standard bridge
906 If you do not specify a bridge, we create a kvm user (NATed) network
907 device, which provides DHCP and DNS services. The following addresses
914 The DHCP server assign addresses to the guest starting from 10.0.2.15.
918 macaddr
=> get_standard_option
('mac-addr', {
919 description
=> "MAC address. That address must be unique withing your network. This is"
920 ." automatically generated if not specified.",
924 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
925 ." very low CPU overhead. If your guest does not support this driver, it is usually"
926 ." best to use 'e1000'.",
927 enum
=> $nic_model_list,
930 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
931 bridge
=> get_standard_option
('pve-bridge-id', {
932 description
=> $net_fmt_bridge_descr,
937 minimum
=> 0, maximum
=> 64,
938 description
=> 'Number of packet queues to be used on the device.',
944 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
949 minimum
=> 1, maximum
=> 4094,
950 description
=> 'VLAN tag to apply to packets on this interface.',
955 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
956 description
=> 'VLAN trunks to pass through this interface.',
957 format_description
=> 'vlanid[;vlanid...]',
962 description
=> 'Whether this interface should be protected by the firewall.',
967 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
972 minimum
=> 1, maximum
=> 65520,
973 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
980 type
=> 'string', format
=> $net_fmt,
981 description
=> "Specify network devices.",
984 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
989 format
=> 'pve-ipv4-config',
990 format_description
=> 'IPv4Format/CIDR',
991 description
=> 'IPv4 address in CIDR format.',
998 format_description
=> 'GatewayIPv4',
999 description
=> 'Default gateway for IPv4 traffic.',
1005 format
=> 'pve-ipv6-config',
1006 format_description
=> 'IPv6Format/CIDR',
1007 description
=> 'IPv6 address in CIDR format.',
1014 format_description
=> 'GatewayIPv6',
1015 description
=> 'Default gateway for IPv6 traffic.',
1020 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
1021 my $ipconfigdesc = {
1023 type
=> 'string', format
=> 'pve-qm-ipconfig',
1024 description
=> <<'EODESCR',
1025 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1027 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1029 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1030 gateway should be provided.
1031 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1032 cloud-init 19.4 or newer.
1034 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1038 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
1040 for (my $i = 0; $i < $MAX_NETS; $i++) {
1041 $confdesc->{"net$i"} = $netdesc;
1042 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1045 foreach my $key (keys %$confdesc_cloudinit) {
1046 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1049 PVE
::JSONSchema
::register_format
('pve-cpuset', \
&pve_verify_cpuset
);
1050 sub pve_verify_cpuset
{
1051 my ($set_text, $noerr) = @_;
1053 my ($count, $members) = eval { PVE
::CpuSet
::parse_cpuset
($set_text) };
1057 die "unable to parse cpuset option\n";
1060 return PVE
::CpuSet-
>new($members)->short_string();
1063 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1064 sub verify_volume_id_or_qm_path
{
1065 my ($volid, $noerr) = @_;
1067 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1069 return verify_volume_id_or_absolute_path
($volid, $noerr);
1072 PVE
::JSONSchema
::register_format
('pve-volume-id-or-absolute-path', \
&verify_volume_id_or_absolute_path
);
1073 sub verify_volume_id_or_absolute_path
{
1074 my ($volid, $noerr) = @_;
1076 return $volid if $volid =~ m
|^/|;
1078 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1089 pattern
=> '(/dev/.+|socket)',
1090 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1091 verbose_description
=> <<EODESCR,
1092 Create a serial device inside the VM (n is 0 to 3), and pass through a
1093 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1094 host side (use 'qm terminal' to open a terminal connection).
1096 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1097 use with special care.
1099 CAUTION: Experimental! User reported problems with this option.
1106 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1107 description
=> "Map host parallel devices (n is 0 to 2).",
1108 verbose_description
=> <<EODESCR,
1109 Map host parallel devices (n is 0 to 2).
1111 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1112 machines - use with special care.
1114 CAUTION: Experimental! User reported problems with this option.
1118 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1119 $confdesc->{"parallel$i"} = $paralleldesc;
1122 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1123 $confdesc->{"serial$i"} = $serialdesc;
1126 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1127 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1130 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1131 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1134 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
1135 $confdesc->{"usb$i"} = $PVE::QemuServer
::USB
::usbdesc
;
1143 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1144 . " Deprecated, use 'order=' instead.",
1145 pattern
=> '[acdn]{1,4}',
1146 format_description
=> "[acdn]{1,4}",
1148 # note: this is also the fallback if boot: is not given at all
1154 format
=> 'pve-qm-bootdev-list',
1155 format_description
=> "device[;device...]",
1156 description
=> <<EODESC,
1157 The guest will attempt to boot from devices in the order they appear here.
1159 Disks, optical drives and passed-through storage USB devices will be directly
1160 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1161 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1163 Note that only devices in this list will be marked as bootable and thus loaded
1164 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1165 (e.g. software-raid), you need to specify all of them here.
1167 Overrides the deprecated 'legacy=[acdn]*' value when given.
1171 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1173 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1174 sub verify_bootdev
{
1175 my ($dev, $noerr) = @_;
1177 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1178 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && !$special;
1182 return 0 if $dev !~ m/^$base\d+$/;
1183 return 0 if !$confdesc->{$dev};
1187 return $dev if $check->("net");
1188 return $dev if $check->("usb");
1189 return $dev if $check->("hostpci");
1192 die "invalid boot device '$dev'\n";
1195 sub print_bootorder
{
1197 return "" if !@$devs;
1198 my $data = { order
=> join(';', @$devs) };
1199 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1202 my $kvm_api_version = 0;
1205 return $kvm_api_version if $kvm_api_version;
1207 open my $fh, '<', '/dev/kvm' or return;
1209 # 0xae00 => KVM_GET_API_VERSION
1210 $kvm_api_version = ioctl($fh, 0xae00, 0);
1213 return $kvm_api_version;
1216 my $kvm_user_version = {};
1219 sub kvm_user_version
{
1222 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1223 my $st = stat($binary);
1225 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1226 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1227 $cachedmtime == $st->mtime;
1229 $kvm_user_version->{$binary} = 'unknown';
1230 $kvm_mtime->{$binary} = $st->mtime;
1234 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1235 $kvm_user_version->{$binary} = $2;
1239 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1242 return $kvm_user_version->{$binary};
1245 my sub extract_version
{
1246 my ($machine_type, $version) = @_;
1247 $version = kvm_user_version
() if !defined($version);
1248 return PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1251 sub kernel_has_vhost_net
{
1252 return -c
'/dev/vhost-net';
1257 return defined($confdesc->{$key});
1261 sub get_cdrom_path
{
1263 return $cdrom_path if defined($cdrom_path);
1265 $cdrom_path = first
{ -l
$_ } map { "/dev/cdrom$_" } ('', '1', '2');
1267 if (!defined($cdrom_path)) {
1268 log_warn
("no physical CD-ROM available, ignoring");
1276 my ($storecfg, $vmid, $cdrom) = @_;
1278 if ($cdrom eq 'cdrom') {
1279 return get_cdrom_path
();
1280 } elsif ($cdrom eq 'none') {
1282 } elsif ($cdrom =~ m
|^/|) {
1285 return PVE
::Storage
::path
($storecfg, $cdrom);
1289 # try to convert old style file names to volume IDs
1290 sub filename_to_volume_id
{
1291 my ($vmid, $file, $media) = @_;
1293 if (!($file eq 'none' || $file eq 'cdrom' ||
1294 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1296 return if $file =~ m
|/|;
1298 if ($media && $media eq 'cdrom') {
1299 $file = "local:iso/$file";
1301 $file = "local:$vmid/$file";
1308 sub verify_media_type
{
1309 my ($opt, $vtype, $media) = @_;
1314 if ($media eq 'disk') {
1316 } elsif ($media eq 'cdrom') {
1319 die "internal error";
1322 return if ($vtype eq $etype);
1324 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1327 sub cleanup_drive_path
{
1328 my ($opt, $storecfg, $drive) = @_;
1330 # try to convert filesystem paths to volume IDs
1332 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1333 ($drive->{file
} !~ m
|^/dev/.+|) &&
1334 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1335 ($drive->{file
} !~ m/^\d+$/)) {
1336 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1337 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1339 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1340 verify_media_type
($opt, $vtype, $drive->{media
});
1341 $drive->{file
} = $volid;
1344 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1347 sub parse_hotplug_features
{
1352 return $res if $data eq '0';
1354 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1356 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1357 if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1360 die "invalid hotplug feature '$feature'\n";
1366 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1367 sub pve_verify_hotplug_features
{
1368 my ($value, $noerr) = @_;
1370 return $value if parse_hotplug_features
($value);
1374 die "unable to parse hotplug option\n";
1378 my($fh, $noerr) = @_;
1381 my $SG_GET_VERSION_NUM = 0x2282;
1383 my $versionbuf = "\x00" x
8;
1384 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1386 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1389 my $version = unpack("I", $versionbuf);
1390 if ($version < 30000) {
1391 die "scsi generic interface too old\n" if !$noerr;
1395 my $buf = "\x00" x
36;
1396 my $sensebuf = "\x00" x
8;
1397 my $cmd = pack("C x3 C x1", 0x12, 36);
1399 # see /usr/include/scsi/sg.h
1400 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1403 $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1406 $ret = ioctl($fh, $SG_IO, $packet);
1408 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1412 my @res = unpack($sg_io_hdr_t, $packet);
1413 if ($res[17] || $res[18]) {
1414 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1419 $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1421 $res->{removable
} = $res->{removable
} & 128 ?
1 : 0;
1422 $res->{type
} &= 0x1F;
1430 my $fh = IO
::File-
>new("+<$path") || return;
1431 my $res = scsi_inquiry
($fh, 1);
1437 sub print_tabletdevice_full
{
1438 my ($conf, $arch) = @_;
1440 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1442 # we use uhci for old VMs because tablet driver was buggy in older qemu
1444 if ($q35 || $arch eq 'aarch64') {
1450 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1453 sub print_keyboarddevice_full
{
1454 my ($conf, $arch) = @_;
1456 return if $arch ne 'aarch64';
1458 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1461 my sub get_drive_id
{
1463 return "$drive->{interface}$drive->{index}";
1466 sub print_drivedevice_full
{
1467 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1472 my $drive_id = get_drive_id
($drive);
1473 if ($drive->{interface
} eq 'virtio') {
1474 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1475 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1476 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1477 } elsif ($drive->{interface
} eq 'scsi') {
1479 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1480 my $unit = $drive->{index} % $maxdev;
1481 my $devicetype = 'hd';
1483 if (drive_is_cdrom
($drive)) {
1486 if ($drive->{file
} =~ m
|^/|) {
1487 $path = $drive->{file
};
1488 if (my $info = path_is_scsi
($path)) {
1489 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1490 $devicetype = 'block';
1491 } elsif ($info->{type
} == 1) { # tape
1492 $devicetype = 'generic';
1496 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1499 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1500 my $version = extract_version
($machine_type, kvm_user_version
());
1501 if ($path =~ m/^iscsi\:\/\
// &&
1502 !min_version
($version, 4, 1)) {
1503 $devicetype = 'generic';
1507 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1508 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1510 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1511 .",lun=$drive->{index}";
1513 $device .= ",drive=drive-$drive_id,id=$drive_id";
1515 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1516 $device .= ",rotation_rate=1";
1518 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1520 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1521 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1522 my $controller = int($drive->{index} / $maxdev);
1523 my $unit = $drive->{index} % $maxdev;
1524 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1526 $device = "ide-$devicetype";
1527 if ($drive->{interface
} eq 'ide') {
1528 $device .= ",bus=ide.$controller,unit=$unit";
1530 $device .= ",bus=ahci$controller.$unit";
1532 $device .= ",drive=drive-$drive_id,id=$drive_id";
1534 if ($devicetype eq 'hd') {
1535 if (my $model = $drive->{model
}) {
1536 $model = URI
::Escape
::uri_unescape
($model);
1537 $device .= ",model=$model";
1539 if ($drive->{ssd
}) {
1540 $device .= ",rotation_rate=1";
1543 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1544 } elsif ($drive->{interface
} eq 'usb') {
1546 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1548 die "unsupported interface type";
1551 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1553 if (my $serial = $drive->{serial
}) {
1554 $serial = URI
::Escape
::uri_unescape
($serial);
1555 $device .= ",serial=$serial";
1562 sub get_initiator_name
{
1565 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1566 while (defined(my $line = <$fh>)) {
1567 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1576 my sub storage_allows_io_uring_default
{
1577 my ($scfg, $cache_direct) = @_;
1579 # io_uring with cache mode writeback or writethrough on krbd will hang...
1580 return if $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1582 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1583 # sometimes, just plain disable...
1584 return if $scfg && $scfg->{type
} eq 'lvm';
1586 # io_uring causes problems when used with CIFS since kernel 5.15
1587 # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1588 return if $scfg && $scfg->{type
} eq 'cifs';
1593 my sub drive_uses_cache_direct
{
1594 my ($drive, $scfg) = @_;
1596 my $cache_direct = 0;
1598 if (my $cache = $drive->{cache
}) {
1599 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1600 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1604 return $cache_direct;
1607 sub print_drive_commandline_full
{
1608 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1611 my $volid = $drive->{file
};
1612 my $format = $drive->{format
};
1613 my $drive_id = get_drive_id
($drive);
1615 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1616 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1618 if (drive_is_cdrom
($drive)) {
1619 $path = get_iso_path
($storecfg, $vmid, $volid);
1620 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1623 $path = PVE
::Storage
::path
($storecfg, $volid);
1624 $format //= qemu_img_format
($scfg, $volname);
1631 my $is_rbd = $path =~ m/^rbd:/;
1634 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1635 foreach my $o (@qemu_drive_options) {
1636 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1639 # snapshot only accepts on|off
1640 if (defined($drive->{snapshot
})) {
1641 my $v = $drive->{snapshot
} ?
'on' : 'off';
1642 $opts .= ",snapshot=$v";
1645 if (defined($drive->{ro
})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1646 $opts .= ",readonly=" . ($drive->{ro
} ?
'on' : 'off');
1649 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1650 my ($dir, $qmpname) = @$type;
1651 if (my $v = $drive->{"mbps$dir"}) {
1652 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1654 if (my $v = $drive->{"mbps${dir}_max"}) {
1655 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1657 if (my $v = $drive->{"bps${dir}_max_length"}) {
1658 $opts .= ",throttling.bps$qmpname-max-length=$v";
1660 if (my $v = $drive->{"iops${dir}"}) {
1661 $opts .= ",throttling.iops$qmpname=$v";
1663 if (my $v = $drive->{"iops${dir}_max"}) {
1664 $opts .= ",throttling.iops$qmpname-max=$v";
1666 if (my $v = $drive->{"iops${dir}_max_length"}) {
1667 $opts .= ",throttling.iops$qmpname-max-length=$v";
1672 $format = "rbd" if $is_rbd;
1673 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1675 $opts .= ",format=alloc-track,file.driver=$format";
1677 $opts .= ",format=$format";
1680 my $cache_direct = drive_uses_cache_direct
($drive, $scfg);
1682 $opts .= ",cache=none" if !$drive->{cache
} && $cache_direct;
1684 if (!$drive->{aio
}) {
1685 if ($io_uring && storage_allows_io_uring_default
($scfg, $cache_direct)) {
1686 # io_uring supports all cache modes
1687 $opts .= ",aio=io_uring";
1689 # aio native works only with O_DIRECT
1691 $opts .= ",aio=native";
1693 $opts .= ",aio=threads";
1698 if (!drive_is_cdrom
($drive)) {
1700 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1701 $detectzeroes = 'off';
1702 } elsif ($drive->{discard
}) {
1703 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1705 # This used to be our default with discard not being specified:
1706 $detectzeroes = 'on';
1709 # note: 'detect-zeroes' works per blockdev and we want it to persist
1710 # after the alloc-track is removed, so put it on 'file' directly
1711 my $dz_param = $pbs_name ?
"file.detect-zeroes" : "detect-zeroes";
1712 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1716 $opts .= ",backing=$pbs_name";
1717 $opts .= ",auto-remove=on";
1720 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1721 my $file_param = "file";
1723 # non-rbd drivers require the underlying file to be a seperate block
1724 # node, so add a second .file indirection
1725 $file_param .= ".file" if !$is_rbd;
1726 $file_param .= ".filename";
1728 my $pathinfo = $path ?
"$file_param=$path," : '';
1730 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1733 sub print_pbs_blockdev
{
1734 my ($pbs_conf, $pbs_name) = @_;
1735 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1736 $blockdev .= ",repository=$pbs_conf->{repository}";
1737 $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace
};
1738 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1739 $blockdev .= ",archive=$pbs_conf->{archive}";
1740 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1744 sub print_netdevice_full
{
1745 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1747 my $device = $net->{model
};
1748 if ($net->{model
} eq 'virtio') {
1749 $device = 'virtio-net-pci';
1752 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1753 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1754 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1755 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1756 # and out of each queue plus one config interrupt and control vector queue
1757 my $vectors = $net->{queues
} * 2 + 2;
1758 $tmpstr .= ",vectors=$vectors,mq=on";
1759 if (min_version
($machine_version, 7, 1)) {
1760 $tmpstr .= ",packed=on";
1764 if (min_version
($machine_version, 7, 1) && $net->{model
} eq 'virtio'){
1765 $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1768 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1770 if (my $mtu = $net->{mtu
}) {
1771 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1772 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1775 } elsif ($mtu < 576) {
1776 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1777 } elsif ($mtu > $bridge_mtu) {
1778 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1780 $tmpstr .= ",host_mtu=$mtu";
1782 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1786 if ($use_old_bios_files) {
1788 if ($device eq 'virtio-net-pci') {
1789 $romfile = 'pxe-virtio.rom';
1790 } elsif ($device eq 'e1000') {
1791 $romfile = 'pxe-e1000.rom';
1792 } elsif ($device eq 'e1000e') {
1793 $romfile = 'pxe-e1000e.rom';
1794 } elsif ($device eq 'ne2k') {
1795 $romfile = 'pxe-ne2k_pci.rom';
1796 } elsif ($device eq 'pcnet') {
1797 $romfile = 'pxe-pcnet.rom';
1798 } elsif ($device eq 'rtl8139') {
1799 $romfile = 'pxe-rtl8139.rom';
1801 $tmpstr .= ",romfile=$romfile" if $romfile;
1807 sub print_netdev_full
{
1808 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1811 if ($netid =~ m/^net(\d+)$/) {
1815 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1817 my $ifname = "tap${vmid}i$i";
1819 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1820 die "interface name '$ifname' is too long (max 15 character)\n"
1821 if length($ifname) >= 16;
1823 my $vhostparam = '';
1824 if (is_native
($arch)) {
1825 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1828 my $vmname = $conf->{name
} || "vm$vmid";
1831 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1833 if ($net->{bridge
}) {
1834 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1835 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1837 $netdev = "type=user,id=$netid,hostname=$vmname";
1840 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1846 'cirrus' => 'cirrus-vga',
1848 'vmware' => 'vmware-svga',
1849 'virtio' => 'virtio-vga',
1850 'virtio-gl' => 'virtio-vga-gl',
1853 sub print_vga_device
{
1854 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1856 my $type = $vga_map->{$vga->{type
}};
1857 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1858 $type = 'virtio-gpu';
1860 my $vgamem_mb = $vga->{memory
};
1862 my $max_outputs = '';
1864 $type = $id ?
'qxl' : 'qxl-vga';
1866 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1867 # set max outputs so linux can have up to 4 qxl displays with one device
1868 if (min_version
($machine_version, 4, 1)) {
1869 $max_outputs = ",max_outputs=4";
1874 die "no devicetype for $vga->{type}\n" if !$type;
1878 if ($vga->{type
} =~ /^virtio/) {
1879 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1880 $memory = ",max_hostmem=$bytes";
1882 # from https://www.spice-space.org/multiple-monitors.html
1883 $memory = ",vgamem_mb=$vga->{memory}";
1884 my $ram = $vgamem_mb * 4;
1885 my $vram = $vgamem_mb * 2;
1886 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1888 $memory = ",vgamem_mb=$vga->{memory}";
1890 } elsif ($qxlnum && $id) {
1891 $memory = ",ram_size=67108864,vram_size=33554432";
1895 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1896 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1899 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1900 my $vgaid = "vga" . ($id // '');
1902 if ($q35 && $vgaid eq 'vga') {
1903 # the first display uses pcie.0 bus on q35 machines
1904 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1906 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1909 if ($vga->{type
} eq 'virtio-gl') {
1910 my $base = '/usr/lib/x86_64-linux-gnu/lib';
1911 die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1912 if !-e
"${base}EGL.so.1" || !-e
"${base}GL.so.1";
1914 die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1915 if !PVE
::Tools
::dir_glob_regex
('/dev/dri/', "renderD.*");
1918 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1921 sub parse_number_sets
{
1924 foreach my $part (split(/;/, $set)) {
1925 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1926 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1927 push @$res, [ $1, $2 ];
1929 die "invalid range: $part\n";
1938 my $res = parse_property_string
($numa_fmt, $data);
1939 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1940 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1944 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1946 my ($data, $disable_mac_autogen) = @_;
1948 my $res = eval { parse_property_string
($net_fmt, $data) };
1953 if (!defined($res->{macaddr
}) && !$disable_mac_autogen) {
1954 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1955 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1960 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1961 sub parse_ipconfig
{
1964 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1970 if ($res->{gw
} && !$res->{ip
}) {
1971 warn 'gateway specified without specifying an IP address';
1974 if ($res->{gw6
} && !$res->{ip6
}) {
1975 warn 'IPv6 gateway specified without specifying an IPv6 address';
1978 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1979 warn 'gateway specified together with DHCP';
1982 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1984 warn "IPv6 gateway specified together with $res->{ip6} address";
1988 if (!$res->{ip
} && !$res->{ip6
}) {
1989 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1998 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
2001 sub add_random_macs
{
2002 my ($settings) = @_;
2004 foreach my $opt (keys %$settings) {
2005 next if $opt !~ m/^net(\d+)$/;
2006 my $net = parse_net
($settings->{$opt});
2008 $settings->{$opt} = print_net
($net);
2012 sub vm_is_volid_owner
{
2013 my ($storecfg, $vmid, $volid) = @_;
2015 if ($volid !~ m
|^/|) {
2017 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
2018 if ($owner && ($owner == $vmid)) {
2026 sub vmconfig_register_unused_drive
{
2027 my ($storecfg, $vmid, $conf, $drive) = @_;
2029 if (drive_is_cloudinit
($drive)) {
2030 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2032 delete $conf->{cloudinit
};
2033 } elsif (!drive_is_cdrom
($drive)) {
2034 my $volid = $drive->{file
};
2035 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2036 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2041 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2045 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2046 format_description
=> 'UUID',
2047 description
=> "Set SMBIOS1 UUID.",
2052 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2053 format_description
=> 'Base64 encoded string',
2054 description
=> "Set SMBIOS1 version.",
2059 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2060 format_description
=> 'Base64 encoded string',
2061 description
=> "Set SMBIOS1 serial number.",
2066 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2067 format_description
=> 'Base64 encoded string',
2068 description
=> "Set SMBIOS1 manufacturer.",
2073 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2074 format_description
=> 'Base64 encoded string',
2075 description
=> "Set SMBIOS1 product ID.",
2080 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2081 format_description
=> 'Base64 encoded string',
2082 description
=> "Set SMBIOS1 SKU string.",
2087 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2088 format_description
=> 'Base64 encoded string',
2089 description
=> "Set SMBIOS1 family string.",
2094 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2102 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2109 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2112 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2114 sub parse_watchdog
{
2119 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2124 sub parse_guest_agent
{
2127 return {} if !defined($conf->{agent
});
2129 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2132 # if the agent is disabled ignore the other potentially set properties
2133 return {} if !$res->{enabled
};
2138 my ($conf, $key) = @_;
2139 return undef if !defined($conf->{agent
});
2141 my $agent = parse_guest_agent
($conf);
2142 return $agent->{$key};
2148 return {} if !$value;
2149 my $res = eval { parse_property_string
($vga_fmt, $value) };
2159 my $res = eval { parse_property_string
($rng_fmt, $value) };
2164 sub parse_meta_info
{
2169 my $res = eval { parse_property_string
($meta_info_fmt, $value) };
2174 sub new_meta_info_string
{
2175 my () = @_; # for now do not allow to override any value
2177 return PVE
::JSONSchema
::print_property_string
(
2179 'creation-qemu' => kvm_user_version
(),
2180 ctime
=> "". int(time()),
2186 sub qemu_created_version_fixups
{
2187 my ($conf, $forcemachine, $kvmver) = @_;
2189 my $meta = parse_meta_info
($conf->{meta
}) // {};
2190 my $forced_vers = PVE
::QemuServer
::Machine
::extract_version
($forcemachine);
2192 # check if we need to apply some handling for VMs that always use the latest machine version but
2193 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2194 # would be required (we do not want to pin machine version for non-windows OS type)
2196 (!defined($conf->{machine
}) || $conf->{machine
} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2197 && (!defined($meta->{'creation-qemu'}) || !min_version
($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2198 && (!$forced_vers || min_version
($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2199 && min_version
($kvmver, 6, 1) # only need to apply the change since 6.1
2201 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2202 if ($q35 && $conf->{ostype
} && $conf->{ostype
} eq 'l26') {
2203 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2204 # and thus with the predictable interface naming of systemd
2205 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2211 # add JSON properties for create and set function
2212 sub json_config_properties
{
2213 my ($prop, $with_disk_alloc) = @_;
2215 my $skip_json_config_opts = {
2219 runningmachine
=> 1,
2224 foreach my $opt (keys %$confdesc) {
2225 next if $skip_json_config_opts->{$opt};
2227 if ($with_disk_alloc && is_valid_drivename
($opt)) {
2228 $prop->{$opt} = $PVE::QemuServer
::Drive
::drivedesc_hash_with_alloc-
>{$opt};
2230 $prop->{$opt} = $confdesc->{$opt};
2237 # Properties that we can read from an OVF file
2238 sub json_ovf_properties
{
2241 for my $device (PVE
::QemuServer
::Drive
::valid_drive_names
()) {
2242 $prop->{$device} = {
2244 format
=> 'pve-volume-id-or-absolute-path',
2245 description
=> "Disk image that gets imported to $device",
2252 description
=> "The number of CPU cores.",
2257 description
=> "Amount of RAM for the VM in MB.",
2262 description
=> "Name of the VM.",
2269 # return copy of $confdesc_cloudinit to generate documentation
2270 sub cloudinit_config_properties
{
2272 return dclone
($confdesc_cloudinit);
2275 sub cloudinit_pending_properties
{
2277 map { $_ => 1 } keys $confdesc_cloudinit->%*,
2280 $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2285 my ($key, $value) = @_;
2287 die "unknown setting '$key'\n" if !$confdesc->{$key};
2289 my $type = $confdesc->{$key}->{type
};
2291 if (!defined($value)) {
2292 die "got undefined value\n";
2295 if ($value =~ m/[\n\r]/) {
2296 die "property contains a line feed\n";
2299 if ($type eq 'boolean') {
2300 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2301 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2302 die "type check ('boolean') failed - got '$value'\n";
2303 } elsif ($type eq 'integer') {
2304 return int($1) if $value =~ m/^(\d+)$/;
2305 die "type check ('integer') failed - got '$value'\n";
2306 } elsif ($type eq 'number') {
2307 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2308 die "type check ('number') failed - got '$value'\n";
2309 } elsif ($type eq 'string') {
2310 if (my $fmt = $confdesc->{$key}->{format
}) {
2311 PVE
::JSONSchema
::check_format
($fmt, $value);
2314 $value =~ s/^\"(.*)\"$/$1/;
2317 die "internal error"
2322 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2324 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2326 if (!$skiplock && !PVE
::QemuConfig-
>has_lock($conf, 'suspended')) {
2327 PVE
::QemuConfig-
>check_lock($conf);
2330 if ($conf->{template
}) {
2331 # check if any base image is still used by a linked clone
2332 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2333 my ($ds, $drive) = @_;
2334 return if drive_is_cdrom
($drive);
2336 my $volid = $drive->{file
};
2337 return if !$volid || $volid =~ m
|^/|;
2339 die "base volume '$volid' is still in use by linked cloned\n"
2340 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2346 my $remove_owned_drive = sub {
2347 my ($ds, $drive) = @_;
2348 return if drive_is_cdrom
($drive, 1);
2350 my $volid = $drive->{file
};
2351 return if !$volid || $volid =~ m
|^/|;
2352 return if $volids->{$volid};
2354 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2355 return if !$path || !$owner || ($owner != $vmid);
2357 $volids->{$volid} = 1;
2358 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2359 warn "Could not remove disk '$volid', check manually: $@" if $@;
2362 # only remove disks owned by this VM (referenced in the config)
2363 my $include_opts = {
2364 include_unused
=> 1,
2365 extra_keys
=> ['vmstate'],
2367 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2369 for my $snap (values %{$conf->{snapshots
}}) {
2370 next if !defined($snap->{vmstate
});
2371 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2372 next if !defined($drive);
2373 $remove_owned_drive->('vmstate', $drive);
2376 PVE
::QemuConfig-
>foreach_volume_full($conf->{pending
}, $include_opts, $remove_owned_drive);
2378 if ($purge_unreferenced) { # also remove unreferenced disk
2379 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2380 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2381 my ($volid, $sid, $volname, $d) = @_;
2382 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2387 if (defined $replacement_conf) {
2388 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2390 PVE
::QemuConfig-
>destroy_config($vmid);
2394 sub parse_vm_config
{
2395 my ($filename, $raw, $strict) = @_;
2397 return if !defined($raw);
2400 digest
=> Digest
::SHA
::sha1_hex
($raw),
2406 my $handle_error = sub {
2416 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2417 || die "got strange filename '$filename'";
2423 my $finish_description = sub {
2424 if (defined($descr)) {
2426 $conf->{description
} = $descr;
2432 my @lines = split(/\n/, $raw);
2433 foreach my $line (@lines) {
2434 next if $line =~ m/^\s*$/;
2436 if ($line =~ m/^\[PENDING\]\s*$/i) {
2437 $section = 'pending';
2438 $finish_description->();
2439 $conf = $res->{$section} = {};
2441 } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2442 $section = 'cloudinit';
2443 $finish_description->();
2444 $conf = $res->{$section} = {};
2447 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2449 $finish_description->();
2450 $conf = $res->{snapshots
}->{$section} = {};
2454 if ($line =~ m/^\#(.*)$/) {
2455 $descr = '' if !defined($descr);
2456 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2460 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2461 $descr = '' if !defined($descr);
2462 $descr .= PVE
::Tools
::decode_text
($2);
2463 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2464 $conf->{snapstate
} = $1;
2465 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2468 $conf->{$key} = $value;
2469 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2471 if ($section eq 'pending') {
2472 $conf->{delete} = $value; # we parse this later
2474 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2476 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2479 if ($section eq 'cloudinit') {
2480 # ignore validation only used for informative purpose
2481 $conf->{$key} = $value;
2484 eval { $value = check_type
($key, $value); };
2486 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2488 $key = 'ide2' if $key eq 'cdrom';
2489 my $fmt = $confdesc->{$key}->{format
};
2490 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2491 my $v = parse_drive
($key, $value);
2492 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2493 $v->{file
} = $volid;
2494 $value = print_drive
($v);
2496 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2501 $conf->{$key} = $value;
2504 $handle_error->("vm $vmid - unable to parse config: $line\n");
2508 $finish_description->();
2509 delete $res->{snapstate
}; # just to be sure
2514 sub write_vm_config
{
2515 my ($filename, $conf) = @_;
2517 delete $conf->{snapstate
}; # just to be sure
2519 if ($conf->{cdrom
}) {
2520 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2521 $conf->{ide2
} = $conf->{cdrom
};
2522 delete $conf->{cdrom
};
2525 # we do not use 'smp' any longer
2526 if ($conf->{sockets
}) {
2527 delete $conf->{smp
};
2528 } elsif ($conf->{smp
}) {
2529 $conf->{sockets
} = $conf->{smp
};
2530 delete $conf->{cores
};
2531 delete $conf->{smp
};
2534 my $used_volids = {};
2536 my $cleanup_config = sub {
2537 my ($cref, $pending, $snapname) = @_;
2539 foreach my $key (keys %$cref) {
2540 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2541 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2542 my $value = $cref->{$key};
2543 if ($key eq 'delete') {
2544 die "propertry 'delete' is only allowed in [PENDING]\n"
2546 # fixme: check syntax?
2549 eval { $value = check_type
($key, $value); };
2550 die "unable to parse value of '$key' - $@" if $@;
2552 $cref->{$key} = $value;
2554 if (!$snapname && is_valid_drivename
($key)) {
2555 my $drive = parse_drive
($key, $value);
2556 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2561 &$cleanup_config($conf);
2563 &$cleanup_config($conf->{pending
}, 1);
2565 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2566 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2567 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2570 # remove 'unusedX' settings if we re-add a volume
2571 foreach my $key (keys %$conf) {
2572 my $value = $conf->{$key};
2573 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2574 delete $conf->{$key};
2578 my $generate_raw_config = sub {
2579 my ($conf, $pending) = @_;
2583 # add description as comment to top of file
2584 if (defined(my $descr = $conf->{description
})) {
2586 foreach my $cl (split(/\n/, $descr)) {
2587 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2590 $raw .= "#\n" if $pending;
2594 foreach my $key (sort keys %$conf) {
2595 next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2596 $raw .= "$key: $conf->{$key}\n";
2601 my $raw = &$generate_raw_config($conf);
2603 if (scalar(keys %{$conf->{pending
}})){
2604 $raw .= "\n[PENDING]\n";
2605 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2608 if (scalar(keys %{$conf->{cloudinit
}}) && PVE
::QemuConfig-
>has_cloudinit($conf)){
2609 $raw .= "\n[special:cloudinit]\n";
2610 $raw .= &$generate_raw_config($conf->{cloudinit
});
2613 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2614 $raw .= "\n[$snapname]\n";
2615 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2625 # we use static defaults from our JSON schema configuration
2626 foreach my $key (keys %$confdesc) {
2627 if (defined(my $default = $confdesc->{$key}->{default})) {
2628 $res->{$key} = $default;
2636 my $vmlist = PVE
::Cluster
::get_vmlist
();
2638 return $res if !$vmlist || !$vmlist->{ids
};
2639 my $ids = $vmlist->{ids
};
2640 my $nodename = nodename
();
2642 foreach my $vmid (keys %$ids) {
2643 my $d = $ids->{$vmid};
2644 next if !$d->{node
} || $d->{node
} ne $nodename;
2645 next if !$d->{type
} || $d->{type
} ne 'qemu';
2646 $res->{$vmid}->{exists} = 1;
2651 # test if VM uses local resources (to prevent migration)
2652 sub check_local_resources
{
2653 my ($conf, $noerr) = @_;
2656 my $mapped_res = [];
2658 my $nodelist = PVE
::Cluster
::get_nodelist
();
2659 my $pci_map = PVE
::Mapping
::PCI
::config
();
2660 my $usb_map = PVE
::Mapping
::USB
::config
();
2662 my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2664 my $add_missing_mapping = sub {
2665 my ($type, $key, $id) = @_;
2666 for my $node (@$nodelist) {
2668 if ($type eq 'pci') {
2669 $entry = PVE
::Mapping
::PCI
::get_node_mapping
($pci_map, $id, $node);
2670 } elsif ($type eq 'usb') {
2671 $entry = PVE
::Mapping
::USB
::get_node_mapping
($usb_map, $id, $node);
2673 if (!scalar($entry->@*)) {
2674 push @{$missing_mappings_by_node->{$node}}, $key;
2679 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2680 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2682 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2684 foreach my $k (keys %$conf) {
2685 if ($k =~ m/^usb/) {
2686 my $entry = parse_property_string
('pve-qm-usb', $conf->{$k});
2687 next if $entry->{host
} =~ m/^spice$/i;
2688 if ($entry->{mapping
}) {
2689 $add_missing_mapping->('usb', $k, $entry->{mapping
});
2690 push @$mapped_res, $k;
2693 if ($k =~ m/^hostpci/) {
2694 my $entry = parse_property_string
('pve-qm-hostpci', $conf->{$k});
2695 if ($entry->{mapping
}) {
2696 $add_missing_mapping->('pci', $k, $entry->{mapping
});
2697 push @$mapped_res, $k;
2700 # sockets are safe: they will recreated be on the target side post-migrate
2701 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2702 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2705 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2707 return wantarray ?
(\
@loc_res, $mapped_res, $missing_mappings_by_node) : \
@loc_res;
2710 # check if used storages are available on all nodes (use by migrate)
2711 sub check_storage_availability
{
2712 my ($storecfg, $conf, $node) = @_;
2714 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2715 my ($ds, $drive) = @_;
2717 my $volid = $drive->{file
};
2720 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2723 # check if storage is available on both nodes
2724 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2725 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2727 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2729 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2730 if !$scfg->{content
}->{$vtype};
2734 # list nodes where all VM images are available (used by has_feature API)
2736 my ($conf, $storecfg) = @_;
2738 my $nodelist = PVE
::Cluster
::get_nodelist
();
2739 my $nodehash = { map { $_ => 1 } @$nodelist };
2740 my $nodename = nodename
();
2742 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2743 my ($ds, $drive) = @_;
2745 my $volid = $drive->{file
};
2748 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2750 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2751 if ($scfg->{disable
}) {
2753 } elsif (my $avail = $scfg->{nodes
}) {
2754 foreach my $node (keys %$nodehash) {
2755 delete $nodehash->{$node} if !$avail->{$node};
2757 } elsif (!$scfg->{shared
}) {
2758 foreach my $node (keys %$nodehash) {
2759 delete $nodehash->{$node} if $node ne $nodename
2768 sub check_local_storage_availability
{
2769 my ($conf, $storecfg) = @_;
2771 my $nodelist = PVE
::Cluster
::get_nodelist
();
2772 my $nodehash = { map { $_ => {} } @$nodelist };
2774 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2775 my ($ds, $drive) = @_;
2777 my $volid = $drive->{file
};
2780 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2782 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2784 if ($scfg->{disable
}) {
2785 foreach my $node (keys %$nodehash) {
2786 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2788 } elsif (my $avail = $scfg->{nodes
}) {
2789 foreach my $node (keys %$nodehash) {
2790 if (!$avail->{$node}) {
2791 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2798 foreach my $node (values %$nodehash) {
2799 if (my $unavail = $node->{unavailable_storages
}) {
2800 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2807 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2809 my ($vmid, $nocheck, $node) = @_;
2811 # $nocheck is set when called during a migration, in which case the config
2812 # file might still or already reside on the *other* node
2813 # - because rename has already happened, and current node is source
2814 # - because rename hasn't happened yet, and current node is target
2815 # - because rename has happened, current node is target, but hasn't yet
2817 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2818 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2823 my $vzlist = config_list
();
2825 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2827 while (defined(my $de = $fd->read)) {
2828 next if $de !~ m/^(\d+)\.pid$/;
2830 next if !defined($vzlist->{$vmid});
2831 if (my $pid = check_running
($vmid)) {
2832 $vzlist->{$vmid}->{pid
} = $pid;
2839 our $vmstatus_return_properties = {
2840 vmid
=> get_standard_option
('pve-vmid'),
2842 description
=> "QEMU process status.",
2844 enum
=> ['stopped', 'running'],
2847 description
=> "Maximum memory in bytes.",
2850 renderer
=> 'bytes',
2853 description
=> "Root disk size in bytes.",
2856 renderer
=> 'bytes',
2859 description
=> "VM name.",
2864 description
=> "VM run state from the 'query-status' QMP monitor command.",
2869 description
=> "PID of running qemu process.",
2874 description
=> "Uptime.",
2877 renderer
=> 'duration',
2880 description
=> "Maximum usable CPUs.",
2885 description
=> "The current config lock, if any.",
2890 description
=> "The current configured tags, if any",
2894 'running-machine' => {
2895 description
=> "The currently running machine type (if running).",
2900 description
=> "The currently running QEMU version (if running).",
2906 my $last_proc_pid_stat;
2908 # get VM status information
2909 # This must be fast and should not block ($full == false)
2910 # We only query KVM using QMP if $full == true (this can be slow)
2912 my ($opt_vmid, $full) = @_;
2916 my $storecfg = PVE
::Storage
::config
();
2918 my $list = vzlist
();
2919 my $defaults = load_defaults
();
2921 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2923 my $cpucount = $cpuinfo->{cpus
} || 1;
2925 foreach my $vmid (keys %$list) {
2926 next if $opt_vmid && ($vmid ne $opt_vmid);
2928 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2930 my $d = { vmid
=> int($vmid) };
2931 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2933 # fixme: better status?
2934 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2936 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2937 if (defined($size)) {
2938 $d->{disk
} = 0; # no info available
2939 $d->{maxdisk
} = $size;
2945 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2946 * ($conf->{cores
} || $defaults->{cores
});
2947 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2948 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2950 $d->{name
} = $conf->{name
} || "VM $vmid";
2951 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2952 : $defaults->{memory
}*(1024*1024);
2954 if ($conf->{balloon
}) {
2955 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2956 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2957 : $defaults->{shares
};
2968 $d->{diskwrite
} = 0;
2970 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2972 $d->{serial
} = 1 if conf_has_serial
($conf);
2973 $d->{lock} = $conf->{lock} if $conf->{lock};
2974 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2979 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2980 foreach my $dev (keys %$netdev) {
2981 next if $dev !~ m/^tap([1-9]\d*)i/;
2983 my $d = $res->{$vmid};
2986 $d->{netout
} += $netdev->{$dev}->{receive
};
2987 $d->{netin
} += $netdev->{$dev}->{transmit
};
2990 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2991 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2996 my $ctime = gettimeofday
;
2998 foreach my $vmid (keys %$list) {
3000 my $d = $res->{$vmid};
3001 my $pid = $d->{pid
};
3004 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
3005 next if !$pstat; # not running
3007 my $used = $pstat->{utime} + $pstat->{stime
};
3009 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
3011 if ($pstat->{vsize
}) {
3012 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
3015 my $old = $last_proc_pid_stat->{$pid};
3017 $last_proc_pid_stat->{$pid} = {
3025 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
3027 if ($dtime > 1000) {
3028 my $dutime = $used - $old->{used
};
3030 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
3031 $last_proc_pid_stat->{$pid} = {
3037 $d->{cpu
} = $old->{cpu
};
3041 return $res if !$full;
3043 my $qmpclient = PVE
::QMPClient-
>new();
3045 my $ballooncb = sub {
3046 my ($vmid, $resp) = @_;
3048 my $info = $resp->{'return'};
3049 return if !$info->{max_mem
};
3051 my $d = $res->{$vmid};
3053 # use memory assigned to VM
3054 $d->{maxmem
} = $info->{max_mem
};
3055 $d->{balloon
} = $info->{actual
};
3057 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
3058 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
3059 $d->{freemem
} = $info->{free_mem
};
3062 $d->{ballooninfo
} = $info;
3065 my $blockstatscb = sub {
3066 my ($vmid, $resp) = @_;
3067 my $data = $resp->{'return'} || [];
3068 my $totalrdbytes = 0;
3069 my $totalwrbytes = 0;
3071 for my $blockstat (@$data) {
3072 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
3073 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
3075 $blockstat->{device
} =~ s/drive-//;
3076 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
3078 $res->{$vmid}->{diskread
} = $totalrdbytes;
3079 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
3082 my $machinecb = sub {
3083 my ($vmid, $resp) = @_;
3084 my $data = $resp->{'return'} || [];
3086 $res->{$vmid}->{'running-machine'} =
3087 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
3090 my $versioncb = sub {
3091 my ($vmid, $resp) = @_;
3092 my $data = $resp->{'return'} // {};
3093 my $version = 'unknown';
3095 if (my $v = $data->{qemu
}) {
3096 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
3099 $res->{$vmid}->{'running-qemu'} = $version;
3102 my $statuscb = sub {
3103 my ($vmid, $resp) = @_;
3105 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3106 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3107 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3108 # this fails if ballon driver is not loaded, so this must be
3109 # the last commnand (following command are aborted if this fails).
3110 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3112 my $status = 'unknown';
3113 if (!defined($status = $resp->{'return'}->{status
})) {
3114 warn "unable to get VM status\n";
3118 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3121 foreach my $vmid (keys %$list) {
3122 next if $opt_vmid && ($vmid ne $opt_vmid);
3123 next if !$res->{$vmid}->{pid
}; # not running
3124 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3127 $qmpclient->queue_execute(undef, 2);
3129 foreach my $vmid (keys %$list) {
3130 next if $opt_vmid && ($vmid ne $opt_vmid);
3131 next if !$res->{$vmid}->{pid
}; #not running
3133 # we can't use the $qmpclient since it might have already aborted on
3134 # 'query-balloon', but this might also fail for older versions...
3135 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
3136 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3139 foreach my $vmid (keys %$list) {
3140 next if $opt_vmid && ($vmid ne $opt_vmid);
3141 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3147 sub conf_has_serial
{
3150 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3151 if ($conf->{"serial$i"}) {
3159 sub conf_has_audio
{
3160 my ($conf, $id) = @_;
3163 my $audio = $conf->{"audio$id"};
3164 return if !defined($audio);
3166 my $audioproperties = parse_property_string
($audio_fmt, $audio);
3167 my $audiodriver = $audioproperties->{driver
} // 'spice';
3170 dev
=> $audioproperties->{device
},
3171 dev_id
=> "audiodev$id",
3172 backend
=> $audiodriver,
3173 backend_id
=> "$audiodriver-backend${id}",
3178 my ($audio, $audiopciaddr, $machine_version) = @_;
3182 my $id = $audio->{dev_id
};
3184 if (min_version
($machine_version, 4, 2)) {
3185 $audiodev = ",audiodev=$audio->{backend_id}";
3188 if ($audio->{dev
} eq 'AC97') {
3189 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3190 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3191 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3192 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3193 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3195 die "unkown audio device '$audio->{dev}', implement me!";
3198 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3206 socket => "/var/run/qemu-server/$vmid.swtpm",
3207 pid
=> "/var/run/qemu-server/$vmid.swtpm.pid",
3211 sub add_tpm_device
{
3212 my ($vmid, $devices, $conf) = @_;
3214 return if !$conf->{tpmstate0
};
3216 my $paths = get_tpm_paths
($vmid);
3218 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3219 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3220 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3224 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3226 return if !$tpmdrive;
3229 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
3230 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
3232 $state = PVE
::Storage
::map_volume
($storecfg, $tpm->{file
});
3234 $state = $tpm->{file
};
3237 my $paths = get_tpm_paths
($vmid);
3239 # during migration, we will get state from remote
3242 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3249 "--create-platform-cert",
3252 "/etc/swtpm_setup.conf", # do not use XDG configs
3254 "0", # force creation as root, error if not possible
3255 "--not-overwrite", # ignore existing state, do not modify
3258 push @$setup_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3259 # TPM 2.0 supports ECC crypto, use if possible
3260 push @$setup_cmd, "--ecc" if $tpm->{version
} eq 'v2.0';
3262 run_command
($setup_cmd, outfunc
=> sub {
3263 print "swtpm_setup: $1\n";
3267 # Used to distinguish different invocations in the log.
3268 my $log_prefix = "[id=" . int(time()) . "] ";
3270 my $emulator_cmd = [
3274 "backend-uri=file://$state,mode=0600",
3276 "type=unixio,path=$paths->{socket},mode=0600",
3278 "file=$paths->{pid}",
3279 "--terminate", # terminate on QEMU disconnect
3282 "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3284 push @$emulator_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3285 run_command
($emulator_cmd, outfunc
=> sub { print $1; });
3287 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3288 while (! -e
$paths->{pid
}) {
3289 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3293 # return untainted PID of swtpm daemon so it can be killed on error
3294 file_read_firstline
($paths->{pid
}) =~ m/(\d+)/;
3298 sub vga_conf_has_spice
{
3301 my $vgaconf = parse_vga
($vga);
3302 my $vgatype = $vgaconf->{type
};
3303 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3310 return get_host_arch
() eq $arch;
3315 return $conf->{arch
} // get_host_arch
();
3318 my $default_machines = {
3323 sub get_installed_machine_version
{
3324 my ($kvmversion) = @_;
3325 $kvmversion = kvm_user_version
() if !defined($kvmversion);
3326 $kvmversion =~ m/^(\d+\.\d+)/;
3330 sub windows_get_pinned_machine_version
{
3331 my ($machine, $base_version, $kvmversion) = @_;
3333 my $pin_version = $base_version;
3334 if (!defined($base_version) ||
3335 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3337 $pin_version = get_installed_machine_version
($kvmversion);
3339 if (!$machine || $machine eq 'pc') {
3340 $machine = "pc-i440fx-$pin_version";
3341 } elsif ($machine eq 'q35') {
3342 $machine = "pc-q35-$pin_version";
3343 } elsif ($machine eq 'virt') {
3344 $machine = "virt-$pin_version";
3346 warn "unknown machine type '$machine', not touching that!\n";
3352 sub get_vm_machine
{
3353 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3355 my $machine = $forcemachine || $conf->{machine
};
3357 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3358 $kvmversion //= kvm_user_version
();
3359 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3360 # layout which confuses windows quite a bit and may result in various regressions..
3361 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3362 if (windows_version
($conf->{ostype
})) {
3363 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3366 $machine ||= $default_machines->{$arch};
3367 if ($add_pve_version) {
3368 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3369 $machine .= "+pve$pvever";
3373 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3374 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3375 $machine = $1 if $is_pxe;
3377 # for version-pinned machines that do not include a pve-version (e.g.
3378 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3379 $machine .= '+pve0';
3381 $machine .= '.pxe' if $is_pxe;
3387 sub get_ovmf_files
($$$) {
3388 my ($arch, $efidisk, $smm) = @_;
3390 my $types = $OVMF->{$arch}
3391 or die "no OVMF images known for architecture '$arch'\n";
3393 my $type = 'default';
3394 if ($arch ne "aarch64" && defined($efidisk->{efitype
}) && $efidisk->{efitype
} eq '4m') {
3395 $type = $smm ?
"4m" : "4m-no-smm";
3396 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3399 my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3400 die "EFI base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3401 die "EFI vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
3403 return ($ovmf_code, $ovmf_vars);
3407 aarch64
=> '/usr/bin/qemu-system-aarch64',
3408 x86_64
=> '/usr/bin/qemu-system-x86_64',
3410 sub get_command_for_arch
($) {
3412 return '/usr/bin/kvm' if is_native
($arch);
3414 my $cmd = $Arch2Qemu->{$arch}
3415 or die "don't know how to emulate architecture '$arch'\n";
3419 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3420 # to use in a QEMU command line (-cpu element), first array_intersect the result
3421 # of query_supported_ with query_understood_. This is necessary because:
3423 # a) query_understood_ returns flags the host cannot use and
3424 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3425 # flags, but CPU settings - with most of them being flags. Those settings
3426 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3428 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3429 # expensive. If you need the value returned from this, you can get it much
3430 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3431 # $accel being 'kvm' or 'tcg'.
3433 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3434 # changes, automatically populating pmxcfs.
3436 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3437 # since kvm and tcg machines support different flags
3439 sub query_supported_cpu_flags
{
3442 $arch //= get_host_arch
();
3443 my $default_machine = $default_machines->{$arch};
3447 # FIXME: Once this is merged, the code below should work for ARM as well:
3448 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3449 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3452 my $kvm_supported = defined(kvm_version
());
3453 my $qemu_cmd = get_command_for_arch
($arch);
3455 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3457 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3458 my $query_supported_run_qemu = sub {
3464 '-machine', $default_machine,
3466 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3467 '-mon', 'chardev=qmp,mode=control',
3468 '-pidfile', $pidfile,
3473 push @$cmd, '-accel', 'tcg';
3476 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3477 die "QEMU flag querying VM exited with code " . $rc if $rc;
3480 my $cmd_result = mon_cmd
(
3482 'query-cpu-model-expansion',
3484 model
=> { name
=> 'host' }
3487 my $props = $cmd_result->{model
}->{props
};
3488 foreach my $prop (keys %$props) {
3489 next if $props->{$prop} ne '1';
3490 # QEMU returns some flags multiple times, with '_', '.' or '-'
3491 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3492 # We only keep those with underscores, to match /proc/cpuinfo
3493 $prop =~ s/\.|-/_/g;
3494 $flags->{$prop} = 1;
3499 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3500 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3504 return [ sort keys %$flags ];
3507 # We need to query QEMU twice, since KVM and TCG have different supported flags
3508 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3509 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3510 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3512 if ($kvm_supported) {
3513 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3514 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3521 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3522 my $understood_cpu_flag_dir = "/usr/share/kvm";
3523 sub query_understood_cpu_flags
{
3524 my $arch = get_host_arch
();
3525 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3527 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3530 my $raw = file_get_contents
($filepath);
3531 $raw =~ s/^\s+|\s+$//g;
3532 my @flags = split(/\s+/, $raw);
3537 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3538 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3539 my sub should_disable_smm
{
3540 my ($conf, $vga, $machine) = @_;
3542 return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3544 return (!defined($conf->{bios
}) || $conf->{bios
} eq 'seabios') &&
3545 $vga->{type
} && $vga->{type
} =~ m/^(serial\d+|none)$/;
3548 my sub print_ovmf_drive_commandlines
{
3549 my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3551 my $d = $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
3553 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch, $d, $q35);
3555 my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3557 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3558 my ($path, $format) = $d->@{'file', 'format'};
3560 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3561 if (!defined($format)) {
3562 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3563 $format = qemu_img_format
($scfg, $volname);
3565 } elsif (!defined($format)) {
3566 die "efidisk format must be specified\n";
3568 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3569 if ($path =~ m/^rbd:/) {
3570 $var_drive_str .= ',cache=writeback';
3571 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3573 $var_drive_str .= ",format=$format,file=$path";
3575 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3576 $var_drive_str .= ',readonly=on' if drive_is_read_only
($conf, $d);
3578 log_warn
("no efidisk configured! Using temporary efivars disk.");
3579 my $path = "/tmp/$vmid-ovmf.fd";
3580 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3581 $var_drive_str .= ",format=raw,file=$path";
3582 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $version_guard->(4, 1, 2);
3585 return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3588 sub config_to_command
{
3589 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3592 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3595 my $ostype = $conf->{ostype
};
3596 my $winversion = windows_version
($ostype);
3597 my $kvm = $conf->{kvm
};
3598 my $nodename = nodename
();
3600 my $arch = get_vm_arch
($conf);
3601 my $kvm_binary = get_command_for_arch
($arch);
3602 my $kvmver = kvm_user_version
($kvm_binary);
3604 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3605 $kvmver //= "undefined";
3606 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3609 my $add_pve_version = min_version
($kvmver, 4, 1);
3611 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3612 my $machine_version = extract_version
($machine_type, $kvmver);
3613 $kvm //= 1 if is_native
($arch);
3615 $machine_version =~ m/(\d+)\.(\d+)/;
3616 my ($machine_major, $machine_minor) = ($1, $2);
3618 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3619 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3620 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3621 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3622 ." please upgrade node '$nodename'\n"
3623 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3624 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3625 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3626 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3627 ." node '$nodename'\n";
3630 # if a specific +pve version is required for a feature, use $version_guard
3631 # instead of min_version to allow machines to be run with the minimum
3633 my $required_pve_version = 0;
3634 my $version_guard = sub {
3635 my ($major, $minor, $pve) = @_;
3636 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3637 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3638 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3639 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3643 if ($kvm && !defined kvm_version
()) {
3644 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3645 ." or enable in BIOS.\n";
3648 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3649 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3650 my $use_old_bios_files = undef;
3651 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3654 if ($conf->{affinity
}) {
3655 push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity
};
3658 push @$cmd, $kvm_binary;
3660 push @$cmd, '-id', $vmid;
3662 my $vmname = $conf->{name
} || "vm$vmid";
3664 push @$cmd, '-name', "$vmname,debug-threads=on";
3666 push @$cmd, '-no-shutdown';
3670 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3671 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3672 push @$cmd, '-mon', "chardev=qmp,mode=control";
3674 if (min_version
($machine_version, 2, 12)) {
3675 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3676 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3679 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3681 push @$cmd, '-daemonize';
3683 if ($conf->{smbios1
}) {
3684 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3685 if ($smbios_conf->{base64
}) {
3686 # Do not pass base64 flag to qemu
3687 delete $smbios_conf->{base64
};
3688 my $smbios_string = "";
3689 foreach my $key (keys %$smbios_conf) {
3691 if ($key eq "uuid") {
3692 $value = $smbios_conf->{uuid
}
3694 $value = decode_base64
($smbios_conf->{$key});
3696 # qemu accepts any binary data, only commas need escaping by double comma
3698 $smbios_string .= "," . $key . "=" . $value if $value;
3700 push @$cmd, '-smbios', "type=1" . $smbios_string;
3702 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3706 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3707 my ($code_drive_str, $var_drive_str) =
3708 print_ovmf_drive_commandlines
($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3709 push $cmd->@*, '-drive', $code_drive_str;
3710 push $cmd->@*, '-drive', $var_drive_str;
3713 if ($q35) { # tell QEMU to load q35 config early
3714 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3715 if (min_version
($machine_version, 4, 0)) {
3716 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3718 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3722 if (defined(my $fixups = qemu_created_version_fixups
($conf, $forcemachine, $kvmver))) {
3723 push @$cmd, $fixups->@*;
3726 if ($conf->{vmgenid
}) {
3727 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3730 # add usb controllers
3731 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3732 $conf, $bridges, $arch, $machine_type, $machine_version);
3733 push @$devices, @usbcontrollers if @usbcontrollers;
3734 my $vga = parse_vga
($conf->{vga
});
3736 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3737 $vga->{type
} = 'qxl' if $qxlnum;
3739 if (!$vga->{type
}) {
3740 if ($arch eq 'aarch64') {
3741 $vga->{type
} = 'virtio';
3742 } elsif (min_version
($machine_version, 2, 9)) {
3743 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3745 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3749 # enable absolute mouse coordinates (needed by vnc)
3750 my $tablet = $conf->{tablet
};
3751 if (!defined($tablet)) {
3752 $tablet = $defaults->{tablet
};
3753 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3754 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3758 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3759 my $kbd = print_keyboarddevice_full
($conf, $arch);
3760 push @$devices, '-device', $kbd if defined($kbd);
3763 my $bootorder = device_bootorder
($conf);
3765 # host pci device passthrough
3766 my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3767 $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3770 my $usb_dev_features = {};
3771 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3773 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3774 $conf, $usb_dev_features, $bootorder, $machine_version);
3775 push @$devices, @usbdevices if @usbdevices;
3778 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3779 my $path = $conf->{"serial$i"} or next;
3780 if ($path eq 'socket') {
3781 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3782 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3783 # On aarch64, serial0 is the UART device. QEMU only allows
3784 # connecting UART devices via the '-serial' command line, as
3785 # the device has a fixed slot on the hardware...
3786 if ($arch eq 'aarch64' && $i == 0) {
3787 push @$devices, '-serial', "chardev:serial$i";
3789 push @$devices, '-device', "isa-serial,chardev=serial$i";
3792 die "no such serial device\n" if ! -c
$path;
3793 push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3794 push @$devices, '-device', "isa-serial,chardev=serial$i";
3799 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3800 if (my $path = $conf->{"parallel$i"}) {
3801 die "no such parallel device\n" if ! -c
$path;
3802 my $devtype = $path =~ m!^/dev/usb/lp! ?
'serial' : 'parallel';
3803 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3804 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3808 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3809 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3810 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3811 push @$devices, @$audio_devs;
3814 add_tpm_device
($vmid, $devices, $conf);
3817 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3818 $sockets = $conf->{sockets
} if $conf->{sockets
};
3820 my $cores = $conf->{cores
} || 1;
3822 my $maxcpus = $sockets * $cores;
3824 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3826 my $allowed_vcpus = $cpuinfo->{cpus
};
3828 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3830 if ($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3831 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3832 for (my $i = 2; $i <= $vcpus; $i++) {
3833 my $cpustr = print_cpu_device
($conf,$i);
3834 push @$cmd, '-device', $cpustr;
3839 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3841 push @$cmd, '-nodefaults';
3843 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3845 push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3847 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3849 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3850 push @$devices, '-device', print_vga_device
(
3851 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3853 push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type
} eq 'virtio-gl'; # VIRGL
3855 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3856 push @$cmd, '-vnc', "unix:$socket,password=on";
3858 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3859 push @$cmd, '-nographic';
3863 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3864 my $useLocaltime = $conf->{localtime};
3866 if ($winversion >= 5) { # windows
3867 $useLocaltime = 1 if !defined($conf->{localtime});
3869 # use time drift fix when acpi is enabled
3870 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3871 $tdf = 1 if !defined($conf->{tdf
});
3875 if ($winversion >= 6) {
3876 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3877 push @$machineFlags, 'hpet=off';
3880 push @$rtcFlags, 'driftfix=slew' if $tdf;
3882 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3883 push @$rtcFlags, "base=$conf->{startdate}";
3884 } elsif ($useLocaltime) {
3885 push @$rtcFlags, 'base=localtime';
3889 push @$cmd, '-cpu', $forcecpu;
3891 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3894 PVE
::QemuServer
::Memory
::config
(
3895 $conf, $vmid, $sockets, $cores, $defaults, $hotplug_features->{memory
}, $cmd);
3897 push @$cmd, '-S' if $conf->{freeze
};
3899 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3901 my $guest_agent = parse_guest_agent
($conf);
3903 if ($guest_agent->{enabled
}) {
3904 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3905 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3907 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3908 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3909 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3910 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3911 } elsif ($guest_agent->{type
} eq 'isa') {
3912 push @$devices, '-device', "isa-serial,chardev=qga0";
3916 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3917 if ($rng && $version_guard->(4, 1, 2)) {
3918 check_rng_source
($rng->{source
});
3920 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3921 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3922 my $limiter_str = "";
3924 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3927 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3928 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3929 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3934 if ($qxlnum || $vga->{type
} =~ /^virtio/) {
3937 for (my $i = 1; $i < $qxlnum; $i++){
3938 push @$devices, '-device', print_vga_device
(
3939 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3942 # assume other OS works like Linux
3943 my ($ram, $vram) = ("134217728", "67108864");
3944 if ($vga->{memory
}) {
3945 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3946 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3948 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3949 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3953 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3955 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3956 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3957 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3959 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3960 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3961 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3963 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3964 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3966 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3967 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3968 if ($spice_enhancement->{foldersharing
}) {
3969 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3970 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3973 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3974 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3975 if $spice_enhancement->{videostreaming
};
3977 push @$devices, '-spice', "$spice_opts";
3980 # enable balloon by default, unless explicitly disabled
3981 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3982 my $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3983 my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3984 $ballooncmd .= ",free-page-reporting=on" if min_version
($machine_version, 6, 2);
3985 push @$devices, '-device', $ballooncmd;
3988 if ($conf->{watchdog
}) {
3989 my $wdopts = parse_watchdog
($conf->{watchdog
});
3990 my $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3991 my $watchdog = $wdopts->{model
} || 'i6300esb';
3992 push @$devices, '-device', "$watchdog$pciaddr";
3993 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3997 my $scsicontroller = {};
3998 my $ahcicontroller = {};
3999 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
4001 # Add iscsi initiator name if available
4002 if (my $initiator = get_initiator_name
()) {
4003 push @$devices, '-iscsi', "initiator-name=$initiator";
4006 PVE
::QemuConfig-
>foreach_volume($conf, sub {
4007 my ($ds, $drive) = @_;
4009 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
4010 check_volume_storage_type
($storecfg, $drive->{file
});
4011 push @$vollist, $drive->{file
};
4014 # ignore efidisk here, already added in bios/fw handling code above
4015 return if $drive->{interface
} eq 'efidisk';
4017 return if $drive->{interface
} eq 'tpmstate';
4019 $use_virtio = 1 if $ds =~ m/^virtio/;
4021 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
4023 if ($drive->{interface
} eq 'virtio'){
4024 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
4027 if ($drive->{interface
} eq 'scsi') {
4029 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
4031 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4032 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4034 my $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
4035 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
4038 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
4039 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4040 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4041 } elsif ($drive->{iothread
}) {
4043 "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4048 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
4049 $queues = ",num_queues=$drive->{queues}";
4052 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4053 if !$scsicontroller->{$controller};
4054 $scsicontroller->{$controller}=1;
4057 if ($drive->{interface
} eq 'sata') {
4058 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
4059 my $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
4060 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4061 if !$ahcicontroller->{$controller};
4062 $ahcicontroller->{$controller}=1;
4065 my $pbs_conf = $pbs_backing->{$ds};
4066 my $pbs_name = undef;
4068 $pbs_name = "drive-$ds-pbs";
4069 push @$devices, '-blockdev', print_pbs_blockdev
($pbs_conf, $pbs_name);
4072 my $drive_cmd = print_drive_commandline_full
(
4073 $storecfg, $vmid, $drive, $pbs_name, min_version
($kvmver, 6, 0));
4075 # extra protection for templates, but SATA and IDE don't support it..
4076 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
4078 push @$devices, '-drive',$drive_cmd;
4079 push @$devices, '-device', print_drivedevice_full
(
4080 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4083 for (my $i = 0; $i < $MAX_NETS; $i++) {
4084 my $netname = "net$i";
4086 next if !$conf->{$netname};
4087 my $d = parse_net
($conf->{$netname});
4089 # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4091 $use_virtio = 1 if $d->{model
} eq 'virtio';
4093 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
4095 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
4096 push @$devices, '-netdev', $netdevfull;
4098 my $netdevicefull = print_netdevice_full
(
4099 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4101 push @$devices, '-device', $netdevicefull;
4104 if ($conf->{ivshmem
}) {
4105 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4109 $bus = print_pcie_addr
("ivshmem");
4111 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4114 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4115 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4117 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4118 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4119 .",size=$ivshmem->{size}M";
4122 # pci.4 is nested in pci.1
4123 $bridges->{1} = 1 if $bridges->{4};
4125 if (!$q35) { # add pci bridges
4126 if (min_version
($machine_version, 2, 3)) {
4130 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4133 for my $k (sort {$b cmp $a} keys %$bridges) {
4134 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4137 if ($k == 2 && $legacy_igd) {
4140 my $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
4141 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4143 if ($q35) { # add after -readconfig pve-q35.cfg
4144 splice @$devices, 2, 0, '-device', $devstr;
4146 unshift @$devices, '-device', $devstr if $k > 0;
4151 push @$machineFlags, 'accel=tcg';
4154 push @$machineFlags, 'smm=off' if should_disable_smm
($conf, $vga, $machine_type);
4156 my $machine_type_min = $machine_type;
4157 if ($add_pve_version) {
4158 $machine_type_min =~ s/\+pve\d+$//;
4159 $machine_type_min .= "+pve$required_pve_version";
4161 push @$machineFlags, "type=${machine_type_min}";
4163 push @$cmd, @$devices;
4164 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4165 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4166 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4168 if (my $vmstate = $conf->{vmstate
}) {
4169 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4170 push @$vollist, $vmstate;
4171 push @$cmd, '-loadstate', $statepath;
4172 print "activating and using '$vmstate' as vmstate\n";
4175 if (PVE
::QemuConfig-
>is_template($conf)) {
4176 # needed to workaround base volumes being read-only
4177 push @$cmd, '-snapshot';
4181 if ($conf->{args
}) {
4182 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4186 return wantarray ?
($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4189 sub check_rng_source
{
4192 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4193 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4196 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4197 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
4198 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4199 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4200 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4201 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4209 my $res = mon_cmd
($vmid, 'query-spice');
4211 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4214 sub vm_devices_list
{
4217 my $res = mon_cmd
($vmid, 'query-pci');
4218 my $devices_to_check = [];
4220 foreach my $pcibus (@$res) {
4221 push @$devices_to_check, @{$pcibus->{devices
}},
4224 while (@$devices_to_check) {
4226 for my $d (@$devices_to_check) {
4227 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4228 next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices
};
4230 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4231 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4233 $devices_to_check = $to_check;
4236 my $resblock = mon_cmd
($vmid, 'query-block');
4237 foreach my $block (@$resblock) {
4238 if($block->{device
} =~ m/^drive-(\S+)/){
4243 my $resmice = mon_cmd
($vmid, 'query-mice');
4244 foreach my $mice (@$resmice) {
4245 if ($mice->{name
} eq 'QEMU HID Tablet') {
4246 $devices->{tablet
} = 1;
4251 # for usb devices there is no query-usb
4252 # but we can iterate over the entries in
4253 # qom-list path=/machine/peripheral
4254 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4255 foreach my $per (@$resperipheral) {
4256 if ($per->{name
} =~ m/^usb(?:redirdev)?\d+$/) {
4257 $devices->{$per->{name
}} = 1;
4265 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4267 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4269 my $devices_list = vm_devices_list
($vmid);
4270 return 1 if defined($devices_list->{$deviceid});
4272 # add PCI bridge if we need it for the device
4273 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4275 if ($deviceid eq 'tablet') {
4276 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4277 } elsif ($deviceid eq 'keyboard') {
4278 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4279 } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4281 qemu_spice_usbredir_chardev_add
($vmid, "usbredirchardev$id");
4282 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_spice_usbdevice
($id, "xhci", $id + 1));
4283 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4284 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device, {}, $1 + 1));
4285 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4286 qemu_iothread_add
($vmid, $deviceid, $device);
4288 qemu_driveadd
($storecfg, $vmid, $device);
4289 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4291 qemu_deviceadd
($vmid, $devicefull);
4292 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4294 eval { qemu_drivedel
($vmid, $deviceid); };
4298 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4299 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4300 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4301 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4303 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4305 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4306 qemu_iothread_add
($vmid, $deviceid, $device);
4307 $devicefull .= ",iothread=iothread-$deviceid";
4310 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4311 $devicefull .= ",num_queues=$device->{queues}";
4314 qemu_deviceadd
($vmid, $devicefull);
4315 qemu_deviceaddverify
($vmid, $deviceid);
4316 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4317 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4318 qemu_driveadd
($storecfg, $vmid, $device);
4320 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4321 eval { qemu_deviceadd
($vmid, $devicefull); };
4323 eval { qemu_drivedel
($vmid, $deviceid); };
4327 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4328 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4330 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4331 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type);
4332 my $use_old_bios_files = undef;
4333 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4335 my $netdevicefull = print_netdevice_full
(
4336 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4337 qemu_deviceadd
($vmid, $netdevicefull);
4339 qemu_deviceaddverify
($vmid, $deviceid);
4340 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4343 eval { qemu_netdevdel
($vmid, $deviceid); };
4347 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4349 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4350 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4352 qemu_deviceadd
($vmid, $devicefull);
4353 qemu_deviceaddverify
($vmid, $deviceid);
4355 die "can't hotplug device '$deviceid'\n";
4361 # fixme: this should raise exceptions on error!
4362 sub vm_deviceunplug
{
4363 my ($vmid, $conf, $deviceid) = @_;
4365 my $devices_list = vm_devices_list
($vmid);
4366 return 1 if !defined($devices_list->{$deviceid});
4368 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4369 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4371 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4372 qemu_devicedel
($vmid, $deviceid);
4373 } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4374 qemu_devicedel
($vmid, $deviceid);
4375 qemu_devicedelverify
($vmid, $deviceid);
4376 } elsif ($deviceid =~ m/^usb\d+$/) {
4377 qemu_devicedel
($vmid, $deviceid);
4378 qemu_devicedelverify
($vmid, $deviceid);
4379 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4380 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4382 qemu_devicedel
($vmid, $deviceid);
4383 qemu_devicedelverify
($vmid, $deviceid);
4384 qemu_drivedel
($vmid, $deviceid);
4385 qemu_iothread_del
($vmid, $deviceid, $device);
4386 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4387 qemu_devicedel
($vmid, $deviceid);
4388 qemu_devicedelverify
($vmid, $deviceid);
4389 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4390 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4392 qemu_devicedel
($vmid, $deviceid);
4393 qemu_devicedelverify
($vmid, $deviceid);
4394 qemu_drivedel
($vmid, $deviceid);
4395 qemu_deletescsihw
($conf, $vmid, $deviceid);
4397 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4398 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4399 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4400 qemu_devicedel
($vmid, $deviceid);
4401 qemu_devicedelverify
($vmid, $deviceid);
4402 qemu_netdevdel
($vmid, $deviceid);
4404 die "can't unplug device '$deviceid'\n";
4410 sub qemu_spice_usbredir_chardev_add
{
4411 my ($vmid, $id) = @_;
4413 mon_cmd
($vmid, "chardev-add" , (
4424 sub qemu_deviceadd
{
4425 my ($vmid, $devicefull) = @_;
4427 $devicefull = "driver=".$devicefull;
4428 my %options = split(/[=,]/, $devicefull);
4430 mon_cmd
($vmid, "device_add" , %options);
4433 sub qemu_devicedel
{
4434 my ($vmid, $deviceid) = @_;
4436 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4439 sub qemu_iothread_add
{
4440 my ($vmid, $deviceid, $device) = @_;
4442 if ($device->{iothread
}) {
4443 my $iothreads = vm_iothreads_list
($vmid);
4444 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4448 sub qemu_iothread_del
{
4449 my ($vmid, $deviceid, $device) = @_;
4451 if ($device->{iothread
}) {
4452 my $iothreads = vm_iothreads_list
($vmid);
4453 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4457 sub qemu_objectadd
{
4458 my ($vmid, $objectid, $qomtype) = @_;
4460 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4465 sub qemu_objectdel
{
4466 my ($vmid, $objectid) = @_;
4468 mon_cmd
($vmid, "object-del", id
=> $objectid);
4474 my ($storecfg, $vmid, $device) = @_;
4476 my $kvmver = get_running_qemu_version
($vmid);
4477 my $io_uring = min_version
($kvmver, 6, 0);
4478 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4479 $drive =~ s/\\/\\\\/g;
4480 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4482 # If the command succeeds qemu prints: "OK
"
4483 return 1 if $ret =~ m/OK/s;
4485 die "adding drive failed
: $ret\n";
4489 my ($vmid, $deviceid) = @_;
4491 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4494 return 1 if $ret eq "";
4496 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4497 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4499 die "deleting drive
$deviceid failed
: $ret\n";
4502 sub qemu_deviceaddverify {
4503 my ($vmid, $deviceid) = @_;
4505 for (my $i = 0; $i <= 5; $i++) {
4506 my $devices_list = vm_devices_list($vmid);
4507 return 1 if defined($devices_list->{$deviceid});
4511 die "error on hotplug device
'$deviceid'\n";
4515 sub qemu_devicedelverify {
4516 my ($vmid, $deviceid) = @_;
4518 # need to verify that the device is correctly removed as device_del
4519 # is async and empty return is not reliable
4521 for (my $i = 0; $i <= 5; $i++) {
4522 my $devices_list = vm_devices_list($vmid);
4523 return 1 if !defined($devices_list->{$deviceid});
4527 die "error on hot-unplugging device
'$deviceid'\n";
4530 sub qemu_findorcreatescsihw {
4531 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4533 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4535 my $scsihwid="$controller_prefix$controller";
4536 my $devices_list = vm_devices_list($vmid);
4538 if (!defined($devices_list->{$scsihwid})) {
4539 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4545 sub qemu_deletescsihw {
4546 my ($conf, $vmid, $opt) = @_;
4548 my $device = parse_drive($opt, $conf->{$opt});
4550 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4551 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4555 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4557 my $devices_list = vm_devices_list($vmid);
4558 foreach my $opt (keys %{$devices_list}) {
4559 if (is_valid_drivename($opt)) {
4560 my $drive = parse_drive($opt, $conf->{$opt});
4561 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4567 my $scsihwid="scsihw
$controller";
4569 vm_deviceunplug($vmid, $conf, $scsihwid);
4574 sub qemu_add_pci_bridge {
4575 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4581 print_pci_addr($device, $bridges, $arch, $machine_type);
4583 while (my ($k, $v) = each %$bridges) {
4586 return 1 if !defined($bridgeid) || $bridgeid < 1;
4588 my $bridge = "pci
.$bridgeid";
4589 my $devices_list = vm_devices_list($vmid);
4591 if (!defined($devices_list->{$bridge})) {
4592 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4598 sub qemu_set_link_status {
4599 my ($vmid, $device, $up) = @_;
4601 mon_cmd($vmid, "set_link
", name => $device,
4602 up => $up ? JSON::true : JSON::false);
4605 sub qemu_netdevadd {
4606 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4608 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4609 my %options = split(/[=,]/, $netdev);
4611 if (defined(my $vhost = $options{vhost})) {
4612 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4615 if (defined(my $queues = $options{queues})) {
4616 $options{queues} = $queues + 0;
4619 mon_cmd($vmid, "netdev_add
", %options);
4623 sub qemu_netdevdel {
4624 my ($vmid, $deviceid) = @_;
4626 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4629 sub qemu_usb_hotplug {
4630 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4634 # remove the old one first
4635 vm_deviceunplug($vmid, $conf, $deviceid);
4637 # check if xhci controller is necessary and available
4638 my $devicelist = vm_devices_list($vmid);
4640 if (!$devicelist->{xhci}) {
4641 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4642 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4646 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4649 sub qemu_cpu_hotplug {
4650 my ($vmid, $conf, $vcpus) = @_;
4652 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4655 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4656 $sockets = $conf->{sockets} if $conf->{sockets};
4657 my $cores = $conf->{cores} || 1;
4658 my $maxcpus = $sockets * $cores;
4660 $vcpus = $maxcpus if !$vcpus;
4662 die "you can
't add more vcpus than maxcpus\n"
4663 if $vcpus > $maxcpus;
4665 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4667 if ($vcpus < $currentvcpus) {
4669 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4671 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4672 qemu_devicedel($vmid, "cpu$i");
4674 my $currentrunningvcpus = undef;
4676 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4677 last if scalar(@{$currentrunningvcpus}) == $i-1;
4678 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4682 #update conf after each succesfull cpu unplug
4683 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4684 PVE::QemuConfig->write_config($vmid, $conf);
4687 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4693 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4694 die "vcpus in running vm does not match its configuration\n"
4695 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4697 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4699 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4700 my $cpustr = print_cpu_device($conf, $i);
4701 qemu_deviceadd($vmid, $cpustr);
4704 my $currentrunningvcpus = undef;
4706 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4707 last if scalar(@{$currentrunningvcpus}) == $i;
4708 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4712 #update conf after each succesfull cpu hotplug
4713 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4714 PVE::QemuConfig->write_config($vmid, $conf);
4718 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4719 mon_cmd($vmid, "cpu-add", id => int($i));
4724 sub qemu_block_set_io_throttle {
4725 my ($vmid, $deviceid,
4726 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4727 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4728 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4729 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4731 return if !check_running($vmid) ;
4733 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4735 bps_rd => int($bps_rd),
4736 bps_wr => int($bps_wr),
4738 iops_rd => int($iops_rd),
4739 iops_wr => int($iops_wr),
4740 bps_max => int($bps_max),
4741 bps_rd_max => int($bps_rd_max),
4742 bps_wr_max => int($bps_wr_max),
4743 iops_max => int($iops_max),
4744 iops_rd_max => int($iops_rd_max),
4745 iops_wr_max => int($iops_wr_max),
4746 bps_max_length => int($bps_max_length),
4747 bps_rd_max_length => int($bps_rd_max_length),
4748 bps_wr_max_length => int($bps_wr_max_length),
4749 iops_max_length => int($iops_max_length),
4750 iops_rd_max_length => int($iops_rd_max_length),
4751 iops_wr_max_length => int($iops_wr_max_length),
4756 sub qemu_block_resize {
4757 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4759 my $running = check_running($vmid);
4761 PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4763 return if !$running;
4765 my $padding = (1024 - $size % 1024) % 1024;
4766 $size = $size + $padding;
4771 device => $deviceid,
4777 sub qemu_volume_snapshot {
4778 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4780 my $running = check_running($vmid);
4782 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4783 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4785 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4789 sub qemu_volume_snapshot_delete {
4790 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4792 my $running = check_running($vmid);
4797 my $conf = PVE::QemuConfig->load_config($vmid);
4798 PVE::QemuConfig->foreach_volume($conf, sub {
4799 my ($ds, $drive) = @_;
4800 $running = 1 if $drive->{file} eq $volid;
4804 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4805 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4807 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4811 sub set_migration_caps {
4812 my ($vmid, $savevm) = @_;
4814 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4816 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4817 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4822 "auto-converge" => 1,
4824 "x-rdma-pin-all" => 0,
4827 "dirty-bitmaps" => $dirty_bitmaps,
4830 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4832 for my $supported_capability (@$supported_capabilities) {
4834 capability => $supported_capability->{capability},
4835 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4839 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4843 my ($conf, $func, @param) = @_;
4847 my $test_volid = sub {
4848 my ($key, $drive, $snapname, $pending) = @_;
4850 my $volid = $drive->{file};
4853 $volhash->{$volid}->{cdrom} //= 1;
4854 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4856 my $replicate = $drive->{replicate} // 1;
4857 $volhash->{$volid}->{replicate} //= 0;
4858 $volhash->{$volid}->{replicate} = 1 if $replicate;
4860 $volhash->{$volid}->{shared} //= 0;
4861 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4863 $volhash->{$volid}->{is_unused} //= 0;
4864 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4866 $volhash->{$volid}->{is_attached} //= 0;
4867 $volhash->{$volid}->{is_attached} = 1
4868 if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4870 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4871 if defined($snapname);
4873 $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4875 my $size = $drive->{size};
4876 $volhash->{$volid}->{size} //= $size if $size;
4878 $volhash->{$volid}->{is_vmstate} //= 0;
4879 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4881 $volhash->{$volid}->{is_tpmstate} //= 0;
4882 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0
';
4884 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4887 my $include_opts = {
4888 extra_keys => ['vmstate
'],
4889 include_unused => 1,
4892 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4894 PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4895 if defined($conf->{pending}) && $conf->{pending}->%*;
4897 foreach my $snapname (keys %{$conf->{snapshots}}) {
4898 my $snap = $conf->{snapshots}->{$snapname};
4899 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4902 foreach my $volid (keys %$volhash) {
4903 &$func($volid, $volhash->{$volid}, @param);
4907 my $fast_plug_option = {
4911 'migrate_downtime
' => 1,
4912 'migrate_speed
' => 1,
4919 'vmstatestorage
' => 1,
4922 for my $opt (keys %$confdesc_cloudinit) {
4923 $fast_plug_option->{$opt} = 1;
4926 # hotplug changes in [PENDING]
4927 # $selection hash can be used to only apply specified options, for
4928 # example: { cores => 1 } (only apply changed 'cores
')
4929 # $errors ref is used to return error messages
4930 sub vmconfig_hotplug_pending {
4931 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4933 my $defaults = load_defaults();
4934 my $arch = get_vm_arch($conf);
4935 my $machine_type = get_vm_machine($conf, undef, $arch);
4937 # commit values which do not have any impact on running VM first
4938 # Note: those option cannot raise errors, we we do not care about
4939 # $selection and always apply them.
4941 my $add_error = sub {
4942 my ($opt, $msg) = @_;
4943 $errors->{$opt} = "hotplug problem - $msg";
4946 my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4948 my $cloudinit_record_changed = sub {
4949 my ($conf, $opt, $old, $new) = @_;
4950 return if !$cloudinit_pending_properties->{$opt};
4952 my $ci = ($conf->{cloudinit} //= {});
4954 my $recorded = $ci->{$opt};
4955 my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4957 if (defined($new)) {
4958 if (defined($old)) {
4959 # an existing value is being modified
4960 if (defined($recorded)) {
4961 # the value was already not in sync
4962 if ($new eq $recorded) {
4963 # a value is being reverted to the cloud-init state:
4965 delete $added{$opt};
4967 # the value was changed multiple times, do nothing
4969 } elsif ($added{$opt}) {
4970 # the value had been marked as added and is being changed, do nothing
4972 # the value is new, record it:
4976 # a new value is being added
4977 if (defined($recorded)) {
4978 # it was already not in sync
4979 if ($new eq $recorded) {
4980 # a value is being reverted to the cloud-init state:
4982 delete $added{$opt};
4984 # the value had temporarily been removed, do nothing
4986 } elsif ($added{$opt}) {
4987 # the value had been marked as added already, do nothing
4989 # the value is new, add it
4993 } elsif (!defined($old)) {
4994 # a non-existent value is being removed? ignore...
4996 # a value is being deleted
4997 if (defined($recorded)) {
4998 # a value was already recorded, just keep it
4999 } elsif ($added{$opt}) {
5000 # the value was marked as added, remove it
5001 delete $added{$opt};
5003 # a previously unrecorded value is being removed, record the old value:
5008 my $added = join(',', sort keys %added);
5009 $ci->{added} = $added if length($added);
5013 foreach my $opt (keys %{$conf->{pending}}) { # add/change
5014 if ($fast_plug_option->{$opt}) {
5015 my $new = delete $conf->{pending}->{$opt};
5016 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
5017 $conf->{$opt} = $new;
5023 PVE::QemuConfig->write_config($vmid, $conf);
5026 my $ostype = $conf->{ostype};
5027 my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5028 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5029 my $usb_hotplug = $hotplug_features->{usb}
5030 && min_version($version, 7, 1)
5031 && defined($ostype) && ($ostype eq 'l26
' || windows_version($ostype) > 7);
5033 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5034 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5036 foreach my $opt (sort keys %$pending_delete_hash) {
5037 next if $selection && !$selection->{$opt};
5038 my $force = $pending_delete_hash->{$opt}->{force};
5040 if ($opt eq 'hotplug
') {
5041 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5042 } elsif ($opt eq 'tablet
') {
5043 die "skip\n" if !$hotplug_features->{usb};
5044 if ($defaults->{tablet}) {
5045 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
5046 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
5047 if $arch eq 'aarch64
';
5049 vm_deviceunplug($vmid, $conf, 'tablet
');
5050 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
5052 } elsif ($opt =~ m/^usb(\d+)$/) {
5054 die "skip\n" if !$usb_hotplug;
5055 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5056 vm_deviceunplug
($vmid, $conf, $opt);
5057 } elsif ($opt eq 'vcpus') {
5058 die "skip\n" if !$hotplug_features->{cpu
};
5059 qemu_cpu_hotplug
($vmid, $conf, undef);
5060 } elsif ($opt eq 'balloon') {
5061 # enable balloon device is not hotpluggable
5062 die "skip\n" if defined($conf->{balloon
}) && $conf->{balloon
} == 0;
5063 # here we reset the ballooning value to memory
5064 my $balloon = $conf->{memory
} || $defaults->{memory
};
5065 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5066 } elsif ($fast_plug_option->{$opt}) {
5068 } elsif ($opt =~ m/^net(\d+)$/) {
5069 die "skip\n" if !$hotplug_features->{network
};
5070 vm_deviceunplug
($vmid, $conf, $opt);
5071 } elsif (is_valid_drivename
($opt)) {
5072 die "skip\n" if !$hotplug_features->{disk
} || $opt =~ m/(ide|sata)(\d+)/;
5073 vm_deviceunplug
($vmid, $conf, $opt);
5074 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5075 } elsif ($opt =~ m/^memory$/) {
5076 die "skip\n" if !$hotplug_features->{memory
};
5077 PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $defaults);
5078 } elsif ($opt eq 'cpuunits') {
5079 $cgroup->change_cpu_shares(undef);
5080 } elsif ($opt eq 'cpulimit') {
5081 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5087 &$add_error($opt, $err) if $err ne "skip\n";
5089 my $old = delete $conf->{$opt};
5090 $cloudinit_record_changed->($conf, $opt, $old, undef);
5091 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5096 foreach my $opt (keys %{$conf->{pending
}}) {
5097 next if $selection && !$selection->{$opt};
5098 my $value = $conf->{pending
}->{$opt};
5100 if ($opt eq 'hotplug') {
5101 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug
} =~ /memory/);
5102 } elsif ($opt eq 'tablet') {
5103 die "skip\n" if !$hotplug_features->{usb
};
5105 vm_deviceplug
($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5106 vm_deviceplug
($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5107 if $arch eq 'aarch64';
5108 } elsif ($value == 0) {
5109 vm_deviceunplug
($vmid, $conf, 'tablet');
5110 vm_deviceunplug
($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5112 } elsif ($opt =~ m/^usb(\d+)$/) {
5114 die "skip\n" if !$usb_hotplug;
5115 my $d = eval { parse_property_string
('pve-qm-usb', $value) };
5117 if ($d->{host
} =~ m/^spice$/i) {
5118 $id = "usbredirdev$index";
5120 qemu_usb_hotplug
($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5121 } elsif ($opt eq 'vcpus') {
5122 die "skip\n" if !$hotplug_features->{cpu
};
5123 qemu_cpu_hotplug
($vmid, $conf, $value);
5124 } elsif ($opt eq 'balloon') {
5125 # enable/disable balloning device is not hotpluggable
5126 my $old_balloon_enabled = !!(!defined($conf->{balloon
}) || $conf->{balloon
});
5127 my $new_balloon_enabled = !!(!defined($conf->{pending
}->{balloon
}) || $conf->{pending
}->{balloon
});
5128 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5130 # allow manual ballooning if shares is set to zero
5131 if ((defined($conf->{shares
}) && ($conf->{shares
} == 0))) {
5132 my $balloon = $conf->{pending
}->{balloon
} || $conf->{memory
} || $defaults->{memory
};
5133 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5135 } elsif ($opt =~ m/^net(\d+)$/) {
5136 # some changes can be done without hotplug
5137 vmconfig_update_net
($storecfg, $conf, $hotplug_features->{network
},
5138 $vmid, $opt, $value, $arch, $machine_type);
5139 } elsif (is_valid_drivename
($opt)) {
5140 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5141 # some changes can be done without hotplug
5142 my $drive = parse_drive
($opt, $value);
5143 if (drive_is_cloudinit
($drive)) {
5144 $cloudinit_opt = [$opt, $drive];
5145 # apply all the other changes first, then generate the cloudinit disk
5148 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5149 $vmid, $opt, $value, $arch, $machine_type);
5150 } elsif ($opt =~ m/^memory$/) { #dimms
5151 die "skip\n" if !$hotplug_features->{memory
};
5152 $value = PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $defaults, $value);
5153 } elsif ($opt eq 'cpuunits') {
5154 my $new_cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{pending
}->{$opt}); #clamp
5155 $cgroup->change_cpu_shares($new_cpuunits);
5156 } elsif ($opt eq 'cpulimit') {
5157 my $cpulimit = $conf->{pending
}->{$opt} == 0 ?
-1 : int($conf->{pending
}->{$opt} * 100000);
5158 $cgroup->change_cpu_quota($cpulimit, 100000);
5159 } elsif ($opt eq 'agent') {
5160 vmconfig_update_agent
($conf, $opt, $value);
5162 die "skip\n"; # skip non-hot-pluggable options
5166 &$add_error($opt, $err) if $err ne "skip\n";
5168 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5169 $conf->{$opt} = $value;
5170 delete $conf->{pending
}->{$opt};
5174 if (defined($cloudinit_opt)) {
5175 my ($opt, $drive) = @$cloudinit_opt;
5176 my $value = $conf->{pending
}->{$opt};
5178 my $temp = {%$conf, $opt => $value};
5179 PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($temp, $vmid);
5180 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5181 $vmid, $opt, $value, $arch, $machine_type);
5184 &$add_error($opt, $err) if $err ne "skip\n";
5186 $conf->{$opt} = $value;
5187 delete $conf->{pending
}->{$opt};
5191 # unplug xhci controller if no usb device is left
5194 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
5195 next if !defined($conf->{"usb$i"});
5200 vm_deviceunplug
($vmid, $conf, 'xhci');
5204 PVE
::QemuConfig-
>write_config($vmid, $conf);
5206 if ($hotplug_features->{cloudinit
} && PVE
::QemuServer
::Cloudinit
::has_changes
($conf)) {
5207 PVE
::QemuServer
::vmconfig_update_cloudinit_drive
($storecfg, $conf, $vmid);
5211 sub try_deallocate_drive
{
5212 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5214 if (($force || $key =~ /^unused/) && !drive_is_cdrom
($drive, 1)) {
5215 my $volid = $drive->{file
};
5216 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
5217 my $sid = PVE
::Storage
::parse_volume_id
($volid);
5218 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5220 # check if the disk is really unused
5221 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5222 if PVE
::QemuServer
::Drive
::is_volume_in_use
($storecfg, $conf, $key, $volid);
5223 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5226 # If vm is not owner of this disk remove from config
5234 sub vmconfig_delete_or_detach_drive
{
5235 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5237 my $drive = parse_drive
($opt, $conf->{$opt});
5239 my $rpcenv = PVE
::RPCEnvironment
::get
();
5240 my $authuser = $rpcenv->get_user();
5243 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5244 try_deallocate_drive
($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5246 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $drive);
5252 sub vmconfig_apply_pending
{
5253 my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5255 return if !scalar(keys %{$conf->{pending
}});
5257 my $add_apply_error = sub {
5258 my ($opt, $msg) = @_;
5259 my $err_msg = "unable to apply pending change $opt : $msg";
5260 $errors->{$opt} = $err_msg;
5266 my $pending_delete_hash = PVE
::QemuConfig-
>parse_pending_delete($conf->{pending
}->{delete});
5267 foreach my $opt (sort keys %$pending_delete_hash) {
5268 my $force = $pending_delete_hash->{$opt}->{force
};
5270 if ($opt =~ m/^unused/) {
5271 die "internal error";
5272 } elsif (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5273 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5277 $add_apply_error->($opt, $err);
5279 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5280 delete $conf->{$opt};
5284 PVE
::QemuConfig-
>cleanup_pending($conf);
5286 my $generate_cloudinit = $skip_cloud_init ?
0 : undef;
5288 foreach my $opt (keys %{$conf->{pending
}}) { # add/change
5289 next if $opt eq 'delete'; # just to be sure
5291 if (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5292 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, parse_drive
($opt, $conf->{$opt}))
5296 $add_apply_error->($opt, $err);
5299 if (is_valid_drivename
($opt)) {
5300 my $drive = parse_drive
($opt, $conf->{pending
}->{$opt});
5301 $generate_cloudinit //= 1 if drive_is_cloudinit
($drive);
5304 $conf->{$opt} = delete $conf->{pending
}->{$opt};
5308 # write all changes at once to avoid unnecessary i/o
5309 PVE
::QemuConfig-
>write_config($vmid, $conf);
5310 if ($generate_cloudinit) {
5311 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5312 # After successful generation and if there were changes to be applied, update the
5313 # config to drop the {cloudinit} entry.
5314 PVE
::QemuConfig-
>write_config($vmid, $conf);
5319 sub vmconfig_update_net
{
5320 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5322 my $newnet = parse_net
($value);
5324 if ($conf->{$opt}) {
5325 my $oldnet = parse_net
($conf->{$opt});
5327 if (safe_string_ne
($oldnet->{model
}, $newnet->{model
}) ||
5328 safe_string_ne
($oldnet->{macaddr
}, $newnet->{macaddr
}) ||
5329 safe_num_ne
($oldnet->{queues
}, $newnet->{queues
}) ||
5330 safe_num_ne
($oldnet->{mtu
}, $newnet->{mtu
}) ||
5331 !($newnet->{bridge
} && $oldnet->{bridge
})) { # bridge/nat mode change
5333 # for non online change, we try to hot-unplug
5334 die "skip\n" if !$hotplug;
5335 vm_deviceunplug
($vmid, $conf, $opt);
5338 die "internal error" if $opt !~ m/net(\d+)/;
5339 my $iface = "tap${vmid}i$1";
5341 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5342 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
}) ||
5343 safe_string_ne
($oldnet->{trunks
}, $newnet->{trunks
}) ||
5344 safe_num_ne
($oldnet->{firewall
}, $newnet->{firewall
})) {
5345 PVE
::Network
::tap_unplug
($iface);
5348 PVE
::Network
::SDN
::Zones
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5350 PVE
::Network
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5352 } elsif (safe_num_ne
($oldnet->{rate
}, $newnet->{rate
})) {
5353 # Rate can be applied on its own but any change above needs to
5354 # include the rate in tap_plug since OVS resets everything.
5355 PVE
::Network
::tap_rate_limit
($iface, $newnet->{rate
});
5358 if (safe_string_ne
($oldnet->{link_down
}, $newnet->{link_down
})) {
5359 qemu_set_link_status
($vmid, $opt, !$newnet->{link_down
});
5367 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5373 sub vmconfig_update_agent
{
5374 my ($conf, $opt, $value) = @_;
5376 die "skip\n" if !$conf->{$opt};
5378 my $hotplug_options = { fstrim_cloned_disks
=> 1 };
5380 my $old_agent = parse_guest_agent
($conf);
5381 my $agent = parse_guest_agent
({$opt => $value});
5383 for my $option (keys %$agent) { # added/changed options
5384 next if defined($hotplug_options->{$option});
5385 die "skip\n" if safe_string_ne
($agent->{$option}, $old_agent->{$option});
5388 for my $option (keys %$old_agent) { # removed options
5389 next if defined($hotplug_options->{$option});
5390 die "skip\n" if safe_string_ne
($old_agent->{$option}, $agent->{$option});
5393 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5396 sub vmconfig_update_disk
{
5397 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5399 my $drive = parse_drive
($opt, $value);
5401 if ($conf->{$opt} && (my $old_drive = parse_drive
($opt, $conf->{$opt}))) {
5402 my $media = $drive->{media
} || 'disk';
5403 my $oldmedia = $old_drive->{media
} || 'disk';
5404 die "unable to change media type\n" if $media ne $oldmedia;
5406 if (!drive_is_cdrom
($old_drive)) {
5408 if ($drive->{file
} ne $old_drive->{file
}) {
5410 die "skip\n" if !$hotplug;
5412 # unplug and register as unused
5413 vm_deviceunplug
($vmid, $conf, $opt);
5414 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive)
5417 # update existing disk
5419 # skip non hotpluggable value
5420 if (safe_string_ne
($drive->{aio
}, $old_drive->{aio
}) ||
5421 safe_string_ne
($drive->{discard
}, $old_drive->{discard
}) ||
5422 safe_string_ne
($drive->{iothread
}, $old_drive->{iothread
}) ||
5423 safe_string_ne
($drive->{queues
}, $old_drive->{queues
}) ||
5424 safe_string_ne
($drive->{cache
}, $old_drive->{cache
}) ||
5425 safe_string_ne
($drive->{ssd
}, $old_drive->{ssd
}) ||
5426 safe_string_ne
($drive->{ro
}, $old_drive->{ro
})) {
5431 if (safe_num_ne
($drive->{mbps
}, $old_drive->{mbps
}) ||
5432 safe_num_ne
($drive->{mbps_rd
}, $old_drive->{mbps_rd
}) ||
5433 safe_num_ne
($drive->{mbps_wr
}, $old_drive->{mbps_wr
}) ||
5434 safe_num_ne
($drive->{iops
}, $old_drive->{iops
}) ||
5435 safe_num_ne
($drive->{iops_rd
}, $old_drive->{iops_rd
}) ||
5436 safe_num_ne
($drive->{iops_wr
}, $old_drive->{iops_wr
}) ||
5437 safe_num_ne
($drive->{mbps_max
}, $old_drive->{mbps_max
}) ||
5438 safe_num_ne
($drive->{mbps_rd_max
}, $old_drive->{mbps_rd_max
}) ||
5439 safe_num_ne
($drive->{mbps_wr_max
}, $old_drive->{mbps_wr_max
}) ||
5440 safe_num_ne
($drive->{iops_max
}, $old_drive->{iops_max
}) ||
5441 safe_num_ne
($drive->{iops_rd_max
}, $old_drive->{iops_rd_max
}) ||
5442 safe_num_ne
($drive->{iops_wr_max
}, $old_drive->{iops_wr_max
}) ||
5443 safe_num_ne
($drive->{bps_max_length
}, $old_drive->{bps_max_length
}) ||
5444 safe_num_ne
($drive->{bps_rd_max_length
}, $old_drive->{bps_rd_max_length
}) ||
5445 safe_num_ne
($drive->{bps_wr_max_length
}, $old_drive->{bps_wr_max_length
}) ||
5446 safe_num_ne
($drive->{iops_max_length
}, $old_drive->{iops_max_length
}) ||
5447 safe_num_ne
($drive->{iops_rd_max_length
}, $old_drive->{iops_rd_max_length
}) ||
5448 safe_num_ne
($drive->{iops_wr_max_length
}, $old_drive->{iops_wr_max_length
})) {
5450 qemu_block_set_io_throttle
(
5452 ($drive->{mbps
} || 0)*1024*1024,
5453 ($drive->{mbps_rd
} || 0)*1024*1024,
5454 ($drive->{mbps_wr
} || 0)*1024*1024,
5455 $drive->{iops
} || 0,
5456 $drive->{iops_rd
} || 0,
5457 $drive->{iops_wr
} || 0,
5458 ($drive->{mbps_max
} || 0)*1024*1024,
5459 ($drive->{mbps_rd_max
} || 0)*1024*1024,
5460 ($drive->{mbps_wr_max
} || 0)*1024*1024,
5461 $drive->{iops_max
} || 0,
5462 $drive->{iops_rd_max
} || 0,
5463 $drive->{iops_wr_max
} || 0,
5464 $drive->{bps_max_length
} || 1,
5465 $drive->{bps_rd_max_length
} || 1,
5466 $drive->{bps_wr_max_length
} || 1,
5467 $drive->{iops_max_length
} || 1,
5468 $drive->{iops_rd_max_length
} || 1,
5469 $drive->{iops_wr_max_length
} || 1,
5479 if ($drive->{file
} eq 'none') {
5480 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5481 if (drive_is_cloudinit
($old_drive)) {
5482 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive);
5485 my $path = get_iso_path
($storecfg, $vmid, $drive->{file
});
5487 # force eject if locked
5488 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5491 mon_cmd
($vmid, "blockdev-change-medium",
5492 id
=> "$opt", filename
=> "$path");
5500 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5502 PVE
::Storage
::activate_volumes
($storecfg, [$drive->{file
}]) if $drive->{file
} !~ m
|^/dev/.+|;
5503 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5506 sub vmconfig_update_cloudinit_drive
{
5507 my ($storecfg, $conf, $vmid) = @_;
5509 my $cloudinit_ds = undef;
5510 my $cloudinit_drive = undef;
5512 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5513 my ($ds, $drive) = @_;
5514 if (PVE
::QemuServer
::drive_is_cloudinit
($drive)) {
5515 $cloudinit_ds = $ds;
5516 $cloudinit_drive = $drive;
5520 return if !$cloudinit_drive;
5522 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5523 PVE
::QemuConfig-
>write_config($vmid, $conf);
5526 my $running = PVE
::QemuServer
::check_running
($vmid);
5529 my $path = PVE
::Storage
::path
($storecfg, $cloudinit_drive->{file
});
5531 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$cloudinit_ds");
5532 mon_cmd
($vmid, "blockdev-change-medium", id
=> "$cloudinit_ds", filename
=> "$path");
5537 # called in locked context by incoming migration
5538 sub vm_migrate_get_nbd_disks
{
5539 my ($storecfg, $conf, $replicated_volumes) = @_;
5541 my $local_volumes = {};
5542 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5543 my ($ds, $drive) = @_;
5545 return if drive_is_cdrom
($drive);
5546 return if $ds eq 'tpmstate0';
5548 my $volid = $drive->{file
};
5552 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid);
5554 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5555 return if $scfg->{shared
};
5557 # replicated disks re-use existing state via bitmap
5558 my $use_existing = $replicated_volumes->{$volid} ?
1 : 0;
5559 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5561 return $local_volumes;
5564 # called in locked context by incoming migration
5565 sub vm_migrate_alloc_nbd_disks
{
5566 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5569 foreach my $opt (sort keys %$source_volumes) {
5570 my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5572 if ($use_existing) {
5573 $nbd->{$opt}->{drivestr
} = print_drive
($drive);
5574 $nbd->{$opt}->{volid
} = $volid;
5575 $nbd->{$opt}->{replicated
} = 1;
5579 # storage mapping + volname = regular migration
5580 # storage mapping + format = remote migration
5581 # order of precedence, filtered by whether storage supports it:
5582 # 1. explicit requested format
5583 # 2. format of current volume
5584 # 3. default format of storage
5585 if (!$storagemap->{identity
}) {
5586 $storeid = PVE
::JSONSchema
::map_id
($storagemap, $storeid);
5587 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5588 if (!$format || !grep { $format eq $_ } @$validFormats) {
5590 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5591 my $fileFormat = qemu_img_format
($scfg, $volname);
5592 $format = $fileFormat
5593 if grep { $fileFormat eq $_ } @$validFormats;
5595 $format //= $defFormat;
5598 # can't happen for remote migration, so $volname is always defined
5599 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5600 $format = qemu_img_format
($scfg, $volname);
5603 my $size = $drive->{size
} / 1024;
5604 my $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $format, undef, $size);
5605 my $newdrive = $drive;
5606 $newdrive->{format
} = $format;
5607 $newdrive->{file
} = $newvolid;
5608 my $drivestr = print_drive
($newdrive);
5609 $nbd->{$opt}->{drivestr
} = $drivestr;
5610 $nbd->{$opt}->{volid
} = $newvolid;
5616 # see vm_start_nolock for parameters, additionally:
5618 # storagemap = parsed storage map for allocating NBD disks
5620 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5622 return PVE
::QemuConfig-
>lock_config($vmid, sub {
5623 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migrate_opts->{migratedfrom
});
5625 die "you can't start a vm if it's a template\n"
5626 if !$params->{skiptemplate
} && PVE
::QemuConfig-
>is_template($conf);
5628 my $has_suspended_lock = PVE
::QemuConfig-
>has_lock($conf, 'suspended');
5629 my $has_backup_lock = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5631 my $running = check_running
($vmid, undef, $migrate_opts->{migratedfrom
});
5633 if ($has_backup_lock && $running) {
5634 # a backup is currently running, attempt to start the guest in the
5635 # existing QEMU instance
5636 return vm_resume
($vmid);
5639 PVE
::QemuConfig-
>check_lock($conf)
5640 if !($params->{skiplock
} || $has_suspended_lock);
5642 $params->{resume
} = $has_suspended_lock || defined($conf->{vmstate
});
5644 die "VM $vmid already running\n" if $running;
5646 if (my $storagemap = $migrate_opts->{storagemap
}) {
5647 my $replicated = $migrate_opts->{replicated_volumes
};
5648 my $disks = vm_migrate_get_nbd_disks
($storecfg, $conf, $replicated);
5649 $migrate_opts->{nbd
} = vm_migrate_alloc_nbd_disks
($storecfg, $vmid, $disks, $storagemap);
5651 foreach my $opt (keys %{$migrate_opts->{nbd
}}) {
5652 $conf->{$opt} = $migrate_opts->{nbd
}->{$opt}->{drivestr
};
5656 return vm_start_nolock
($storecfg, $vmid, $conf, $params, $migrate_opts);
5662 # statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5663 # skiplock => 0/1, skip checking for config lock
5664 # skiptemplate => 0/1, skip checking whether VM is template
5665 # forcemachine => to force QEMU machine (rollback/migration)
5666 # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5667 # timeout => in seconds
5668 # paused => start VM in paused state (backup)
5669 # resume => resume from hibernation
5680 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5681 # migratedfrom => source node
5682 # spice_ticket => used for spice migration, passed via tunnel/stdin
5683 # network => CIDR of migration network
5684 # type => secure/insecure - tunnel over encrypted connection or plain-text
5685 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5686 # replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5687 # offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5688 # contained in config
5689 sub vm_start_nolock
{
5690 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5692 my $statefile = $params->{statefile
};
5693 my $resume = $params->{resume
};
5695 my $migratedfrom = $migrate_opts->{migratedfrom
};
5696 my $migration_type = $migrate_opts->{type
};
5700 # clean up leftover reboot request files
5701 eval { clear_reboot_request
($vmid); };
5704 if (!$statefile && scalar(keys %{$conf->{pending
}})) {
5705 vmconfig_apply_pending
($vmid, $conf, $storecfg);
5706 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5709 # don't regenerate the ISO if the VM is started as part of a live migration
5710 # this way we can reuse the old ISO with the correct config
5711 if (!$migratedfrom) {
5712 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5713 # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5714 # $conf->{cloudinit}, so we could just not do this?
5715 # But we do it above, so for now let's be consistent.
5716 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5720 # override offline migrated volumes, conf is out of date still
5721 if (my $offline_volumes = $migrate_opts->{offline_volumes
}) {
5722 for my $key (sort keys $offline_volumes->%*) {
5723 my $parsed = parse_drive
($key, $conf->{$key});
5724 $parsed->{file
} = $offline_volumes->{$key};
5725 $conf->{$key} = print_drive
($parsed);
5729 my $defaults = load_defaults
();
5731 # set environment variable useful inside network script
5732 # for remote migration the config is available on the target node!
5733 if (!$migrate_opts->{remote_node
}) {
5734 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom;
5737 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5739 my $forcemachine = $params->{forcemachine
};
5740 my $forcecpu = $params->{forcecpu
};
5742 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5743 $forcemachine = $conf->{runningmachine
};
5744 $forcecpu = $conf->{runningcpu
};
5745 print "Resuming suspended VM\n";
5748 my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command
($storecfg, $vmid,
5749 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5752 my $get_migration_ip = sub {
5753 my ($nodename) = @_;
5755 return $migration_ip if defined($migration_ip);
5757 my $cidr = $migrate_opts->{network
};
5759 if (!defined($cidr)) {
5760 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5761 $cidr = $dc_conf->{migration
}->{network
};
5764 if (defined($cidr)) {
5765 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5767 die "could not get IP: no address configured on local " .
5768 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5770 die "could not get IP: multiple addresses configured on local " .
5771 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5773 $migration_ip = @$ips[0];
5776 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5777 if !defined($migration_ip);
5779 return $migration_ip;
5783 if ($statefile eq 'tcp') {
5784 my $migrate = $res->{migrate
} = { proto
=> 'tcp' };
5785 $migrate->{addr
} = "localhost";
5786 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5787 my $nodename = nodename
();
5789 if (!defined($migration_type)) {
5790 if (defined($datacenterconf->{migration
}->{type
})) {
5791 $migration_type = $datacenterconf->{migration
}->{type
};
5793 $migration_type = 'secure';
5797 if ($migration_type eq 'insecure') {
5798 $migrate->{addr
} = $get_migration_ip->($nodename);
5799 $migrate->{addr
} = "[$migrate->{addr}]" if Net
::IP
::ip_is_ipv6
($migrate->{addr
});
5802 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5803 $migrate->{port
} = PVE
::Tools
::next_migrate_port
($pfamily);
5804 $migrate->{uri
} = "tcp:$migrate->{addr}:$migrate->{port}";
5805 push @$cmd, '-incoming', $migrate->{uri
};
5808 } elsif ($statefile eq 'unix') {
5809 # should be default for secure migrations as a ssh TCP forward
5810 # tunnel is not deterministic reliable ready and fails regurarly
5811 # to set up in time, so use UNIX socket forwards
5812 my $migrate = $res->{migrate
} = { proto
=> 'unix' };
5813 $migrate->{addr
} = "/run/qemu-server/$vmid.migrate";
5814 unlink $migrate->{addr
};
5816 $migrate->{uri
} = "unix:$migrate->{addr}";
5817 push @$cmd, '-incoming', $migrate->{uri
};
5820 } elsif (-e
$statefile) {
5821 push @$cmd, '-loadstate', $statefile;
5823 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5824 push @$vollist, $statefile;
5825 push @$cmd, '-loadstate', $statepath;
5827 } elsif ($params->{paused
}) {
5831 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $resume);
5833 my $pci_reserve_list = [];
5834 for my $device (values $pci_devices->%*) {
5835 next if $device->{mdev
}; # we don't reserve for mdev devices
5836 push $pci_reserve_list->@*, map { $_->{id
} } $device->{ids
}->@*;
5839 # reserve all PCI IDs before actually doing anything with them
5840 PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, $start_timeout);
5844 for my $id (sort keys %$pci_devices) {
5845 my $d = $pci_devices->{$id};
5846 my ($index) = ($id =~ m/^hostpci(\d+)$/);
5849 for my $dev ($d->{ids
}->@*) {
5850 my $info = eval { PVE
::QemuServer
::PCI
::prepare_pci_device
($vmid, $dev->{id
}, $index, $d->{mdev
}) };
5853 $chosen_mdev = $info;
5854 last if $chosen_mdev; # if successful, we're done
5860 next if !$d->{mdev
};
5861 die "could not create mediated device\n" if !defined($chosen_mdev);
5863 # nvidia grid needs the uuid of the mdev as qemu parameter
5864 if (!defined($uuid) && $chosen_mdev->{vendor
} =~ m/^(0x)?10de$/) {
5865 if (defined($conf->{smbios1
})) {
5866 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
5867 $uuid = $smbios_conf->{uuid
} if defined($smbios_conf->{uuid
});
5869 $uuid = PVE
::QemuServer
::PCI
::generate_mdev_uuid
($vmid, $index) if !defined($uuid);
5872 push @$cmd, '-uuid', $uuid if defined($uuid);
5875 eval { cleanup_pci_devices
($vmid, $conf) };
5880 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5883 my %silence_std_outs = (outfunc
=> sub {}, errfunc
=> sub {});
5884 eval { run_command
(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5885 eval { run_command
(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5886 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5887 # timeout should be more than enough here...
5888 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 20);
5890 my $cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{cpuunits
});
5893 timeout
=> $statefile ?
undef : $start_timeout,
5898 # when migrating, prefix QEMU output so other side can pick up any
5899 # errors that might occur and show the user
5900 if ($migratedfrom) {
5901 $run_params{quiet
} = 1;
5902 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5905 my %systemd_properties = (
5906 Slice
=> 'qemu.slice',
5907 KillMode
=> 'process',
5909 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5912 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5913 $systemd_properties{CPUWeight
} = $cpuunits;
5915 $systemd_properties{CPUShares
} = $cpuunits;
5918 if (my $cpulimit = $conf->{cpulimit
}) {
5919 $systemd_properties{CPUQuota
} = int($cpulimit * 100);
5921 $systemd_properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5923 my $run_qemu = sub {
5924 PVE
::Tools
::run_fork
sub {
5925 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5928 if (my $tpm = $conf->{tpmstate0
}) {
5929 # start the TPM emulator so QEMU can connect on start
5930 $tpmpid = start_swtpm
($storecfg, $vmid, $tpm, $migratedfrom);
5933 my $exitcode = run_command
($cmd, %run_params);
5936 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5937 kill 'TERM', $tpmpid;
5939 die "QEMU exited with code $exitcode\n";
5944 if ($conf->{hugepages
}) {
5947 my $hotplug_features =
5948 parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
5949 my $hugepages_topology =
5950 PVE
::QemuServer
::Memory
::hugepages_topology
($conf, $hotplug_features->{memory
});
5952 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5954 PVE
::QemuServer
::Memory
::hugepages_mount
();
5955 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5957 eval { $run_qemu->() };
5959 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5960 if !$conf->{keephugepages
};
5964 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5965 if !$conf->{keephugepages
};
5967 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5970 eval { $run_qemu->() };
5974 # deactivate volumes if start fails
5975 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5977 eval { cleanup_pci_devices
($vmid, $conf) };
5980 die "start failed: $err";
5983 # re-reserve all PCI IDs now that we can know the actual VM PID
5984 my $pid = PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
5985 eval { PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, undef, $pid) };
5988 if (defined($res->{migrate
})) {
5989 print "migration listens on $res->{migrate}->{uri}\n";
5990 } elsif ($statefile) {
5991 eval { mon_cmd
($vmid, "cont"); };
5995 #start nbd server for storage migration
5996 if (my $nbd = $migrate_opts->{nbd
}) {
5997 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5999 my $migrate_storage_uri;
6000 # nbd_protocol_version > 0 for unix socket support
6001 if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
6002 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
6003 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
6004 $migrate_storage_uri = "nbd:unix:$socket_path";
6005 $res->{migrate
}->{unix_sockets
} = [$socket_path];
6007 my $nodename = nodename
();
6008 my $localip = $get_migration_ip->($nodename);
6009 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
6010 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
6012 mon_cmd
($vmid, "nbd-server-start", addr
=> {
6015 host
=> "${localip}",
6016 port
=> "${storage_migrate_port}",
6019 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
6020 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
6023 my $block_info = mon_cmd
($vmid, "query-block");
6024 $block_info = { map { $_->{device
} => $_ } $block_info->@* };
6026 foreach my $opt (sort keys %$nbd) {
6027 my $drivestr = $nbd->{$opt}->{drivestr
};
6028 my $volid = $nbd->{$opt}->{volid
};
6030 my $block_node = $block_info->{"drive-$opt"}->{inserted
}->{'node-name'};
6036 'node-name' => $block_node,
6037 writable
=> JSON
::true
,
6039 name
=> "drive-$opt", # NBD export name
6042 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6043 print "storage migration listens on $nbd_uri volume:$drivestr\n";
6044 print "re-using replicated volume: $opt - $volid\n"
6045 if $nbd->{$opt}->{replicated
};
6047 $res->{drives
}->{$opt} = $nbd->{$opt};
6048 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
6052 if ($migratedfrom) {
6054 set_migration_caps
($vmid);
6059 print "spice listens on port $spice_port\n";
6060 $res->{spice_port
} = $spice_port;
6061 if ($migrate_opts->{spice_ticket
}) {
6062 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
6063 $migrate_opts->{spice_ticket
});
6064 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
6069 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
6070 if !$statefile && $conf->{balloon
};
6072 foreach my $opt (keys %$conf) {
6073 next if $opt !~ m/^net\d+$/;
6074 my $nicconf = parse_net
($conf->{$opt});
6075 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
6077 add_nets_bridge_fdb
($conf, $vmid);
6080 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
6085 path
=> "machine/peripheral/balloon0",
6086 property
=> "guest-stats-polling-interval",
6090 log_warn
("could not set polling interval for ballooning - $@") if $@;
6094 print "Resumed VM, removing state\n";
6095 if (my $vmstate = $conf->{vmstate
}) {
6096 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6097 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6099 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6100 PVE
::QemuConfig-
>write_config($vmid, $conf);
6103 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
6108 sub vm_commandline
{
6109 my ($storecfg, $vmid, $snapname) = @_;
6111 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6113 my ($forcemachine, $forcecpu);
6115 my $snapshot = $conf->{snapshots
}->{$snapname};
6116 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6118 # check for machine or CPU overrides in snapshot
6119 $forcemachine = $snapshot->{runningmachine
};
6120 $forcecpu = $snapshot->{runningcpu
};
6122 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
6127 my $defaults = load_defaults
();
6129 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6131 return PVE
::Tools
::cmd2string
($cmd);
6135 my ($vmid, $skiplock) = @_;
6137 PVE
::QemuConfig-
>lock_config($vmid, sub {
6139 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6141 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6143 mon_cmd
($vmid, "system_reset");
6147 sub get_vm_volumes
{
6151 foreach_volid
($conf, sub {
6152 my ($volid, $attr) = @_;
6154 return if $volid =~ m
|^/|;
6156 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6159 push @$vollist, $volid;
6165 sub cleanup_pci_devices
{
6166 my ($vmid, $conf) = @_;
6168 foreach my $key (keys %$conf) {
6169 next if $key !~ m/^hostpci(\d+)$/;
6170 my $hostpciindex = $1;
6171 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
6172 my $d = parse_hostpci
($conf->{$key});
6174 # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6175 # don't want to break ABI just for this two liner
6176 my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6178 # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6179 # out when we do it first. so wait for 10 seconds and then try it
6180 if ($d->{ids
}->[0]->[0]->{vendor
} =~ m/^(0x)?10de$/) {
6184 PVE
::SysFSTools
::file_write
("$dev_sysfs_dir/remove", "1") if -e
$dev_sysfs_dir;
6187 PVE
::QemuServer
::PCI
::remove_pci_reservation
($vmid);
6190 sub vm_stop_cleanup
{
6191 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6196 my $vollist = get_vm_volumes
($conf);
6197 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6199 if (my $tpmdrive = $conf->{tpmstate0
}) {
6200 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
6201 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
6203 PVE
::Storage
::unmap_volume
($storecfg, $tpm->{file
});
6208 foreach my $ext (qw(mon qmp pid vnc qga)) {
6209 unlink "/var/run/qemu-server/${vmid}.$ext";
6212 if ($conf->{ivshmem
}) {
6213 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
6214 # just delete it for now, VMs which have this already open do not
6215 # are affected, but new VMs will get a separated one. If this
6216 # becomes an issue we either add some sort of ref-counting or just
6217 # add a "don't delete on stop" flag to the ivshmem format.
6218 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
6221 cleanup_pci_devices
($vmid, $conf);
6223 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
6225 warn $@ if $@; # avoid errors - just warn
6228 # call only in locked context
6230 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6232 my $pid = check_running
($vmid, $nocheck);
6237 $conf = PVE
::QemuConfig-
>load_config($vmid);
6238 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6239 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
6240 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
6241 $timeout = $opts->{down
} if $opts->{down
};
6243 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
6248 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
6249 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
6251 mon_cmd
($vmid, "system_powerdown");
6254 mon_cmd
($vmid, "quit");
6260 $timeout = 60 if !defined($timeout);
6263 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6268 if ($count >= $timeout) {
6270 warn "VM still running - terminating now with SIGTERM\n";
6273 die "VM quit/powerdown failed - got timeout\n";
6276 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6280 if (!check_running
($vmid, $nocheck)) {
6281 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6285 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6288 die "VM quit/powerdown failed\n";
6296 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6301 if ($count >= $timeout) {
6302 warn "VM still running - terminating now with SIGKILL\n";
6307 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6310 # Note: use $nocheck to skip tests if VM configuration file exists.
6311 # We need that when migration VMs to other nodes (files already moved)
6312 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6314 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6316 $force = 1 if !defined($force) && !$shutdown;
6319 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
6320 kill 15, $pid if $pid;
6321 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
6322 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
6326 PVE
::QemuConfig-
>lock_config($vmid, sub {
6327 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6332 my ($vmid, $timeout) = @_;
6334 PVE
::QemuConfig-
>lock_config($vmid, sub {
6337 # only reboot if running, as qmeventd starts it again on a stop event
6338 return if !check_running
($vmid);
6340 create_reboot_request
($vmid);
6342 my $storecfg = PVE
::Storage
::config
();
6343 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
6347 # avoid that the next normal shutdown will be confused for a reboot
6348 clear_reboot_request
($vmid);
6354 # note: if using the statestorage parameter, the caller has to check privileges
6356 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6363 PVE
::QemuConfig-
>lock_config($vmid, sub {
6365 $conf = PVE
::QemuConfig-
>load_config($vmid);
6367 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
6368 PVE
::QemuConfig-
>check_lock($conf)
6369 if !($skiplock || $is_backing_up);
6371 die "cannot suspend to disk during backup\n"
6372 if $is_backing_up && $includestate;
6374 if ($includestate) {
6375 $conf->{lock} = 'suspending';
6376 my $date = strftime
("%Y-%m-%d", localtime(time()));
6377 $storecfg = PVE
::Storage
::config
();
6378 if (!$statestorage) {
6379 $statestorage = find_vmstate_storage
($conf, $storecfg);
6380 # check permissions for the storage
6381 my $rpcenv = PVE
::RPCEnvironment
::get
();
6382 if ($rpcenv->{type
} ne 'cli') {
6383 my $authuser = $rpcenv->get_user();
6384 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6389 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
6390 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6391 $path = PVE
::Storage
::path
($storecfg, $vmstate);
6392 PVE
::QemuConfig-
>write_config($vmid, $conf);
6394 mon_cmd
($vmid, "stop");
6398 if ($includestate) {
6400 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
6403 set_migration_caps
($vmid, 1);
6404 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
6406 my $state = mon_cmd
($vmid, "query-savevm");
6407 if (!$state->{status
}) {
6408 die "savevm not active\n";
6409 } elsif ($state->{status
} eq 'active') {
6412 } elsif ($state->{status
} eq 'completed') {
6413 print "State saved, quitting\n";
6415 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
6416 die "query-savevm failed with error '$state->{error}'\n"
6418 die "query-savevm returned status '$state->{status}'\n";
6424 PVE
::QemuConfig-
>lock_config($vmid, sub {
6425 $conf = PVE
::QemuConfig-
>load_config($vmid);
6427 # cleanup, but leave suspending lock, to indicate something went wrong
6429 mon_cmd
($vmid, "savevm-end");
6430 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6431 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6432 delete $conf->@{qw(vmstate runningmachine runningcpu)};
6433 PVE
::QemuConfig-
>write_config($vmid, $conf);
6439 die "lock changed unexpectedly\n"
6440 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
6442 mon_cmd
($vmid, "quit");
6443 $conf->{lock} = 'suspended';
6444 PVE
::QemuConfig-
>write_config($vmid, $conf);
6449 # $nocheck is set when called as part of a migration - in this context the
6450 # location of the config file (source or target node) is not deterministic,
6451 # since migration cannot wait for pmxcfs to process the rename
6453 my ($vmid, $skiplock, $nocheck) = @_;
6455 PVE
::QemuConfig-
>lock_config($vmid, sub {
6456 my $res = mon_cmd
($vmid, 'query-status');
6457 my $resume_cmd = 'cont';
6461 $conf = eval { PVE
::QemuConfig-
>load_config($vmid) }; # try on target node
6463 my $vmlist = PVE
::Cluster
::get_vmlist
();
6464 if (exists($vmlist->{ids
}->{$vmid})) {
6465 my $node = $vmlist->{ids
}->{$vmid}->{node
};
6466 $conf = eval { PVE
::QemuConfig-
>load_config($vmid, $node) }; # try on source node
6469 PVE
::Cluster
::cfs_update
(); # vmlist was wrong, invalidate cache
6470 $conf = PVE
::QemuConfig-
>load_config($vmid); # last try on target node again
6474 $conf = PVE
::QemuConfig-
>load_config($vmid);
6477 if ($res->{status
}) {
6478 return if $res->{status
} eq 'running'; # job done, go home
6479 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
6480 $reset = 1 if $res->{status
} eq 'shutdown';
6484 PVE
::QemuConfig-
>check_lock($conf)
6485 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
6489 # required if a VM shuts down during a backup and we get a resume
6490 # request before the backup finishes for example
6491 mon_cmd
($vmid, "system_reset");
6494 add_nets_bridge_fdb
($conf, $vmid) if $resume_cmd eq 'cont';
6496 mon_cmd
($vmid, $resume_cmd);
6501 my ($vmid, $skiplock, $key) = @_;
6503 PVE
::QemuConfig-
>lock_config($vmid, sub {
6505 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6507 # there is no qmp command, so we use the human monitor command
6508 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
6509 die $res if $res ne '';
6513 sub check_bridge_access
{
6514 my ($rpcenv, $authuser, $conf) = @_;
6516 return 1 if $authuser eq 'root@pam';
6518 for my $opt (sort keys $conf->%*) {
6519 next if $opt !~ m/^net\d+$/;
6520 my $net = parse_net
($conf->{$opt});
6521 my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6522 PVE
::GuestHelpers
::check_vnet_access
($rpcenv, $authuser, $bridge, $tag, $trunks);
6527 sub check_mapping_access
{
6528 my ($rpcenv, $user, $conf) = @_;
6530 for my $opt (keys $conf->%*) {
6531 if ($opt =~ m/^usb\d+$/) {
6532 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-usb', $conf->{$opt});
6533 if (my $host = $device->{host
}) {
6534 die "only root can set '$opt' config for real devices\n"
6535 if $host !~ m/^spice$/i && $user ne 'root@pam';
6536 } elsif ($device->{mapping
}) {
6537 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6539 die "either 'host' or 'mapping' must be set.\n";
6541 } elsif ($opt =~ m/^hostpci\d+$/) {
6542 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-hostpci', $conf->{$opt});
6543 if ($device->{host
}) {
6544 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6545 } elsif ($device->{mapping
}) {
6546 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6548 die "either 'host' or 'mapping' must be set.\n";
6554 sub check_restore_permissions
{
6555 my ($rpcenv, $user, $conf) = @_;
6557 check_bridge_access
($rpcenv, $user, $conf);
6558 check_mapping_access
($rpcenv, $user, $conf);
6560 # vzdump restore implementaion
6562 sub tar_archive_read_firstfile
{
6563 my $archive = shift;
6565 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
6567 # try to detect archive type first
6568 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6569 die "unable to open file '$archive'\n";
6570 my $firstfile = <$fh>;
6574 die "ERROR: archive contaions no data\n" if !$firstfile;
6580 sub tar_restore_cleanup
{
6581 my ($storecfg, $statfile) = @_;
6583 print STDERR
"starting cleanup\n";
6585 if (my $fd = IO
::File-
>new($statfile, "r")) {
6586 while (defined(my $line = <$fd>)) {
6587 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6590 if ($volid =~ m
|^/|) {
6591 unlink $volid || die 'unlink failed\n';
6593 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6595 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6597 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6599 print STDERR
"unable to parse line in statfile - $line";
6606 sub restore_file_archive
{
6607 my ($archive, $vmid, $user, $opts) = @_;
6609 return restore_vma_archive
($archive, $vmid, $user, $opts)
6612 my $info = PVE
::Storage
::archive_info
($archive);
6613 my $format = $opts->{format
} // $info->{format
};
6614 my $comp = $info->{compression
};
6616 # try to detect archive format
6617 if ($format eq 'tar') {
6618 return restore_tar_archive
($archive, $vmid, $user, $opts);
6620 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
6624 # hepler to remove disks that will not be used after restore
6625 my $restore_cleanup_oldconf = sub {
6626 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6628 my $kept_disks = {};
6630 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
6631 my ($ds, $drive) = @_;
6633 return if drive_is_cdrom
($drive, 1);
6635 my $volid = $drive->{file
};
6636 return if !$volid || $volid =~ m
|^/|;
6638 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
6639 return if !$path || !$owner || ($owner != $vmid);
6641 # Note: only delete disk we want to restore
6642 # other volumes will become unused
6643 if ($virtdev_hash->{$ds}) {
6644 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
6649 $kept_disks->{$volid} = 1;
6653 # after the restore we have no snapshots anymore
6654 for my $snapname (keys $oldconf->{snapshots
}->%*) {
6655 my $snap = $oldconf->{snapshots
}->{$snapname};
6656 if ($snap->{vmstate
}) {
6657 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
6663 for my $volid (keys $kept_disks->%*) {
6664 eval { PVE
::Storage
::volume_snapshot_delete
($storecfg, $volid, $snapname); };
6670 # Helper to parse vzdump backup device hints
6672 # $rpcenv: Environment, used to ckeck storage permissions
6673 # $user: User ID, to check storage permissions
6674 # $storecfg: Storage configuration
6675 # $fh: the file handle for reading the configuration
6676 # $devinfo: should contain device sizes for all backu-up'ed devices
6677 # $options: backup options (pool, default storage)
6679 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6680 my $parse_backup_hints = sub {
6681 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6683 my $check_storage = sub { # assert if an image can be allocate
6684 my ($storeid, $scfg) = @_;
6685 die "Content type 'images' is not available on storage '$storeid'\n"
6686 if !$scfg->{content
}->{images
};
6687 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6688 if $user ne 'root@pam';
6691 my $virtdev_hash = {};
6692 while (defined(my $line = <$fh>)) {
6693 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6694 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6695 die "archive does not contain data for drive '$virtdev'\n"
6696 if !$devinfo->{$devname};
6698 if (defined($options->{storage
})) {
6699 $storeid = $options->{storage
} || 'local';
6700 } elsif (!$storeid) {
6703 $format = 'raw' if !$format;
6704 $devinfo->{$devname}->{devname
} = $devname;
6705 $devinfo->{$devname}->{virtdev
} = $virtdev;
6706 $devinfo->{$devname}->{format
} = $format;
6707 $devinfo->{$devname}->{storeid
} = $storeid;
6709 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6710 $check_storage->($storeid, $scfg); # permission and content type check
6712 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6713 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6715 my $drive = parse_drive
($virtdev, $2);
6717 if (drive_is_cloudinit
($drive)) {
6718 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6719 $storeid = $options->{storage
} if defined ($options->{storage
});
6720 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6721 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6723 $check_storage->($storeid, $scfg); # permission and content type check
6725 $virtdev_hash->{$virtdev} = {
6727 storeid
=> $storeid,
6728 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6735 return $virtdev_hash;
6738 # Helper to allocate and activate all volumes required for a restore
6740 # $storecfg: Storage configuration
6741 # $virtdev_hash: as returned by parse_backup_hints()
6743 # Returns: { $virtdev => $volid }
6744 my $restore_allocate_devices = sub {
6745 my ($storecfg, $virtdev_hash, $vmid) = @_;
6748 foreach my $virtdev (sort keys %$virtdev_hash) {
6749 my $d = $virtdev_hash->{$virtdev};
6750 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6751 my $storeid = $d->{storeid
};
6752 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6754 # test if requested format is supported
6755 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6756 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6757 $d->{format
} = $defFormat if !$supported;
6760 if ($d->{is_cloudinit
}) {
6761 $name = "vm-$vmid-cloudinit";
6762 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6763 if ($scfg->{path
}) {
6764 $name .= ".$d->{format}";
6768 my $volid = PVE
::Storage
::vdisk_alloc
(
6769 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6771 print STDERR
"new volume ID is '$volid'\n";
6772 $d->{volid
} = $volid;
6774 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6776 $map->{$virtdev} = $volid;
6782 sub restore_update_config_line
{
6783 my ($cookie, $map, $line, $unique) = @_;
6785 return '' if $line =~ m/^\#qmdump\#/;
6786 return '' if $line =~ m/^\#vzdump\#/;
6787 return '' if $line =~ m/^lock:/;
6788 return '' if $line =~ m/^unused\d+:/;
6789 return '' if $line =~ m/^parent:/;
6793 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6794 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6795 # try to convert old 1.X settings
6796 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6797 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6798 my ($model, $macaddr) = split(/\=/, $devconfig);
6799 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6802 bridge
=> "vmbr$ind",
6803 macaddr
=> $macaddr,
6805 my $netstr = print_net
($net);
6807 $res .= "net$cookie->{netcount}: $netstr\n";
6808 $cookie->{netcount
}++;
6810 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6811 my ($id, $netstr) = ($1, $2);
6812 my $net = parse_net
($netstr);
6813 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6814 $netstr = print_net
($net);
6815 $res .= "$id: $netstr\n";
6816 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6819 my $di = parse_drive
($virtdev, $value);
6820 if (defined($di->{backup
}) && !$di->{backup
}) {
6822 } elsif ($map->{$virtdev}) {
6823 delete $di->{format
}; # format can change on restore
6824 $di->{file
} = $map->{$virtdev};
6825 $value = print_drive
($di);
6826 $res .= "$virtdev: $value\n";
6830 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6832 if ($vmgenid ne '0') {
6833 # always generate a new vmgenid if there was a valid one setup
6834 $vmgenid = generate_uuid
();
6836 $res .= "vmgenid: $vmgenid\n";
6837 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6838 my ($uuid, $uuid_str);
6839 UUID
::generate
($uuid);
6840 UUID
::unparse
($uuid, $uuid_str);
6841 my $smbios1 = parse_smbios1
($2);
6842 $smbios1->{uuid
} = $uuid_str;
6843 $res .= $1.print_smbios1
($smbios1)."\n";
6851 my $restore_deactivate_volumes = sub {
6852 my ($storecfg, $virtdev_hash) = @_;
6855 for my $dev (values $virtdev_hash->%*) {
6856 push $vollist->@*, $dev->{volid
} if $dev->{volid
};
6859 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
6860 print STDERR
$@ if $@;
6863 my $restore_destroy_volumes = sub {
6864 my ($storecfg, $virtdev_hash) = @_;
6866 for my $dev (values $virtdev_hash->%*) {
6867 my $volid = $dev->{volid
} or next;
6869 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6870 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6872 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6876 sub restore_merge_config
{
6877 my ($filename, $backup_conf_raw, $override_conf) = @_;
6879 my $backup_conf = parse_vm_config
($filename, $backup_conf_raw);
6880 for my $key (keys $override_conf->%*) {
6881 $backup_conf->{$key} = $override_conf->{$key};
6884 return $backup_conf;
6888 my ($cfg, $vmid) = @_;
6890 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6892 my $volid_hash = {};
6893 foreach my $storeid (keys %$info) {
6894 foreach my $item (@{$info->{$storeid}}) {
6895 next if !($item->{volid
} && $item->{size
});
6896 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6897 $volid_hash->{$item->{volid
}} = $item;
6904 sub update_disk_config
{
6905 my ($vmid, $conf, $volid_hash) = @_;
6908 my $prefix = "VM $vmid";
6910 # used and unused disks
6911 my $referenced = {};
6913 # Note: it is allowed to define multiple storages with same path (alias), so
6914 # we need to check both 'volid' and real 'path' (two different volid can point
6915 # to the same path).
6917 my $referencedpath = {};
6920 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6921 my ($opt, $drive) = @_;
6923 my $volid = $drive->{file
};
6925 my $volume = $volid_hash->{$volid};
6927 # mark volid as "in-use" for next step
6928 $referenced->{$volid} = 1;
6929 if ($volume && (my $path = $volume->{path
})) {
6930 $referencedpath->{$path} = 1;
6933 return if drive_is_cdrom
($drive);
6936 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6937 if (defined($updated)) {
6939 $conf->{$opt} = print_drive
($updated);
6940 print "$prefix ($opt): $msg\n";
6944 # remove 'unusedX' entry if volume is used
6945 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6946 my ($opt, $drive) = @_;
6948 my $volid = $drive->{file
};
6952 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6953 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6954 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6956 delete $conf->{$opt};
6959 $referenced->{$volid} = 1;
6960 $referencedpath->{$path} = 1 if $path;
6963 foreach my $volid (sort keys %$volid_hash) {
6964 next if $volid =~ m/vm-$vmid-state-/;
6965 next if $referenced->{$volid};
6966 my $path = $volid_hash->{$volid}->{path
};
6967 next if !$path; # just to be sure
6968 next if $referencedpath->{$path};
6970 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6971 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6972 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6979 my ($vmid, $nolock, $dryrun) = @_;
6981 my $cfg = PVE
::Storage
::config
();
6983 print "rescan volumes...\n";
6984 my $volid_hash = scan_volids
($cfg, $vmid);
6986 my $updatefn = sub {
6989 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6991 PVE
::QemuConfig-
>check_lock($conf);
6994 foreach my $volid (keys %$volid_hash) {
6995 my $info = $volid_hash->{$volid};
6996 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6999 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
7001 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
7004 if (defined($vmid)) {
7008 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
7011 my $vmlist = config_list
();
7012 foreach my $vmid (keys %$vmlist) {
7016 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
7022 sub restore_proxmox_backup_archive
{
7023 my ($archive, $vmid, $user, $options) = @_;
7025 my $storecfg = PVE
::Storage
::config
();
7027 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
7028 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7030 my $fingerprint = $scfg->{fingerprint
};
7031 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
7033 my $repo = PVE
::PBSClient
::get_repository
($scfg);
7034 my $namespace = $scfg->{namespace
};
7036 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7037 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
7038 local $ENV{PBS_PASSWORD
} = $password;
7039 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
7041 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7042 PVE
::Storage
::parse_volname
($storecfg, $archive);
7044 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7046 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7048 my $tmpdir = "/var/tmp/vzdumptmp$$";
7052 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7053 # disable interrupts (always do cleanups)
7057 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7059 # Note: $oldconf is undef if VM does not exists
7060 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7061 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7062 my $new_conf_raw = '';
7064 my $rpcenv = PVE
::RPCEnvironment
::get
();
7065 my $devinfo = {}; # info about drives included in backup
7066 my $virtdev_hash = {}; # info about allocated drives
7074 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7076 my $cfgfn = "$tmpdir/qemu-server.conf";
7077 my $firewall_config_fn = "$tmpdir/fw.conf";
7078 my $index_fn = "$tmpdir/index.json";
7080 my $cmd = "restore";
7082 my $param = [$pbs_backup_name, "index.json", $index_fn];
7083 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7084 my $index = PVE
::Tools
::file_get_contents
($index_fn);
7085 $index = decode_json
($index);
7087 foreach my $info (@{$index->{files
}}) {
7088 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
7090 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
7091 $devinfo->{$devname}->{size
} = $1;
7093 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7098 my $is_qemu_server_backup = scalar(
7099 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
7101 if (!$is_qemu_server_backup) {
7102 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7104 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
7106 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7107 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7109 if ($has_firewall_config) {
7110 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7111 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7113 my $pve_firewall_dir = '/etc/pve/firewall';
7114 mkdir $pve_firewall_dir; # make sure the dir exists
7115 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7118 my $fh = IO
::File-
>new($cfgfn, "r") ||
7119 die "unable to read qemu-server.conf - $!\n";
7121 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7123 # fixme: rate limit?
7125 # create empty/temp config
7126 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
7128 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7131 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7133 foreach my $virtdev (sort keys %$virtdev_hash) {
7134 my $d = $virtdev_hash->{$virtdev};
7135 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7137 # this fails if storage is unavailable
7138 my $volid = $d->{volid
};
7139 my $path = PVE
::Storage
::path
($storecfg, $volid);
7141 # for live-restore we only want to preload the efidisk and TPM state
7142 next if $options->{live
} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7145 if (defined(my $ns = $scfg->{namespace
})) {
7146 @ns_arg = ('--ns', $ns);
7149 my $pbs_restore_cmd = [
7150 '/usr/bin/pbs-restore',
7151 '--repository', $repo,
7154 "$d->{devname}.img.fidx",
7159 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
7160 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
7162 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
7163 push @$pbs_restore_cmd, '--skip-zero';
7166 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
7167 print "restore proxmox backup image: $dbg_cmdstring\n";
7168 run_command
($pbs_restore_cmd);
7171 $fh->seek(0, 0) || die "seek failed - $!\n";
7173 my $cookie = { netcount
=> 0 };
7174 while (defined(my $line = <$fh>)) {
7175 $new_conf_raw .= restore_update_config_line
(
7187 if ($err || !$options->{live
}) {
7188 $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7194 $restore_destroy_volumes->($storecfg, $virtdev_hash);
7198 if ($options->{live
}) {
7199 # keep lock during live-restore
7200 $new_conf_raw .= "\nlock: create";
7203 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $options->{override_conf
});
7204 check_restore_permissions
($rpcenv, $user, $new_conf);
7205 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7207 eval { rescan
($vmid, 1); };
7210 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
7212 if ($options->{live
}) {
7218 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
7220 my $conf = PVE
::QemuConfig-
>load_config($vmid);
7221 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
7223 # these special drives are already restored before start
7224 delete $devinfo->{'drive-efidisk0'};
7225 delete $devinfo->{'drive-tpmstate0-backup'};
7229 keyfile
=> $keyfile,
7230 snapshot
=> $pbs_backup_name,
7231 namespace
=> $namespace,
7233 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7235 PVE
::QemuConfig-
>remove_lock($vmid, "create");
7239 sub pbs_live_restore
{
7240 my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7242 print "starting VM for live-restore\n";
7243 print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7245 my $pbs_backing = {};
7246 for my $ds (keys %$restored_disks) {
7247 $ds =~ m/^drive-(.*)$/;
7249 $pbs_backing->{$confname} = {
7250 repository
=> $opts->{repo
},
7251 snapshot
=> $opts->{snapshot
},
7252 archive
=> "$ds.img.fidx",
7254 $pbs_backing->{$confname}->{keyfile
} = $opts->{keyfile
} if -e
$opts->{keyfile
};
7255 $pbs_backing->{$confname}->{namespace
} = $opts->{namespace
} if defined($opts->{namespace
});
7257 my $drive = parse_drive
($confname, $conf->{$confname});
7258 print "restoring '$ds' to '$drive->{file}'\n";
7261 my $drives_streamed = 0;
7263 # make sure HA doesn't interrupt our restore by stopping the VM
7264 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
7265 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7268 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7269 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7270 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'pbs-backing' => $pbs_backing}, {});
7272 my $qmeventd_fd = register_qmeventd_handle
($vmid);
7274 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7275 # this will effectively collapse the backing image chain consisting of
7276 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7277 # removes itself once all backing images vanish with 'auto-remove=on')
7279 for my $ds (sort keys %$restored_disks) {
7280 my $job_id = "restore-$ds";
7281 mon_cmd
($vmid, 'block-stream',
7282 'job-id' => $job_id,
7285 $jobs->{$job_id} = {};
7288 mon_cmd
($vmid, 'cont');
7289 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
7291 print "restore-drive jobs finished successfully, removing all tracking block devices"
7292 ." to disconnect from Proxmox Backup Server\n";
7294 for my $ds (sort keys %$restored_disks) {
7295 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7298 close($qmeventd_fd);
7304 warn "An error occurred during live-restore: $err\n";
7305 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
7306 die "live-restore failed\n";
7310 sub restore_vma_archive
{
7311 my ($archive, $vmid, $user, $opts, $comp) = @_;
7313 my $readfrom = $archive;
7315 my $cfg = PVE
::Storage
::config
();
7317 my $bwlimit = $opts->{bwlimit
};
7319 my $dbg_cmdstring = '';
7320 my $add_pipe = sub {
7322 push @$commands, $cmd;
7323 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7324 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
7329 if ($archive eq '-') {
7332 # If we use a backup from a PVE defined storage we also consider that
7333 # storage's rate limit:
7334 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
7335 if (defined($volid)) {
7336 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
7337 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
7339 print STDERR
"applying read rate limit: $readlimit\n";
7340 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7341 $add_pipe->($cstream);
7347 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
7348 my $cmd = $info->{decompressor
};
7349 push @$cmd, $readfrom;
7353 my $tmpdir = "/var/tmp/vzdumptmp$$";
7356 # disable interrupts (always do cleanups)
7360 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
7362 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7363 POSIX
::mkfifo
($mapfifo, 0600);
7365 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7367 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7372 my $devinfo = {}; # info about drives included in backup
7373 my $virtdev_hash = {}; # info about allocated drives
7375 my $rpcenv = PVE
::RPCEnvironment
::get
();
7377 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7379 # Note: $oldconf is undef if VM does not exist
7380 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7381 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7382 my $new_conf_raw = '';
7386 my $print_devmap = sub {
7387 my $cfgfn = "$tmpdir/qemu-server.conf";
7389 # we can read the config - that is already extracted
7390 my $fh = IO
::File-
>new($cfgfn, "r") ||
7391 die "unable to read qemu-server.conf - $!\n";
7393 my $fwcfgfn = "$tmpdir/qemu-server.fw";
7395 my $pve_firewall_dir = '/etc/pve/firewall';
7396 mkdir $pve_firewall_dir; # make sure the dir exists
7397 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7400 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7402 foreach my $info (values %{$virtdev_hash}) {
7403 my $storeid = $info->{storeid
};
7404 next if defined($storage_limits{$storeid});
7406 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
7407 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7408 $storage_limits{$storeid} = $limit * 1024;
7411 foreach my $devname (keys %$devinfo) {
7412 die "found no device mapping information for device '$devname'\n"
7413 if !$devinfo->{$devname}->{virtdev
};
7416 # create empty/temp config
7418 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
7419 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7423 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7425 # print restore information to $fifofh
7426 foreach my $virtdev (sort keys %$virtdev_hash) {
7427 my $d = $virtdev_hash->{$virtdev};
7428 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7430 my $storeid = $d->{storeid
};
7431 my $volid = $d->{volid
};
7434 if (my $limit = $storage_limits{$storeid}) {
7435 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7438 my $write_zeros = 1;
7439 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
7443 my $path = PVE
::Storage
::path
($cfg, $volid);
7445 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7447 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7450 $fh->seek(0, 0) || die "seek failed - $!\n";
7452 my $cookie = { netcount
=> 0 };
7453 while (defined(my $line = <$fh>)) {
7454 $new_conf_raw .= restore_update_config_line
(
7471 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7472 local $SIG{ALRM
} = sub { die "got timeout\n"; };
7474 $oldtimeout = alarm($timeout);
7481 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7482 my ($dev_id, $size, $devname) = ($1, $2, $3);
7483 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
7484 } elsif ($line =~ m/^CTIME: /) {
7485 # we correctly received the vma config, so we can disable
7486 # the timeout now for disk allocation (set to 10 minutes, so
7487 # that we always timeout if something goes wrong)
7490 print $fifofh "done\n";
7491 my $tmp = $oldtimeout || 0;
7492 $oldtimeout = undef;
7499 print "restore vma archive: $dbg_cmdstring\n";
7500 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
7504 alarm($oldtimeout) if $oldtimeout;
7506 $restore_deactivate_volumes->($cfg, $virtdev_hash);
7508 close($fifofh) if $fifofh;
7513 $restore_destroy_volumes->($cfg, $virtdev_hash);
7517 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $opts->{override_conf
});
7518 check_restore_permissions
($rpcenv, $user, $new_conf);
7519 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7521 eval { rescan
($vmid, 1); };
7524 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
7527 sub restore_tar_archive
{
7528 my ($archive, $vmid, $user, $opts) = @_;
7530 if (scalar(keys $opts->{override_conf
}->%*) > 0) {
7531 my $keystring = join(' ', keys $opts->{override_conf
}->%*);
7532 die "cannot pass along options ($keystring) when restoring from tar archive\n";
7535 if ($archive ne '-') {
7536 my $firstfile = tar_archive_read_firstfile
($archive);
7537 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7538 if $firstfile ne 'qemu-server.conf';
7541 my $storecfg = PVE
::Storage
::config
();
7543 # avoid zombie disks when restoring over an existing VM -> cleanup first
7544 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7545 # skiplock=1 because qmrestore has set the 'create' lock itself already
7546 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
7547 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
7549 my $tocmd = "/usr/lib/qemu-server/qmextract";
7551 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
7552 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
7553 $tocmd .= ' --prealloc' if $opts->{prealloc
};
7554 $tocmd .= ' --info' if $opts->{info
};
7556 # tar option "xf" does not autodetect compression when read from STDIN,
7557 # so we pipe to zcat
7558 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
7559 PVE
::Tools
::shellquote
("--to-command=$tocmd");
7561 my $tmpdir = "/var/tmp/vzdumptmp$$";
7564 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
7565 local $ENV{VZDUMP_VMID
} = $vmid;
7566 local $ENV{VZDUMP_USER
} = $user;
7568 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7569 my $new_conf_raw = '';
7571 # disable interrupts (always do cleanups)
7575 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7583 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7585 if ($archive eq '-') {
7586 print "extracting archive from STDIN\n";
7587 run_command
($cmd, input
=> "<&STDIN");
7589 print "extracting archive '$archive'\n";
7593 return if $opts->{info
};
7597 my $statfile = "$tmpdir/qmrestore.stat";
7598 if (my $fd = IO
::File-
>new($statfile, "r")) {
7599 while (defined (my $line = <$fd>)) {
7600 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7601 $map->{$1} = $2 if $1;
7603 print STDERR
"unable to parse line in statfile - $line\n";
7609 my $confsrc = "$tmpdir/qemu-server.conf";
7611 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7613 my $cookie = { netcount
=> 0 };
7614 while (defined (my $line = <$srcfd>)) {
7615 $new_conf_raw .= restore_update_config_line
(
7626 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
7632 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7634 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7636 eval { rescan
($vmid, 1); };
7640 sub foreach_storage_used_by_vm
{
7641 my ($conf, $func) = @_;
7645 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7646 my ($ds, $drive) = @_;
7647 return if drive_is_cdrom
($drive);
7649 my $volid = $drive->{file
};
7651 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
7652 $sidhash->{$sid} = $sid if $sid;
7655 foreach my $sid (sort keys %$sidhash) {
7660 my $qemu_snap_storage = {
7663 sub do_snapshots_with_qemu
{
7664 my ($storecfg, $volid, $deviceid) = @_;
7666 return if $deviceid =~ m/tpmstate0/;
7668 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
7669 my $scfg = $storecfg->{ids
}->{$storage_name};
7670 die "could not find storage '$storage_name'\n" if !defined($scfg);
7672 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
7676 if ($volid =~ m/\.(qcow2|qed)$/){
7683 sub qga_check_running
{
7684 my ($vmid, $nowarn) = @_;
7686 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
7688 warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7694 sub template_create
{
7695 my ($vmid, $conf, $disk) = @_;
7697 my $storecfg = PVE
::Storage
::config
();
7699 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7700 my ($ds, $drive) = @_;
7702 return if drive_is_cdrom
($drive);
7703 return if $disk && $ds ne $disk;
7705 my $volid = $drive->{file
};
7706 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
7708 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
7709 $drive->{file
} = $voliddst;
7710 $conf->{$ds} = print_drive
($drive);
7711 PVE
::QemuConfig-
>write_config($vmid, $conf);
7715 sub convert_iscsi_path
{
7718 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
7723 my $initiator_name = get_initiator_name
();
7725 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7726 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7729 die "cannot convert iscsi path '$path', unkown format\n";
7732 sub qemu_img_convert
{
7733 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7735 my $storecfg = PVE
::Storage
::config
();
7736 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
7737 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
7739 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7743 my $src_is_iscsi = 0;
7747 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
7748 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7749 $src_format = qemu_img_format
($src_scfg, $src_volname);
7750 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
7751 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
7752 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
7753 } elsif (-f
$src_volid || -b
$src_volid) {
7754 $src_path = $src_volid;
7755 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7760 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7762 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7763 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7764 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7765 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7768 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7769 push @$cmd, '-l', "snapshot.name=$snapname"
7770 if $snapname && $src_format && $src_format eq "qcow2";
7771 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7772 push @$cmd, '-T', $cachemode if defined($cachemode);
7773 push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7775 if ($src_is_iscsi) {
7776 push @$cmd, '--image-opts';
7777 $src_path = convert_iscsi_path
($src_path);
7778 } elsif ($src_format) {
7779 push @$cmd, '-f', $src_format;
7782 if ($dst_is_iscsi) {
7783 push @$cmd, '--target-image-opts';
7784 $dst_path = convert_iscsi_path
($dst_path);
7786 push @$cmd, '-O', $dst_format;
7789 push @$cmd, $src_path;
7791 if (!$dst_is_iscsi && $is_zero_initialized) {
7792 push @$cmd, "zeroinit:$dst_path";
7794 push @$cmd, $dst_path;
7799 if($line =~ m/\((\S+)\/100\
%\)/){
7801 my $transferred = int($size * $percent / 100);
7802 my $total_h = render_bytes
($size, 1);
7803 my $transferred_h = render_bytes
($transferred, 1);
7805 print "transferred $transferred_h of $total_h ($percent%)\n";
7810 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7812 die "copy failed: $err" if $err;
7815 sub qemu_img_format
{
7816 my ($scfg, $volname) = @_;
7818 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7825 sub qemu_drive_mirror
{
7826 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7828 $jobs = {} if !$jobs;
7832 $jobs->{"drive-$drive"} = {};
7834 if ($dst_volid =~ /^nbd:/) {
7835 $qemu_target = $dst_volid;
7838 my $storecfg = PVE
::Storage
::config
();
7839 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7841 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7843 $format = qemu_img_format
($dst_scfg, $dst_volname);
7845 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7847 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7850 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7851 $opts->{format
} = $format if $format;
7853 if (defined($src_bitmap)) {
7854 $opts->{sync
} = 'incremental';
7855 $opts->{bitmap
} = $src_bitmap;
7856 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7859 if (defined($bwlimit)) {
7860 $opts->{speed
} = $bwlimit * 1024;
7861 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7863 print "drive mirror is starting for drive-$drive\n";
7866 # if a job already runs for this device we get an error, catch it for cleanup
7867 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7869 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7871 die "mirroring error: $err\n";
7874 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7877 # $completion can be either
7878 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7879 # 'cancel': wait until all jobs are ready, block-job-cancel them
7880 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7881 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7882 sub qemu_drive_mirror_monitor
{
7883 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7885 $completion //= 'complete';
7889 my $err_complete = 0;
7891 my $starttime = time ();
7893 die "block job ('$op') timed out\n" if $err_complete > 300;
7895 my $stats = mon_cmd
($vmid, "query-block-jobs");
7898 my $running_jobs = {};
7899 for my $stat (@$stats) {
7900 next if $stat->{type
} ne $op;
7901 $running_jobs->{$stat->{device
}} = $stat;
7904 my $readycounter = 0;
7906 for my $job_id (sort keys %$jobs) {
7907 my $job = $running_jobs->{$job_id};
7909 my $vanished = !defined($job);
7910 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7911 if($complete || ($vanished && $completion eq 'auto')) {
7912 print "$job_id: $op-job finished\n";
7913 delete $jobs->{$job_id};
7917 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7919 my $busy = $job->{busy
};
7920 my $ready = $job->{ready
};
7921 if (my $total = $job->{len
}) {
7922 my $transferred = $job->{offset
} || 0;
7923 my $remaining = $total - $transferred;
7924 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7926 my $duration = $ctime - $starttime;
7927 my $total_h = render_bytes
($total, 1);
7928 my $transferred_h = render_bytes
($transferred, 1);
7930 my $status = sprintf(
7931 "transferred $transferred_h of $total_h ($percent%%) in %s",
7932 render_duration
($duration),
7937 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7939 $status .= ", ready";
7942 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
7943 $jobs->{$job_id}->{ready
} = $ready;
7946 $readycounter++ if $job->{ready
};
7949 last if scalar(keys %$jobs) == 0;
7951 if ($readycounter == scalar(keys %$jobs)) {
7952 print "all '$op' jobs are ready\n";
7954 # do the complete later (or has already been done)
7955 last if $completion eq 'skip' || $completion eq 'auto';
7957 if ($vmiddst && $vmiddst != $vmid) {
7958 my $agent_running = $qga && qga_check_running
($vmid);
7959 if ($agent_running) {
7960 print "freeze filesystem\n";
7961 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
7964 print "suspend vm\n";
7965 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
7969 # if we clone a disk for a new target vm, we don't switch the disk
7970 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
7972 if ($agent_running) {
7973 print "unfreeze filesystem\n";
7974 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
7977 print "resume vm\n";
7978 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
7985 for my $job_id (sort keys %$jobs) {
7986 # try to switch the disk if source and destination are on the same guest
7987 print "$job_id: Completing block job_id...\n";
7990 if ($completion eq 'complete') {
7991 $op = 'block-job-complete';
7992 } elsif ($completion eq 'cancel') {
7993 $op = 'block-job-cancel';
7995 die "invalid completion value: $completion\n";
7997 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
7998 if ($@ =~ m/cannot be completed/) {
7999 print "$job_id: block job cannot be completed, trying again.\n";
8002 print "$job_id: Completed successfully.\n";
8003 $jobs->{$job_id}->{complete
} = 1;
8014 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
8015 die "block job ($op) error: $err";
8019 sub qemu_blockjobs_cancel
{
8020 my ($vmid, $jobs) = @_;
8022 foreach my $job (keys %$jobs) {
8023 print "$job: Cancelling block job\n";
8024 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
8025 $jobs->{$job}->{cancel
} = 1;
8029 my $stats = mon_cmd
($vmid, "query-block-jobs");
8031 my $running_jobs = {};
8032 foreach my $stat (@$stats) {
8033 $running_jobs->{$stat->{device
}} = $stat;
8036 foreach my $job (keys %$jobs) {
8038 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
8039 print "$job: Done.\n";
8040 delete $jobs->{$job};
8044 last if scalar(keys %$jobs) == 0;
8050 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8051 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8052 my sub clone_disk_check_io_uring
{
8053 my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8055 return if !$use_drive_mirror;
8057 # Don't complain when not changing storage.
8058 # Assume if it works for the source, it'll work for the target too.
8059 return if $src_storeid eq $dst_storeid;
8061 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
8062 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
8064 my $cache_direct = drive_uses_cache_direct
($src_drive);
8066 my $src_uses_io_uring;
8067 if ($src_drive->{aio
}) {
8068 $src_uses_io_uring = $src_drive->{aio
} eq 'io_uring';
8070 $src_uses_io_uring = storage_allows_io_uring_default
($src_scfg, $cache_direct);
8073 die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8074 if $src_uses_io_uring && !storage_allows_io_uring_default
($dst_scfg, $cache_direct);
8078 my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8080 my ($vmid, $running) = $source->@{qw(vmid running)};
8081 my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8083 my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8084 my ($storage, $format) = $dest->@{qw(storage format)};
8086 my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8088 if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8089 die "cloning from/to EFI disk requires EFI disk\n"
8090 if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8091 die "cloning from/to TPM state requires TPM state\n"
8092 if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8094 # This would lead to two device nodes in QEMU pointing to the same backing image!
8095 die "cannot change drive name when cloning disk from/to the same VM\n"
8096 if $use_drive_mirror && $vmid == $newvmid;
8099 die "cannot move TPM state while VM is running\n"
8100 if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8104 print "create " . ($full ?
'full' : 'linked') . " clone of drive ";
8105 print "$src_drivename " if $src_drivename;
8106 print "($drive->{file})\n";
8109 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
8110 push @$newvollist, $newvolid;
8112 my ($src_storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
8113 my $storeid = $storage || $src_storeid;
8115 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
8119 if (drive_is_cloudinit
($drive)) {
8120 $name = "vm-$newvmid-cloudinit";
8121 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8122 if ($scfg->{path
}) {
8123 $name .= ".$dst_format";
8126 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
8127 } elsif ($dst_drivename eq 'efidisk0') {
8128 $size = $efisize or die "internal error - need to specify EFI disk size\n";
8129 } elsif ($dst_drivename eq 'tpmstate0') {
8130 $dst_format = 'raw';
8131 $size = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8133 clone_disk_check_io_uring
($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8135 $size = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
8137 $newvolid = PVE
::Storage
::vdisk_alloc
(
8138 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8140 push @$newvollist, $newvolid;
8142 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
8144 if (drive_is_cloudinit
($drive)) {
8145 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8146 # if this is the case, we have to complete any block-jobs still there from
8147 # previous drive-mirrors
8148 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8149 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
8154 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
8155 if ($use_drive_mirror) {
8156 qemu_drive_mirror
($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8157 $completion, $qga, $bwlimit);
8159 if ($dst_drivename eq 'efidisk0') {
8160 # the relevant data on the efidisk may be smaller than the source
8161 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8162 # that is given by the OVMF_VARS.fd
8163 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
8164 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
8166 my $src_format = (PVE
::Storage
::parse_volname
($storecfg, $drive->{file
}))[6];
8168 # better for Ceph if block size is not too small, see bug #3324
8171 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8173 if ($src_format eq 'qcow2' && $snapname) {
8174 die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8175 if !min_version
(kvm_user_version
(), 6, 2);
8176 push $cmd->@*, '-l', $snapname;
8178 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8181 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8187 my $size = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
8189 my $disk = dclone
($drive);
8190 delete $disk->{format
};
8191 $disk->{file
} = $newvolid;
8192 $disk->{size
} = $size if defined($size);
8197 sub get_running_qemu_version
{
8199 my $res = mon_cmd
($vmid, "query-version");
8200 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8203 sub qemu_use_old_bios_files
{
8204 my ($machine_type) = @_;
8206 return if !$machine_type;
8208 my $use_old_bios_files = undef;
8210 if ($machine_type =~ m/^(\S+)\.pxe$/) {
8212 $use_old_bios_files = 1;
8214 my $version = extract_version
($machine_type, kvm_user_version
());
8215 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8216 # load new efi bios files on migration. So this hack is required to allow
8217 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8218 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8219 $use_old_bios_files = !min_version
($version, 2, 4);
8222 return ($use_old_bios_files, $machine_type);
8225 sub get_efivars_size
{
8226 my ($conf, $efidisk) = @_;
8228 my $arch = get_vm_arch
($conf);
8229 $efidisk //= $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
8230 my $smm = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
8231 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8232 return -s
$ovmf_vars;
8235 sub update_efidisk_size
{
8238 return if !defined($conf->{efidisk0
});
8240 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
8241 $disk->{size
} = get_efivars_size
($conf);
8242 $conf->{efidisk0
} = print_drive
($disk);
8247 sub update_tpmstate_size
{
8250 my $disk = PVE
::QemuServer
::parse_drive
('tpmstate0', $conf->{tpmstate0
});
8251 $disk->{size
} = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8252 $conf->{tpmstate0
} = print_drive
($disk);
8255 sub create_efidisk
($$$$$$$) {
8256 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8258 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8260 my $vars_size_b = -s
$ovmf_vars;
8261 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
8262 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8263 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
8265 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
8266 my $size = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
8268 return ($volid, $size/1024);
8271 sub vm_iothreads_list
{
8274 my $res = mon_cmd
($vmid, 'query-iothreads');
8277 foreach my $iothread (@$res) {
8278 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
8285 my ($conf, $drive) = @_;
8289 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
8291 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
8297 my $controller = int($drive->{index} / $maxdev);
8298 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
8302 return ($maxdev, $controller, $controller_prefix);
8305 sub resolve_dst_disk_format
{
8306 my ($storecfg, $storeid, $src_volname, $format) = @_;
8307 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
8310 # if no target format is specified, use the source disk format as hint
8312 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8313 $format = qemu_img_format
($scfg, $src_volname);
8319 # test if requested format is supported - else use default
8320 my $supported = grep { $_ eq $format } @$validFormats;
8321 $format = $defFormat if !$supported;
8325 # NOTE: if this logic changes, please update docs & possibly gui logic
8326 sub find_vmstate_storage
{
8327 my ($conf, $storecfg) = @_;
8329 # first, return storage from conf if set
8330 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
8332 my ($target, $shared, $local);
8334 foreach_storage_used_by_vm
($conf, sub {
8336 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
8337 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
8338 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
8341 # second, use shared storage where VM has at least one disk
8342 # third, use local storage where VM has at least one disk
8343 # fall back to local storage
8344 $target = $shared // $local // 'local';
8350 my ($uuid, $uuid_str);
8351 UUID
::generate
($uuid);
8352 UUID
::unparse
($uuid, $uuid_str);
8356 sub generate_smbios1_uuid
{
8357 return "uuid=".generate_uuid
();
8363 mon_cmd
($vmid, 'nbd-server-stop');
8366 sub create_reboot_request
{
8368 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8369 or die "failed to create reboot trigger file: $!\n";
8373 sub clear_reboot_request
{
8375 my $path = "/run/qemu-server/$vmid.reboot";
8378 $res = unlink($path);
8379 die "could not remove reboot request for $vmid: $!"
8380 if !$res && $! != POSIX
::ENOENT
;
8385 sub bootorder_from_legacy
{
8386 my ($conf, $bootcfg) = @_;
8388 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
8389 my $bootindex_hash = {};
8391 foreach my $o (split(//, $boot)) {
8392 $bootindex_hash->{$o} = $i*100;
8398 PVE
::QemuConfig-
>foreach_volume($conf, sub {
8399 my ($ds, $drive) = @_;
8401 if (drive_is_cdrom
($drive, 1)) {
8402 if ($bootindex_hash->{d
}) {
8403 $bootorder->{$ds} = $bootindex_hash->{d
};
8404 $bootindex_hash->{d
} += 1;
8406 } elsif ($bootindex_hash->{c
}) {
8407 $bootorder->{$ds} = $bootindex_hash->{c
}
8408 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
8409 $bootindex_hash->{c
} += 1;
8413 if ($bootindex_hash->{n
}) {
8414 for (my $i = 0; $i < $MAX_NETS; $i++) {
8415 my $netname = "net$i";
8416 next if !$conf->{$netname};
8417 $bootorder->{$netname} = $bootindex_hash->{n
};
8418 $bootindex_hash->{n
} += 1;
8425 # Generate default device list for 'boot: order=' property. Matches legacy
8426 # default boot order, but with explicit device names. This is important, since
8427 # the fallback for when neither 'order' nor the old format is specified relies
8428 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8429 sub get_default_bootdevices
{
8435 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
8436 push @ret, $first if $first;
8439 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
8440 push @ret, $first if $first;
8443 for (my $i = 0; $i < $MAX_NETS; $i++) {
8444 my $netname = "net$i";
8445 next if !$conf->{$netname};
8446 push @ret, $netname;
8453 sub device_bootorder
{
8456 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
8458 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
8461 if (!defined($boot) || $boot->{legacy
}) {
8462 $bootorder = bootorder_from_legacy
($conf, $boot);
8463 } elsif ($boot->{order
}) {
8464 my $i = 100; # start at 100 to allow user to insert devices before us with -args
8465 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
8466 $bootorder->{$dev} = $i++;
8473 sub register_qmeventd_handle
{
8477 my $peer = "/var/run/qmeventd.sock";
8482 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
8484 if ($! != EINTR
&& $! != EAGAIN
) {
8485 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8488 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8489 . "after $count retries\n";
8494 # send handshake to mark VM as backing up
8495 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
8497 # return handle to be closed later when inhibit is no longer required
8501 # bash completion helper
8503 sub complete_backup_archives
{
8504 my ($cmdname, $pname, $cvalue) = @_;
8506 my $cfg = PVE
::Storage
::config
();
8510 if ($cvalue =~ m/^([^:]+):/) {
8514 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
8517 foreach my $id (keys %$data) {
8518 foreach my $item (@{$data->{$id}}) {
8519 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8520 push @$res, $item->{volid
} if defined($item->{volid
});
8527 my $complete_vmid_full = sub {
8530 my $idlist = vmstatus
();
8534 foreach my $id (keys %$idlist) {
8535 my $d = $idlist->{$id};
8536 if (defined($running)) {
8537 next if $d->{template
};
8538 next if $running && $d->{status
} ne 'running';
8539 next if !$running && $d->{status
} eq 'running';
8548 return &$complete_vmid_full();
8551 sub complete_vmid_stopped
{
8552 return &$complete_vmid_full(0);
8555 sub complete_vmid_running
{
8556 return &$complete_vmid_full(1);
8559 sub complete_storage
{
8561 my $cfg = PVE
::Storage
::config
();
8562 my $ids = $cfg->{ids
};
8565 foreach my $sid (keys %$ids) {
8566 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
8567 next if !$ids->{$sid}->{content
}->{images
};
8574 sub complete_migration_storage
{
8575 my ($cmd, $param, $current_value, $all_args) = @_;
8577 my $targetnode = @$all_args[1];
8579 my $cfg = PVE
::Storage
::config
();
8580 my $ids = $cfg->{ids
};
8583 foreach my $sid (keys %$ids) {
8584 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
8585 next if !$ids->{$sid}->{content
}->{images
};
8594 my $qmpstatus = eval {
8595 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
8596 mon_cmd
($vmid, "query-status");
8599 return $qmpstatus && $qmpstatus->{status
} eq "paused";
8602 sub check_volume_storage_type
{
8603 my ($storecfg, $vol) = @_;
8605 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
8606 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8607 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
8609 die "storage '$storeid' does not support content-type '$vtype'\n"
8610 if !$scfg->{content
}->{$vtype};
8615 sub add_nets_bridge_fdb
{
8616 my ($conf, $vmid) = @_;
8618 for my $opt (keys %$conf) {
8619 next if $opt !~ m/^net(\d+)$/;
8620 my $iface = "tap${vmid}i$1";
8621 # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8622 my $net = parse_net
($conf->{$opt}, 1) or next;
8624 my $mac = $net->{macaddr
};
8626 log_warn
("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8627 if !file_read_firstline
("/sys/class/net/$iface/brport/learning");
8631 my $bridge = $net->{bridge
};
8633 log_warn
("Interface '$iface' not attached to any bridge.");
8637 PVE
::Network
::SDN
::Zones
::add_bridge_fdb
($iface, $mac, $bridge, $net->{firewall
});
8638 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8639 PVE
::Network
::add_bridge_fdb
($iface, $mac, $net->{firewall
});
8644 sub del_nets_bridge_fdb
{
8645 my ($conf, $vmid) = @_;
8647 for my $opt (keys %$conf) {
8648 next if $opt !~ m/^net(\d+)$/;
8649 my $iface = "tap${vmid}i$1";
8651 my $net = parse_net
($conf->{$opt}) or next;
8652 my $mac = $net->{macaddr
} or next;
8654 my $bridge = $net->{bridge
};
8656 PVE
::Network
::SDN
::Zones
::del_bridge_fdb
($iface, $mac, $bridge, $net->{firewall
});
8657 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8658 PVE
::Network
::del_bridge_fdb
($iface, $mac, $net->{firewall
});