1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
22 use List
::Util
qw(first);
25 use Storable
qw(dclone);
26 use Time
::HiRes
qw(gettimeofday usleep);
30 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
33 use PVE
::DataCenterConfig
;
34 use PVE
::Exception
qw(raise raise_param_exc);
35 use PVE
::Format
qw(render_duration render_bytes);
36 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
37 use PVE
::Mapping
::PCI
;
38 use PVE
::Mapping
::USB
;
40 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
43 use PVE
::RESTEnvironment
qw(log_warn);
44 use PVE
::RPCEnvironment
;
48 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
52 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout windows_version);
53 use PVE
::QemuServer
::Cloudinit
;
54 use PVE
::QemuServer
::CGroup
;
55 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
56 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
57 use PVE
::QemuServer
::Machine
;
58 use PVE
::QemuServer
::Memory
;
59 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
60 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
61 use PVE
::QemuServer
::USB
;
65 require PVE
::Network
::SDN
::Zones
;
69 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
73 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
74 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
77 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
78 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
81 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
82 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
85 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
86 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
89 "$EDK2_FW_BASE/OVMF_CODE.fd",
90 "$EDK2_FW_BASE/OVMF_VARS.fd",
95 "$EDK2_FW_BASE/AAVMF_CODE.fd",
96 "$EDK2_FW_BASE/AAVMF_VARS.fd",
101 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
103 # Note about locking: we use flock on the config file protect against concurent actions.
104 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
105 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
106 # But you can ignore this kind of lock with the --skiplock flag.
114 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
115 description
=> "Some command save/restore state from this location.",
121 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
122 description
=> "Specifies the QEMU machine type.",
124 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
129 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
132 $nodename_cache //= PVE
::INotify
::nodename
();
133 return $nodename_cache;
140 enum
=> [qw(i6300esb ib700)],
141 description
=> "Watchdog type to emulate.",
142 default => 'i6300esb',
147 enum
=> [qw(reset shutdown poweroff pause debug none)],
148 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
152 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
156 description
=> "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
161 fstrim_cloned_disks
=> {
162 description
=> "Run fstrim after moving a disk or migrating the VM.",
167 'freeze-fs-on-backup' => {
168 description
=> "Freeze/thaw guest filesystems on backup for consistency.",
174 description
=> "Select the agent type",
178 enum
=> [qw(virtio isa)],
184 description
=> "Select the VGA type.",
189 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
192 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
204 description
=> "The size of the file in MB.",
208 pattern
=> '[a-zA-Z0-9\-]+',
210 format_description
=> 'string',
211 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
218 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
219 description
=> "Configure an audio device."
223 enum
=> ['spice', 'none'],
226 description
=> "Driver backend for the audio device."
230 my $spice_enhancements_fmt = {
235 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
239 enum
=> ['off', 'all', 'filter'],
242 description
=> "Enable video streaming. Uses compression for detected video streams."
249 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
251 description
=> "The file on the host to gather entropy from. In most cases '/dev/urandom'"
252 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
253 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
254 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
255 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
256 ." a hardware RNG from the host.",
260 description
=> "Maximum bytes of entropy allowed to get injected into the guest every"
261 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
262 ." `0` to disable limiting (potentially dangerous!).",
265 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
266 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
267 # reading from /dev/urandom
272 description
=> "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
273 ." the guest to retrieve another 'max_bytes' of entropy.",
279 my $meta_info_fmt = {
282 description
=> "The guest creation timestamp as UNIX epoch time",
288 description
=> "The QEMU (machine) version from the time this VM was created.",
289 pattern
=> '\d+(\.\d+)+',
298 description
=> "Specifies whether a VM will be started during system bootup.",
304 description
=> "Automatic restart after crash (currently ignored).",
309 type
=> 'string', format
=> 'pve-hotplug-features',
310 description
=> "Selectively enable hotplug features. This is a comma separated list of"
311 ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
312 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
313 ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
315 default => 'network,disk,usb',
320 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
326 description
=> "Lock/unlock the VM.",
327 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
332 description
=> "Limit of CPU usage.",
333 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
334 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
342 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
343 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
344 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
345 ." weights of all the other running VMs.",
348 default => 'cgroup v1: 1024, cgroup v2: 100',
353 description
=> "Amount of RAM for the VM in MiB. This is the maximum available memory when"
354 ." you use the balloon device.",
361 description
=> "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
367 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the"
368 ." more memory this VM gets. Number is relative to weights of all other running VMs."
369 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
377 description
=> "Keyboard layout for VNC server. This option is generally not required and"
378 ." is often better handled from within the guest OS.",
379 enum
=> PVE
::Tools
::kvmkeymaplist
(),
384 type
=> 'string', format
=> 'dns-name',
385 description
=> "Set a name for the VM. Only used on the configuration web interface.",
390 description
=> "SCSI controller model",
391 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
397 description
=> "Description for the VM. Shown in the web-interface VM's summary."
398 ." This is saved as comment inside the configuration file.",
399 maxLength
=> 1024 * 8,
404 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
405 description
=> "Specify guest operating system.",
406 verbose_description
=> <<EODESC,
407 Specify guest operating system. This is used to enable special
408 optimization/features for specific operating systems:
411 other;; unspecified OS
412 wxp;; Microsoft Windows XP
413 w2k;; Microsoft Windows 2000
414 w2k3;; Microsoft Windows 2003
415 w2k8;; Microsoft Windows 2008
416 wvista;; Microsoft Windows Vista
417 win7;; Microsoft Windows 7
418 win8;; Microsoft Windows 8/2012/2012r2
419 win10;; Microsoft Windows 10/2016/2019
420 win11;; Microsoft Windows 11/2022
421 l24;; Linux 2.4 Kernel
422 l26;; Linux 2.6 - 6.X Kernel
423 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
428 type
=> 'string', format
=> 'pve-qm-boot',
429 description
=> "Specify guest boot order. Use the 'order=' sub-property as usage with no"
430 ." key or 'legacy=' is deprecated.",
434 type
=> 'string', format
=> 'pve-qm-bootdisk',
435 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
436 pattern
=> '(ide|sata|scsi|virtio)\d+',
441 description
=> "The number of CPUs. Please use option -sockets instead.",
448 description
=> "The number of CPU sockets.",
455 description
=> "The number of cores per socket.",
462 description
=> "Enable/disable NUMA.",
468 description
=> "Enable/disable hugepages memory.",
469 enum
=> [qw(any 2 1024)],
475 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
476 ." after VM shutdown and can be used for subsequent starts.",
481 description
=> "Number of hotplugged vcpus.",
488 description
=> "Enable/disable ACPI.",
493 description
=> "Enable/disable communication with the QEMU Guest Agent and its properties.",
495 format
=> $agent_fmt,
500 description
=> "Enable/disable KVM hardware virtualization.",
506 description
=> "Enable/disable time drift fix.",
512 description
=> "Set the real time clock (RTC) to local time. This is enabled by default if"
513 ." the `ostype` indicates a Microsoft Windows OS.",
518 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
522 type
=> 'string', format
=> $vga_fmt,
523 description
=> "Configure the VGA hardware.",
524 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
525 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
526 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
527 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
528 ." display server. For win* OS you can select how many independent displays you want,"
529 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
530 ." using a serial device as terminal.",
534 type
=> 'string', format
=> 'pve-qm-watchdog',
535 description
=> "Create a virtual hardware watchdog device.",
536 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
537 ." action), the watchdog must be periodically polled by an agent inside the guest or"
538 ." else the watchdog will reset the guest (or execute the respective action specified)",
543 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
544 description
=> "Set the initial date of the real time clock. Valid format for date are:"
545 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
546 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
549 startup
=> get_standard_option
('pve-startup-order'),
553 description
=> "Enable/disable Template.",
559 description
=> "Arbitrary arguments passed to kvm.",
560 verbose_description
=> <<EODESCR,
561 Arbitrary arguments passed to kvm, for example:
563 args: -no-reboot -smbios 'type=0,vendor=FOO'
565 NOTE: this option is for experts only.
572 description
=> "Enable/disable the USB tablet device.",
573 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
574 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
575 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
576 ." may consider disabling this to save some context switches. This is turned off by"
577 ." default if you use spice (`qm set <vmid> --vga qxl`).",
582 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
586 migrate_downtime
=> {
589 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
595 type
=> 'string', format
=> 'pve-qm-ide',
596 typetext
=> '<volume>',
597 description
=> "This is an alias for option -ide2",
601 description
=> "Emulated CPU type.",
603 format
=> 'pve-vm-cpu-conf',
605 parent
=> get_standard_option
('pve-snapshot-name', {
607 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
611 description
=> "Timestamp for snapshots.",
617 type
=> 'string', format
=> 'pve-volume-id',
618 description
=> "Reference to a volume which stores the VM state. This is used internally"
621 vmstatestorage
=> get_standard_option
('pve-storage-id', {
622 description
=> "Default storage for VM state volumes/files.",
625 runningmachine
=> get_standard_option
('pve-qemu-machine', {
626 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
630 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
631 ." internally for snapshots.",
634 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
635 format_description
=> 'QEMU -cpu parameter'
637 machine
=> get_standard_option
('pve-qemu-machine'),
639 description
=> "Virtual processor architecture. Defaults to the host.",
642 enum
=> [qw(x86_64 aarch64)],
645 description
=> "Specify SMBIOS type 1 fields.",
646 type
=> 'string', format
=> 'pve-qm-smbios1',
653 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
654 ." remove disk operations.",
660 enum
=> [ qw(seabios ovmf) ],
661 description
=> "Select BIOS implementation.",
662 default => 'seabios',
666 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
667 format_description
=> 'UUID',
668 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
669 ." to disable explicitly.",
670 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
671 ." value identifier to the guest OS. This allows to notify the guest operating system"
672 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
673 ." execution or creation from a template). The guest operating system notices the"
674 ." change, and is then able to react as appropriate by marking its copies of"
675 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
676 ."Note that auto-creation only works when done through API/CLI create or update methods"
677 .", but not when manually editing the config file.",
678 default => "1 (autogenerated)",
683 format
=> 'pve-volume-id',
685 description
=> "Script that will be executed during various steps in the vms lifetime.",
689 format
=> $ivshmem_fmt,
690 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
696 format
=> $audio_fmt,
697 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
700 spice_enhancements
=> {
702 format
=> $spice_enhancements_fmt,
703 description
=> "Configure additional enhancements for SPICE.",
707 type
=> 'string', format
=> 'pve-tag-list',
708 description
=> 'Tags of the VM. This is only meta information.',
714 description
=> "Configure a VirtIO-based Random Number Generator.",
719 format
=> $meta_info_fmt,
720 description
=> "Some (read-only) meta-information about this guest.",
724 type
=> 'string', format
=> 'pve-cpuset',
725 description
=> "List of host cores used to execute guest processes, for example: 0,5,8-11",
734 description
=> 'Specify a custom file containing all meta data passed to the VM via"
735 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
736 format
=> 'pve-volume-id',
737 format_description
=> 'volume',
742 description
=> 'To pass a custom file containing all network data to the VM via cloud-init.',
743 format
=> 'pve-volume-id',
744 format_description
=> 'volume',
749 description
=> 'To pass a custom file containing all user data to the VM via cloud-init.',
750 format
=> 'pve-volume-id',
751 format_description
=> 'volume',
756 description
=> 'To pass a custom file containing all vendor data to the VM via cloud-init.',
757 format
=> 'pve-volume-id',
758 format_description
=> 'volume',
761 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
763 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
764 my $confdesc_cloudinit = {
768 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
769 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
770 .' and `configdrive2` for windows.',
771 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
776 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
777 ." image's configured default user.",
782 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
783 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
784 .' support hashed passwords.',
789 description
=> 'cloud-init: do an automatic package upgrade after the first boot.',
795 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
797 format
=> 'pve-qm-cicustom',
802 description
=> 'cloud-init: Sets DNS search domains for a container. Create will'
803 .' automatically use the setting from the host if neither searchdomain nor nameserver'
808 type
=> 'string', format
=> 'address-list',
809 description
=> 'cloud-init: Sets DNS server IP address for a container. Create will'
810 .' automatically use the setting from the host if neither searchdomain nor nameserver'
816 format
=> 'urlencoded',
817 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
821 # what about other qemu settings ?
823 #machine => 'string',
836 ##soundhw => 'string',
838 while (my ($k, $v) = each %$confdesc) {
839 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
843 my $MAX_SERIAL_PORTS = 4;
844 my $MAX_PARALLEL_PORTS = 3;
850 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
851 description
=> "CPUs accessing this NUMA node.",
852 format_description
=> "id[-id];...",
856 description
=> "Amount of memory this NUMA node provides.",
861 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
862 description
=> "Host NUMA nodes to use.",
863 format_description
=> "id[-id];...",
868 enum
=> [qw(preferred bind interleave)],
869 description
=> "NUMA allocation policy.",
873 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
876 type
=> 'string', format
=> $numa_fmt,
877 description
=> "NUMA topology.",
879 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
881 for (my $i = 0; $i < $MAX_NUMA; $i++) {
882 $confdesc->{"numa$i"} = $numadesc;
885 my $nic_model_list = [
901 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
903 my $net_fmt_bridge_descr = <<__EOD__;
904 Bridge to attach the network device to. The Proxmox VE standard bridge
907 If you do not specify a bridge, we create a kvm user (NATed) network
908 device, which provides DHCP and DNS services. The following addresses
915 The DHCP server assign addresses to the guest starting from 10.0.2.15.
919 macaddr
=> get_standard_option
('mac-addr', {
920 description
=> "MAC address. That address must be unique withing your network. This is"
921 ." automatically generated if not specified.",
925 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
926 ." very low CPU overhead. If your guest does not support this driver, it is usually"
927 ." best to use 'e1000'.",
928 enum
=> $nic_model_list,
931 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
932 bridge
=> get_standard_option
('pve-bridge-id', {
933 description
=> $net_fmt_bridge_descr,
938 minimum
=> 0, maximum
=> 64,
939 description
=> 'Number of packet queues to be used on the device.',
945 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
950 minimum
=> 1, maximum
=> 4094,
951 description
=> 'VLAN tag to apply to packets on this interface.',
956 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
957 description
=> 'VLAN trunks to pass through this interface.',
958 format_description
=> 'vlanid[;vlanid...]',
963 description
=> 'Whether this interface should be protected by the firewall.',
968 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
973 minimum
=> 1, maximum
=> 65520,
974 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
981 type
=> 'string', format
=> $net_fmt,
982 description
=> "Specify network devices.",
985 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
990 format
=> 'pve-ipv4-config',
991 format_description
=> 'IPv4Format/CIDR',
992 description
=> 'IPv4 address in CIDR format.',
999 format_description
=> 'GatewayIPv4',
1000 description
=> 'Default gateway for IPv4 traffic.',
1006 format
=> 'pve-ipv6-config',
1007 format_description
=> 'IPv6Format/CIDR',
1008 description
=> 'IPv6 address in CIDR format.',
1015 format_description
=> 'GatewayIPv6',
1016 description
=> 'Default gateway for IPv6 traffic.',
1021 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
1022 my $ipconfigdesc = {
1024 type
=> 'string', format
=> 'pve-qm-ipconfig',
1025 description
=> <<'EODESCR',
1026 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1028 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1030 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1031 gateway should be provided.
1032 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1033 cloud-init 19.4 or newer.
1035 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1039 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
1041 for (my $i = 0; $i < $MAX_NETS; $i++) {
1042 $confdesc->{"net$i"} = $netdesc;
1043 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1046 foreach my $key (keys %$confdesc_cloudinit) {
1047 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1050 PVE
::JSONSchema
::register_format
('pve-cpuset', \
&pve_verify_cpuset
);
1051 sub pve_verify_cpuset
{
1052 my ($set_text, $noerr) = @_;
1054 my ($count, $members) = eval { PVE
::CpuSet
::parse_cpuset
($set_text) };
1058 die "unable to parse cpuset option\n";
1061 return PVE
::CpuSet-
>new($members)->short_string();
1064 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1065 sub verify_volume_id_or_qm_path
{
1066 my ($volid, $noerr) = @_;
1068 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1070 return verify_volume_id_or_absolute_path
($volid, $noerr);
1073 PVE
::JSONSchema
::register_format
('pve-volume-id-or-absolute-path', \
&verify_volume_id_or_absolute_path
);
1074 sub verify_volume_id_or_absolute_path
{
1075 my ($volid, $noerr) = @_;
1077 return $volid if $volid =~ m
|^/|;
1079 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1090 pattern
=> '(/dev/.+|socket)',
1091 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1092 verbose_description
=> <<EODESCR,
1093 Create a serial device inside the VM (n is 0 to 3), and pass through a
1094 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1095 host side (use 'qm terminal' to open a terminal connection).
1097 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1098 use with special care.
1100 CAUTION: Experimental! User reported problems with this option.
1107 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1108 description
=> "Map host parallel devices (n is 0 to 2).",
1109 verbose_description
=> <<EODESCR,
1110 Map host parallel devices (n is 0 to 2).
1112 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1113 machines - use with special care.
1115 CAUTION: Experimental! User reported problems with this option.
1119 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1120 $confdesc->{"parallel$i"} = $paralleldesc;
1123 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1124 $confdesc->{"serial$i"} = $serialdesc;
1127 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1128 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1131 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1132 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1135 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
1136 $confdesc->{"usb$i"} = $PVE::QemuServer
::USB
::usbdesc
;
1144 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1145 . " Deprecated, use 'order=' instead.",
1146 pattern
=> '[acdn]{1,4}',
1147 format_description
=> "[acdn]{1,4}",
1149 # note: this is also the fallback if boot: is not given at all
1155 format
=> 'pve-qm-bootdev-list',
1156 format_description
=> "device[;device...]",
1157 description
=> <<EODESC,
1158 The guest will attempt to boot from devices in the order they appear here.
1160 Disks, optical drives and passed-through storage USB devices will be directly
1161 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1162 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1164 Note that only devices in this list will be marked as bootable and thus loaded
1165 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1166 (e.g. software-raid), you need to specify all of them here.
1168 Overrides the deprecated 'legacy=[acdn]*' value when given.
1172 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1174 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1175 sub verify_bootdev
{
1176 my ($dev, $noerr) = @_;
1178 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1179 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && !$special;
1183 return 0 if $dev !~ m/^$base\d+$/;
1184 return 0 if !$confdesc->{$dev};
1188 return $dev if $check->("net");
1189 return $dev if $check->("usb");
1190 return $dev if $check->("hostpci");
1193 die "invalid boot device '$dev'\n";
1196 sub print_bootorder
{
1198 return "" if !@$devs;
1199 my $data = { order
=> join(';', @$devs) };
1200 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1203 my $kvm_api_version = 0;
1206 return $kvm_api_version if $kvm_api_version;
1208 open my $fh, '<', '/dev/kvm' or return;
1210 # 0xae00 => KVM_GET_API_VERSION
1211 $kvm_api_version = ioctl($fh, 0xae00, 0);
1214 return $kvm_api_version;
1217 my $kvm_user_version = {};
1220 sub kvm_user_version
{
1223 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1224 my $st = stat($binary);
1226 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1227 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1228 $cachedmtime == $st->mtime;
1230 $kvm_user_version->{$binary} = 'unknown';
1231 $kvm_mtime->{$binary} = $st->mtime;
1235 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1236 $kvm_user_version->{$binary} = $2;
1240 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1243 return $kvm_user_version->{$binary};
1246 my sub extract_version
{
1247 my ($machine_type, $version) = @_;
1248 $version = kvm_user_version
() if !defined($version);
1249 return PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1252 sub kernel_has_vhost_net
{
1253 return -c
'/dev/vhost-net';
1258 return defined($confdesc->{$key});
1262 sub get_cdrom_path
{
1264 return $cdrom_path if defined($cdrom_path);
1266 $cdrom_path = first
{ -l
$_ } map { "/dev/cdrom$_" } ('', '1', '2');
1268 if (!defined($cdrom_path)) {
1269 log_warn
("no physical CD-ROM available, ignoring");
1277 my ($storecfg, $vmid, $cdrom) = @_;
1279 if ($cdrom eq 'cdrom') {
1280 return get_cdrom_path
();
1281 } elsif ($cdrom eq 'none') {
1283 } elsif ($cdrom =~ m
|^/|) {
1286 return PVE
::Storage
::path
($storecfg, $cdrom);
1290 # try to convert old style file names to volume IDs
1291 sub filename_to_volume_id
{
1292 my ($vmid, $file, $media) = @_;
1294 if (!($file eq 'none' || $file eq 'cdrom' ||
1295 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1297 return if $file =~ m
|/|;
1299 if ($media && $media eq 'cdrom') {
1300 $file = "local:iso/$file";
1302 $file = "local:$vmid/$file";
1309 sub verify_media_type
{
1310 my ($opt, $vtype, $media) = @_;
1315 if ($media eq 'disk') {
1317 } elsif ($media eq 'cdrom') {
1320 die "internal error";
1323 return if ($vtype eq $etype);
1325 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1328 sub cleanup_drive_path
{
1329 my ($opt, $storecfg, $drive) = @_;
1331 # try to convert filesystem paths to volume IDs
1333 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1334 ($drive->{file
} !~ m
|^/dev/.+|) &&
1335 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1336 ($drive->{file
} !~ m/^\d+$/)) {
1337 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1338 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1340 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1341 verify_media_type
($opt, $vtype, $drive->{media
});
1342 $drive->{file
} = $volid;
1345 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1348 sub parse_hotplug_features
{
1353 return $res if $data eq '0';
1355 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1357 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1358 if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1361 die "invalid hotplug feature '$feature'\n";
1367 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1368 sub pve_verify_hotplug_features
{
1369 my ($value, $noerr) = @_;
1371 return $value if parse_hotplug_features
($value);
1375 die "unable to parse hotplug option\n";
1379 my($fh, $noerr) = @_;
1382 my $SG_GET_VERSION_NUM = 0x2282;
1384 my $versionbuf = "\x00" x
8;
1385 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1387 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1390 my $version = unpack("I", $versionbuf);
1391 if ($version < 30000) {
1392 die "scsi generic interface too old\n" if !$noerr;
1396 my $buf = "\x00" x
36;
1397 my $sensebuf = "\x00" x
8;
1398 my $cmd = pack("C x3 C x1", 0x12, 36);
1400 # see /usr/include/scsi/sg.h
1401 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1404 $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1407 $ret = ioctl($fh, $SG_IO, $packet);
1409 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1413 my @res = unpack($sg_io_hdr_t, $packet);
1414 if ($res[17] || $res[18]) {
1415 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1420 $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1422 $res->{removable
} = $res->{removable
} & 128 ?
1 : 0;
1423 $res->{type
} &= 0x1F;
1431 my $fh = IO
::File-
>new("+<$path") || return;
1432 my $res = scsi_inquiry
($fh, 1);
1438 sub print_tabletdevice_full
{
1439 my ($conf, $arch) = @_;
1441 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1443 # we use uhci for old VMs because tablet driver was buggy in older qemu
1445 if ($q35 || $arch eq 'aarch64') {
1451 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1454 sub print_keyboarddevice_full
{
1455 my ($conf, $arch) = @_;
1457 return if $arch ne 'aarch64';
1459 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1462 my sub get_drive_id
{
1464 return "$drive->{interface}$drive->{index}";
1467 sub print_drivedevice_full
{
1468 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1473 my $drive_id = get_drive_id
($drive);
1474 if ($drive->{interface
} eq 'virtio') {
1475 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1476 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1477 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1478 } elsif ($drive->{interface
} eq 'scsi') {
1480 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1481 my $unit = $drive->{index} % $maxdev;
1482 my $devicetype = 'hd';
1484 if (drive_is_cdrom
($drive)) {
1487 if ($drive->{file
} =~ m
|^/|) {
1488 $path = $drive->{file
};
1489 if (my $info = path_is_scsi
($path)) {
1490 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1491 $devicetype = 'block';
1492 } elsif ($info->{type
} == 1) { # tape
1493 $devicetype = 'generic';
1497 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1500 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1501 my $version = extract_version
($machine_type, kvm_user_version
());
1502 if ($path =~ m/^iscsi\:\/\
// &&
1503 !min_version
($version, 4, 1)) {
1504 $devicetype = 'generic';
1508 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1509 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1511 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1512 .",lun=$drive->{index}";
1514 $device .= ",drive=drive-$drive_id,id=$drive_id";
1516 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1517 $device .= ",rotation_rate=1";
1519 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1521 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1522 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1523 my $controller = int($drive->{index} / $maxdev);
1524 my $unit = $drive->{index} % $maxdev;
1526 # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1527 # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1528 # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1529 # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1530 # were before. Move odd ones up by 2 where they don't clash.
1531 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) && $drive->{interface
} eq 'ide') {
1532 $controller += 2 * ($unit % 2);
1536 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1538 $device = "ide-$devicetype";
1539 if ($drive->{interface
} eq 'ide') {
1540 $device .= ",bus=ide.$controller,unit=$unit";
1542 $device .= ",bus=ahci$controller.$unit";
1544 $device .= ",drive=drive-$drive_id,id=$drive_id";
1546 if ($devicetype eq 'hd') {
1547 if (my $model = $drive->{model
}) {
1548 $model = URI
::Escape
::uri_unescape
($model);
1549 $device .= ",model=$model";
1551 if ($drive->{ssd
}) {
1552 $device .= ",rotation_rate=1";
1555 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1556 } elsif ($drive->{interface
} eq 'usb') {
1558 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1560 die "unsupported interface type";
1563 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1565 if (my $serial = $drive->{serial
}) {
1566 $serial = URI
::Escape
::uri_unescape
($serial);
1567 $device .= ",serial=$serial";
1574 sub get_initiator_name
{
1577 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1578 while (defined(my $line = <$fh>)) {
1579 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1588 my sub storage_allows_io_uring_default
{
1589 my ($scfg, $cache_direct) = @_;
1591 # io_uring with cache mode writeback or writethrough on krbd will hang...
1592 return if $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1594 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1595 # sometimes, just plain disable...
1596 return if $scfg && $scfg->{type
} eq 'lvm';
1598 # io_uring causes problems when used with CIFS since kernel 5.15
1599 # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1600 return if $scfg && $scfg->{type
} eq 'cifs';
1605 my sub drive_uses_cache_direct
{
1606 my ($drive, $scfg) = @_;
1608 my $cache_direct = 0;
1610 if (my $cache = $drive->{cache
}) {
1611 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1612 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1616 return $cache_direct;
1619 sub print_drive_commandline_full
{
1620 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1623 my $volid = $drive->{file
};
1624 my $format = $drive->{format
};
1625 my $drive_id = get_drive_id
($drive);
1627 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1628 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1630 if (drive_is_cdrom
($drive)) {
1631 $path = get_iso_path
($storecfg, $vmid, $volid);
1632 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1635 $path = PVE
::Storage
::path
($storecfg, $volid);
1636 $format //= qemu_img_format
($scfg, $volname);
1643 my $is_rbd = $path =~ m/^rbd:/;
1646 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1647 foreach my $o (@qemu_drive_options) {
1648 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1651 # snapshot only accepts on|off
1652 if (defined($drive->{snapshot
})) {
1653 my $v = $drive->{snapshot
} ?
'on' : 'off';
1654 $opts .= ",snapshot=$v";
1657 if (defined($drive->{ro
})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1658 $opts .= ",readonly=" . ($drive->{ro
} ?
'on' : 'off');
1661 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1662 my ($dir, $qmpname) = @$type;
1663 if (my $v = $drive->{"mbps$dir"}) {
1664 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1666 if (my $v = $drive->{"mbps${dir}_max"}) {
1667 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1669 if (my $v = $drive->{"bps${dir}_max_length"}) {
1670 $opts .= ",throttling.bps$qmpname-max-length=$v";
1672 if (my $v = $drive->{"iops${dir}"}) {
1673 $opts .= ",throttling.iops$qmpname=$v";
1675 if (my $v = $drive->{"iops${dir}_max"}) {
1676 $opts .= ",throttling.iops$qmpname-max=$v";
1678 if (my $v = $drive->{"iops${dir}_max_length"}) {
1679 $opts .= ",throttling.iops$qmpname-max-length=$v";
1684 $format = "rbd" if $is_rbd;
1685 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1687 $opts .= ",format=alloc-track,file.driver=$format";
1689 $opts .= ",format=$format";
1692 my $cache_direct = drive_uses_cache_direct
($drive, $scfg);
1694 $opts .= ",cache=none" if !$drive->{cache
} && $cache_direct;
1696 if (!$drive->{aio
}) {
1697 if ($io_uring && storage_allows_io_uring_default
($scfg, $cache_direct)) {
1698 # io_uring supports all cache modes
1699 $opts .= ",aio=io_uring";
1701 # aio native works only with O_DIRECT
1703 $opts .= ",aio=native";
1705 $opts .= ",aio=threads";
1710 if (!drive_is_cdrom
($drive)) {
1712 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1713 $detectzeroes = 'off';
1714 } elsif ($drive->{discard
}) {
1715 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1717 # This used to be our default with discard not being specified:
1718 $detectzeroes = 'on';
1721 # note: 'detect-zeroes' works per blockdev and we want it to persist
1722 # after the alloc-track is removed, so put it on 'file' directly
1723 my $dz_param = $pbs_name ?
"file.detect-zeroes" : "detect-zeroes";
1724 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1728 $opts .= ",backing=$pbs_name";
1729 $opts .= ",auto-remove=on";
1732 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1733 my $file_param = "file";
1735 # non-rbd drivers require the underlying file to be a seperate block
1736 # node, so add a second .file indirection
1737 $file_param .= ".file" if !$is_rbd;
1738 $file_param .= ".filename";
1740 my $pathinfo = $path ?
"$file_param=$path," : '';
1742 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1745 sub print_pbs_blockdev
{
1746 my ($pbs_conf, $pbs_name) = @_;
1747 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1748 $blockdev .= ",repository=$pbs_conf->{repository}";
1749 $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace
};
1750 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1751 $blockdev .= ",archive=$pbs_conf->{archive}";
1752 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1756 sub print_netdevice_full
{
1757 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1759 my $device = $net->{model
};
1760 if ($net->{model
} eq 'virtio') {
1761 $device = 'virtio-net-pci';
1764 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1765 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1766 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1767 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1768 # and out of each queue plus one config interrupt and control vector queue
1769 my $vectors = $net->{queues
} * 2 + 2;
1770 $tmpstr .= ",vectors=$vectors,mq=on";
1771 if (min_version
($machine_version, 7, 1)) {
1772 $tmpstr .= ",packed=on";
1776 if (min_version
($machine_version, 7, 1) && $net->{model
} eq 'virtio'){
1777 $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1780 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1782 if (my $mtu = $net->{mtu
}) {
1783 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1784 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1787 } elsif ($mtu < 576) {
1788 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1789 } elsif ($mtu > $bridge_mtu) {
1790 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1792 $tmpstr .= ",host_mtu=$mtu";
1794 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1798 if ($use_old_bios_files) {
1800 if ($device eq 'virtio-net-pci') {
1801 $romfile = 'pxe-virtio.rom';
1802 } elsif ($device eq 'e1000') {
1803 $romfile = 'pxe-e1000.rom';
1804 } elsif ($device eq 'e1000e') {
1805 $romfile = 'pxe-e1000e.rom';
1806 } elsif ($device eq 'ne2k') {
1807 $romfile = 'pxe-ne2k_pci.rom';
1808 } elsif ($device eq 'pcnet') {
1809 $romfile = 'pxe-pcnet.rom';
1810 } elsif ($device eq 'rtl8139') {
1811 $romfile = 'pxe-rtl8139.rom';
1813 $tmpstr .= ",romfile=$romfile" if $romfile;
1819 sub print_netdev_full
{
1820 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1823 if ($netid =~ m/^net(\d+)$/) {
1827 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1829 my $ifname = "tap${vmid}i$i";
1831 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1832 die "interface name '$ifname' is too long (max 15 character)\n"
1833 if length($ifname) >= 16;
1835 my $vhostparam = '';
1836 if (is_native
($arch)) {
1837 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1840 my $vmname = $conf->{name
} || "vm$vmid";
1843 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1845 if ($net->{bridge
}) {
1846 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1847 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1849 $netdev = "type=user,id=$netid,hostname=$vmname";
1852 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1858 'cirrus' => 'cirrus-vga',
1860 'vmware' => 'vmware-svga',
1861 'virtio' => 'virtio-vga',
1862 'virtio-gl' => 'virtio-vga-gl',
1865 sub print_vga_device
{
1866 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1868 my $type = $vga_map->{$vga->{type
}};
1869 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1870 $type = 'virtio-gpu';
1872 my $vgamem_mb = $vga->{memory
};
1874 my $max_outputs = '';
1876 $type = $id ?
'qxl' : 'qxl-vga';
1878 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1879 # set max outputs so linux can have up to 4 qxl displays with one device
1880 if (min_version
($machine_version, 4, 1)) {
1881 $max_outputs = ",max_outputs=4";
1886 die "no devicetype for $vga->{type}\n" if !$type;
1890 if ($vga->{type
} =~ /^virtio/) {
1891 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1892 $memory = ",max_hostmem=$bytes";
1894 # from https://www.spice-space.org/multiple-monitors.html
1895 $memory = ",vgamem_mb=$vga->{memory}";
1896 my $ram = $vgamem_mb * 4;
1897 my $vram = $vgamem_mb * 2;
1898 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1900 $memory = ",vgamem_mb=$vga->{memory}";
1902 } elsif ($qxlnum && $id) {
1903 $memory = ",ram_size=67108864,vram_size=33554432";
1907 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1908 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1911 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1912 my $vgaid = "vga" . ($id // '');
1914 if ($q35 && $vgaid eq 'vga') {
1915 # the first display uses pcie.0 bus on q35 machines
1916 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1918 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1921 if ($vga->{type
} eq 'virtio-gl') {
1922 my $base = '/usr/lib/x86_64-linux-gnu/lib';
1923 die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1924 if !-e
"${base}EGL.so.1" || !-e
"${base}GL.so.1";
1926 die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1927 if !PVE
::Tools
::dir_glob_regex
('/dev/dri/', "renderD.*");
1930 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1933 sub parse_number_sets
{
1936 foreach my $part (split(/;/, $set)) {
1937 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1938 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1939 push @$res, [ $1, $2 ];
1941 die "invalid range: $part\n";
1950 my $res = parse_property_string
($numa_fmt, $data);
1951 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1952 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1956 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1958 my ($data, $disable_mac_autogen) = @_;
1960 my $res = eval { parse_property_string
($net_fmt, $data) };
1965 if (!defined($res->{macaddr
}) && !$disable_mac_autogen) {
1966 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1967 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1972 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1973 sub parse_ipconfig
{
1976 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1982 if ($res->{gw
} && !$res->{ip
}) {
1983 warn 'gateway specified without specifying an IP address';
1986 if ($res->{gw6
} && !$res->{ip6
}) {
1987 warn 'IPv6 gateway specified without specifying an IPv6 address';
1990 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1991 warn 'gateway specified together with DHCP';
1994 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1996 warn "IPv6 gateway specified together with $res->{ip6} address";
2000 if (!$res->{ip
} && !$res->{ip6
}) {
2001 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
2010 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
2013 sub add_random_macs
{
2014 my ($settings) = @_;
2016 foreach my $opt (keys %$settings) {
2017 next if $opt !~ m/^net(\d+)$/;
2018 my $net = parse_net
($settings->{$opt});
2020 $settings->{$opt} = print_net
($net);
2024 sub vm_is_volid_owner
{
2025 my ($storecfg, $vmid, $volid) = @_;
2027 if ($volid !~ m
|^/|) {
2029 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
2030 if ($owner && ($owner == $vmid)) {
2038 sub vmconfig_register_unused_drive
{
2039 my ($storecfg, $vmid, $conf, $drive) = @_;
2041 if (drive_is_cloudinit
($drive)) {
2042 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2044 delete $conf->{cloudinit
};
2045 } elsif (!drive_is_cdrom
($drive)) {
2046 my $volid = $drive->{file
};
2047 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2048 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2053 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2057 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2058 format_description
=> 'UUID',
2059 description
=> "Set SMBIOS1 UUID.",
2064 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2065 format_description
=> 'Base64 encoded string',
2066 description
=> "Set SMBIOS1 version.",
2071 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2072 format_description
=> 'Base64 encoded string',
2073 description
=> "Set SMBIOS1 serial number.",
2078 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2079 format_description
=> 'Base64 encoded string',
2080 description
=> "Set SMBIOS1 manufacturer.",
2085 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2086 format_description
=> 'Base64 encoded string',
2087 description
=> "Set SMBIOS1 product ID.",
2092 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2093 format_description
=> 'Base64 encoded string',
2094 description
=> "Set SMBIOS1 SKU string.",
2099 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2100 format_description
=> 'Base64 encoded string',
2101 description
=> "Set SMBIOS1 family string.",
2106 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2114 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2121 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2124 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2126 sub parse_watchdog
{
2131 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2136 sub parse_guest_agent
{
2139 return {} if !defined($conf->{agent
});
2141 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2144 # if the agent is disabled ignore the other potentially set properties
2145 return {} if !$res->{enabled
};
2150 my ($conf, $key) = @_;
2151 return undef if !defined($conf->{agent
});
2153 my $agent = parse_guest_agent
($conf);
2154 return $agent->{$key};
2160 return {} if !$value;
2161 my $res = eval { parse_property_string
($vga_fmt, $value) };
2171 my $res = eval { parse_property_string
($rng_fmt, $value) };
2176 sub parse_meta_info
{
2181 my $res = eval { parse_property_string
($meta_info_fmt, $value) };
2186 sub new_meta_info_string
{
2187 my () = @_; # for now do not allow to override any value
2189 return PVE
::JSONSchema
::print_property_string
(
2191 'creation-qemu' => kvm_user_version
(),
2192 ctime
=> "". int(time()),
2198 sub qemu_created_version_fixups
{
2199 my ($conf, $forcemachine, $kvmver) = @_;
2201 my $meta = parse_meta_info
($conf->{meta
}) // {};
2202 my $forced_vers = PVE
::QemuServer
::Machine
::extract_version
($forcemachine);
2204 # check if we need to apply some handling for VMs that always use the latest machine version but
2205 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2206 # would be required (we do not want to pin machine version for non-windows OS type)
2208 (!defined($conf->{machine
}) || $conf->{machine
} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2209 && (!defined($meta->{'creation-qemu'}) || !min_version
($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2210 && (!$forced_vers || min_version
($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2211 && min_version
($kvmver, 6, 1) # only need to apply the change since 6.1
2213 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2214 if ($q35 && $conf->{ostype
} && $conf->{ostype
} eq 'l26') {
2215 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2216 # and thus with the predictable interface naming of systemd
2217 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2223 # add JSON properties for create and set function
2224 sub json_config_properties
{
2225 my ($prop, $with_disk_alloc) = @_;
2227 my $skip_json_config_opts = {
2231 runningmachine
=> 1,
2236 foreach my $opt (keys %$confdesc) {
2237 next if $skip_json_config_opts->{$opt};
2239 if ($with_disk_alloc && is_valid_drivename
($opt)) {
2240 $prop->{$opt} = $PVE::QemuServer
::Drive
::drivedesc_hash_with_alloc-
>{$opt};
2242 $prop->{$opt} = $confdesc->{$opt};
2249 # Properties that we can read from an OVF file
2250 sub json_ovf_properties
{
2253 for my $device (PVE
::QemuServer
::Drive
::valid_drive_names
()) {
2254 $prop->{$device} = {
2256 format
=> 'pve-volume-id-or-absolute-path',
2257 description
=> "Disk image that gets imported to $device",
2264 description
=> "The number of CPU cores.",
2269 description
=> "Amount of RAM for the VM in MB.",
2274 description
=> "Name of the VM.",
2281 # return copy of $confdesc_cloudinit to generate documentation
2282 sub cloudinit_config_properties
{
2284 return dclone
($confdesc_cloudinit);
2287 sub cloudinit_pending_properties
{
2289 map { $_ => 1 } keys $confdesc_cloudinit->%*,
2292 $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2297 my ($key, $value) = @_;
2299 die "unknown setting '$key'\n" if !$confdesc->{$key};
2301 my $type = $confdesc->{$key}->{type
};
2303 if (!defined($value)) {
2304 die "got undefined value\n";
2307 if ($value =~ m/[\n\r]/) {
2308 die "property contains a line feed\n";
2311 if ($type eq 'boolean') {
2312 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2313 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2314 die "type check ('boolean') failed - got '$value'\n";
2315 } elsif ($type eq 'integer') {
2316 return int($1) if $value =~ m/^(\d+)$/;
2317 die "type check ('integer') failed - got '$value'\n";
2318 } elsif ($type eq 'number') {
2319 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2320 die "type check ('number') failed - got '$value'\n";
2321 } elsif ($type eq 'string') {
2322 if (my $fmt = $confdesc->{$key}->{format
}) {
2323 PVE
::JSONSchema
::check_format
($fmt, $value);
2326 $value =~ s/^\"(.*)\"$/$1/;
2329 die "internal error"
2334 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2336 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2338 if (!$skiplock && !PVE
::QemuConfig-
>has_lock($conf, 'suspended')) {
2339 PVE
::QemuConfig-
>check_lock($conf);
2342 if ($conf->{template
}) {
2343 # check if any base image is still used by a linked clone
2344 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2345 my ($ds, $drive) = @_;
2346 return if drive_is_cdrom
($drive);
2348 my $volid = $drive->{file
};
2349 return if !$volid || $volid =~ m
|^/|;
2351 die "base volume '$volid' is still in use by linked cloned\n"
2352 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2358 my $remove_owned_drive = sub {
2359 my ($ds, $drive) = @_;
2360 return if drive_is_cdrom
($drive, 1);
2362 my $volid = $drive->{file
};
2363 return if !$volid || $volid =~ m
|^/|;
2364 return if $volids->{$volid};
2366 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2367 return if !$path || !$owner || ($owner != $vmid);
2369 $volids->{$volid} = 1;
2370 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2371 warn "Could not remove disk '$volid', check manually: $@" if $@;
2374 # only remove disks owned by this VM (referenced in the config)
2375 my $include_opts = {
2376 include_unused
=> 1,
2377 extra_keys
=> ['vmstate'],
2379 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2381 for my $snap (values %{$conf->{snapshots
}}) {
2382 next if !defined($snap->{vmstate
});
2383 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2384 next if !defined($drive);
2385 $remove_owned_drive->('vmstate', $drive);
2388 PVE
::QemuConfig-
>foreach_volume_full($conf->{pending
}, $include_opts, $remove_owned_drive);
2390 if ($purge_unreferenced) { # also remove unreferenced disk
2391 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2392 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2393 my ($volid, $sid, $volname, $d) = @_;
2394 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2399 if (defined $replacement_conf) {
2400 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2402 PVE
::QemuConfig-
>destroy_config($vmid);
2406 sub parse_vm_config
{
2407 my ($filename, $raw, $strict) = @_;
2409 return if !defined($raw);
2412 digest
=> Digest
::SHA
::sha1_hex
($raw),
2418 my $handle_error = sub {
2428 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2429 || die "got strange filename '$filename'";
2435 my $finish_description = sub {
2436 if (defined($descr)) {
2438 $conf->{description
} = $descr;
2444 my @lines = split(/\n/, $raw);
2445 foreach my $line (@lines) {
2446 next if $line =~ m/^\s*$/;
2448 if ($line =~ m/^\[PENDING\]\s*$/i) {
2449 $section = 'pending';
2450 $finish_description->();
2451 $conf = $res->{$section} = {};
2453 } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2454 $section = 'cloudinit';
2455 $finish_description->();
2456 $conf = $res->{$section} = {};
2459 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2461 $finish_description->();
2462 $conf = $res->{snapshots
}->{$section} = {};
2466 if ($line =~ m/^\#(.*)$/) {
2467 $descr = '' if !defined($descr);
2468 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2472 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2473 $descr = '' if !defined($descr);
2474 $descr .= PVE
::Tools
::decode_text
($2);
2475 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2476 $conf->{snapstate
} = $1;
2477 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2480 $conf->{$key} = $value;
2481 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2483 if ($section eq 'pending') {
2484 $conf->{delete} = $value; # we parse this later
2486 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2488 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2491 if ($section eq 'cloudinit') {
2492 # ignore validation only used for informative purpose
2493 $conf->{$key} = $value;
2496 eval { $value = check_type
($key, $value); };
2498 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2500 $key = 'ide2' if $key eq 'cdrom';
2501 my $fmt = $confdesc->{$key}->{format
};
2502 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2503 my $v = parse_drive
($key, $value);
2504 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2505 $v->{file
} = $volid;
2506 $value = print_drive
($v);
2508 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2513 $conf->{$key} = $value;
2516 $handle_error->("vm $vmid - unable to parse config: $line\n");
2520 $finish_description->();
2521 delete $res->{snapstate
}; # just to be sure
2526 sub write_vm_config
{
2527 my ($filename, $conf) = @_;
2529 delete $conf->{snapstate
}; # just to be sure
2531 if ($conf->{cdrom
}) {
2532 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2533 $conf->{ide2
} = $conf->{cdrom
};
2534 delete $conf->{cdrom
};
2537 # we do not use 'smp' any longer
2538 if ($conf->{sockets
}) {
2539 delete $conf->{smp
};
2540 } elsif ($conf->{smp
}) {
2541 $conf->{sockets
} = $conf->{smp
};
2542 delete $conf->{cores
};
2543 delete $conf->{smp
};
2546 my $used_volids = {};
2548 my $cleanup_config = sub {
2549 my ($cref, $pending, $snapname) = @_;
2551 foreach my $key (keys %$cref) {
2552 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2553 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2554 my $value = $cref->{$key};
2555 if ($key eq 'delete') {
2556 die "propertry 'delete' is only allowed in [PENDING]\n"
2558 # fixme: check syntax?
2561 eval { $value = check_type
($key, $value); };
2562 die "unable to parse value of '$key' - $@" if $@;
2564 $cref->{$key} = $value;
2566 if (!$snapname && is_valid_drivename
($key)) {
2567 my $drive = parse_drive
($key, $value);
2568 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2573 &$cleanup_config($conf);
2575 &$cleanup_config($conf->{pending
}, 1);
2577 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2578 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2579 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2582 # remove 'unusedX' settings if we re-add a volume
2583 foreach my $key (keys %$conf) {
2584 my $value = $conf->{$key};
2585 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2586 delete $conf->{$key};
2590 my $generate_raw_config = sub {
2591 my ($conf, $pending) = @_;
2595 # add description as comment to top of file
2596 if (defined(my $descr = $conf->{description
})) {
2598 foreach my $cl (split(/\n/, $descr)) {
2599 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2602 $raw .= "#\n" if $pending;
2606 foreach my $key (sort keys %$conf) {
2607 next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2608 $raw .= "$key: $conf->{$key}\n";
2613 my $raw = &$generate_raw_config($conf);
2615 if (scalar(keys %{$conf->{pending
}})){
2616 $raw .= "\n[PENDING]\n";
2617 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2620 if (scalar(keys %{$conf->{cloudinit
}}) && PVE
::QemuConfig-
>has_cloudinit($conf)){
2621 $raw .= "\n[special:cloudinit]\n";
2622 $raw .= &$generate_raw_config($conf->{cloudinit
});
2625 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2626 $raw .= "\n[$snapname]\n";
2627 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2637 # we use static defaults from our JSON schema configuration
2638 foreach my $key (keys %$confdesc) {
2639 if (defined(my $default = $confdesc->{$key}->{default})) {
2640 $res->{$key} = $default;
2648 my $vmlist = PVE
::Cluster
::get_vmlist
();
2650 return $res if !$vmlist || !$vmlist->{ids
};
2651 my $ids = $vmlist->{ids
};
2652 my $nodename = nodename
();
2654 foreach my $vmid (keys %$ids) {
2655 my $d = $ids->{$vmid};
2656 next if !$d->{node
} || $d->{node
} ne $nodename;
2657 next if !$d->{type
} || $d->{type
} ne 'qemu';
2658 $res->{$vmid}->{exists} = 1;
2663 # test if VM uses local resources (to prevent migration)
2664 sub check_local_resources
{
2665 my ($conf, $noerr) = @_;
2668 my $mapped_res = [];
2670 my $nodelist = PVE
::Cluster
::get_nodelist
();
2671 my $pci_map = PVE
::Mapping
::PCI
::config
();
2672 my $usb_map = PVE
::Mapping
::USB
::config
();
2674 my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2676 my $add_missing_mapping = sub {
2677 my ($type, $key, $id) = @_;
2678 for my $node (@$nodelist) {
2680 if ($type eq 'pci') {
2681 $entry = PVE
::Mapping
::PCI
::get_node_mapping
($pci_map, $id, $node);
2682 } elsif ($type eq 'usb') {
2683 $entry = PVE
::Mapping
::USB
::get_node_mapping
($usb_map, $id, $node);
2685 if (!scalar($entry->@*)) {
2686 push @{$missing_mappings_by_node->{$node}}, $key;
2691 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2692 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2694 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2696 foreach my $k (keys %$conf) {
2697 if ($k =~ m/^usb/) {
2698 my $entry = parse_property_string
('pve-qm-usb', $conf->{$k});
2699 next if $entry->{host
} =~ m/^spice$/i;
2700 if ($entry->{mapping
}) {
2701 $add_missing_mapping->('usb', $k, $entry->{mapping
});
2702 push @$mapped_res, $k;
2705 if ($k =~ m/^hostpci/) {
2706 my $entry = parse_property_string
('pve-qm-hostpci', $conf->{$k});
2707 if ($entry->{mapping
}) {
2708 $add_missing_mapping->('pci', $k, $entry->{mapping
});
2709 push @$mapped_res, $k;
2712 # sockets are safe: they will recreated be on the target side post-migrate
2713 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2714 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2717 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2719 return wantarray ?
(\
@loc_res, $mapped_res, $missing_mappings_by_node) : \
@loc_res;
2722 # check if used storages are available on all nodes (use by migrate)
2723 sub check_storage_availability
{
2724 my ($storecfg, $conf, $node) = @_;
2726 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2727 my ($ds, $drive) = @_;
2729 my $volid = $drive->{file
};
2732 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2735 # check if storage is available on both nodes
2736 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2737 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2739 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2741 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2742 if !$scfg->{content
}->{$vtype};
2746 # list nodes where all VM images are available (used by has_feature API)
2748 my ($conf, $storecfg) = @_;
2750 my $nodelist = PVE
::Cluster
::get_nodelist
();
2751 my $nodehash = { map { $_ => 1 } @$nodelist };
2752 my $nodename = nodename
();
2754 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2755 my ($ds, $drive) = @_;
2757 my $volid = $drive->{file
};
2760 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2762 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2763 if ($scfg->{disable
}) {
2765 } elsif (my $avail = $scfg->{nodes
}) {
2766 foreach my $node (keys %$nodehash) {
2767 delete $nodehash->{$node} if !$avail->{$node};
2769 } elsif (!$scfg->{shared
}) {
2770 foreach my $node (keys %$nodehash) {
2771 delete $nodehash->{$node} if $node ne $nodename
2780 sub check_local_storage_availability
{
2781 my ($conf, $storecfg) = @_;
2783 my $nodelist = PVE
::Cluster
::get_nodelist
();
2784 my $nodehash = { map { $_ => {} } @$nodelist };
2786 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2787 my ($ds, $drive) = @_;
2789 my $volid = $drive->{file
};
2792 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2794 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2796 if ($scfg->{disable
}) {
2797 foreach my $node (keys %$nodehash) {
2798 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2800 } elsif (my $avail = $scfg->{nodes
}) {
2801 foreach my $node (keys %$nodehash) {
2802 if (!$avail->{$node}) {
2803 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2810 foreach my $node (values %$nodehash) {
2811 if (my $unavail = $node->{unavailable_storages
}) {
2812 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2819 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2821 my ($vmid, $nocheck, $node) = @_;
2823 # $nocheck is set when called during a migration, in which case the config
2824 # file might still or already reside on the *other* node
2825 # - because rename has already happened, and current node is source
2826 # - because rename hasn't happened yet, and current node is target
2827 # - because rename has happened, current node is target, but hasn't yet
2829 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2830 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2835 my $vzlist = config_list
();
2837 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2839 while (defined(my $de = $fd->read)) {
2840 next if $de !~ m/^(\d+)\.pid$/;
2842 next if !defined($vzlist->{$vmid});
2843 if (my $pid = check_running
($vmid)) {
2844 $vzlist->{$vmid}->{pid
} = $pid;
2851 our $vmstatus_return_properties = {
2852 vmid
=> get_standard_option
('pve-vmid'),
2854 description
=> "QEMU process status.",
2856 enum
=> ['stopped', 'running'],
2859 description
=> "Maximum memory in bytes.",
2862 renderer
=> 'bytes',
2865 description
=> "Root disk size in bytes.",
2868 renderer
=> 'bytes',
2871 description
=> "VM name.",
2876 description
=> "VM run state from the 'query-status' QMP monitor command.",
2881 description
=> "PID of running qemu process.",
2886 description
=> "Uptime.",
2889 renderer
=> 'duration',
2892 description
=> "Maximum usable CPUs.",
2897 description
=> "The current config lock, if any.",
2902 description
=> "The current configured tags, if any",
2906 'running-machine' => {
2907 description
=> "The currently running machine type (if running).",
2912 description
=> "The currently running QEMU version (if running).",
2918 my $last_proc_pid_stat;
2920 # get VM status information
2921 # This must be fast and should not block ($full == false)
2922 # We only query KVM using QMP if $full == true (this can be slow)
2924 my ($opt_vmid, $full) = @_;
2928 my $storecfg = PVE
::Storage
::config
();
2930 my $list = vzlist
();
2931 my $defaults = load_defaults
();
2933 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2935 my $cpucount = $cpuinfo->{cpus
} || 1;
2937 foreach my $vmid (keys %$list) {
2938 next if $opt_vmid && ($vmid ne $opt_vmid);
2940 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2942 my $d = { vmid
=> int($vmid) };
2943 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2945 # fixme: better status?
2946 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2948 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2949 if (defined($size)) {
2950 $d->{disk
} = 0; # no info available
2951 $d->{maxdisk
} = $size;
2957 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2958 * ($conf->{cores
} || $defaults->{cores
});
2959 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2960 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2962 $d->{name
} = $conf->{name
} || "VM $vmid";
2963 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2964 : $defaults->{memory
}*(1024*1024);
2966 if ($conf->{balloon
}) {
2967 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2968 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2969 : $defaults->{shares
};
2980 $d->{diskwrite
} = 0;
2982 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2984 $d->{serial
} = 1 if conf_has_serial
($conf);
2985 $d->{lock} = $conf->{lock} if $conf->{lock};
2986 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2991 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2992 foreach my $dev (keys %$netdev) {
2993 next if $dev !~ m/^tap([1-9]\d*)i/;
2995 my $d = $res->{$vmid};
2998 $d->{netout
} += $netdev->{$dev}->{receive
};
2999 $d->{netin
} += $netdev->{$dev}->{transmit
};
3002 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
3003 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
3008 my $ctime = gettimeofday
;
3010 foreach my $vmid (keys %$list) {
3012 my $d = $res->{$vmid};
3013 my $pid = $d->{pid
};
3016 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
3017 next if !$pstat; # not running
3019 my $used = $pstat->{utime} + $pstat->{stime
};
3021 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
3023 if ($pstat->{vsize
}) {
3024 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
3027 my $old = $last_proc_pid_stat->{$pid};
3029 $last_proc_pid_stat->{$pid} = {
3037 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
3039 if ($dtime > 1000) {
3040 my $dutime = $used - $old->{used
};
3042 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
3043 $last_proc_pid_stat->{$pid} = {
3049 $d->{cpu
} = $old->{cpu
};
3053 return $res if !$full;
3055 my $qmpclient = PVE
::QMPClient-
>new();
3057 my $ballooncb = sub {
3058 my ($vmid, $resp) = @_;
3060 my $info = $resp->{'return'};
3061 return if !$info->{max_mem
};
3063 my $d = $res->{$vmid};
3065 # use memory assigned to VM
3066 $d->{maxmem
} = $info->{max_mem
};
3067 $d->{balloon
} = $info->{actual
};
3069 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
3070 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
3071 $d->{freemem
} = $info->{free_mem
};
3074 $d->{ballooninfo
} = $info;
3077 my $blockstatscb = sub {
3078 my ($vmid, $resp) = @_;
3079 my $data = $resp->{'return'} || [];
3080 my $totalrdbytes = 0;
3081 my $totalwrbytes = 0;
3083 for my $blockstat (@$data) {
3084 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
3085 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
3087 $blockstat->{device
} =~ s/drive-//;
3088 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
3090 $res->{$vmid}->{diskread
} = $totalrdbytes;
3091 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
3094 my $machinecb = sub {
3095 my ($vmid, $resp) = @_;
3096 my $data = $resp->{'return'} || [];
3098 $res->{$vmid}->{'running-machine'} =
3099 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
3102 my $versioncb = sub {
3103 my ($vmid, $resp) = @_;
3104 my $data = $resp->{'return'} // {};
3105 my $version = 'unknown';
3107 if (my $v = $data->{qemu
}) {
3108 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
3111 $res->{$vmid}->{'running-qemu'} = $version;
3114 my $statuscb = sub {
3115 my ($vmid, $resp) = @_;
3117 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3118 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3119 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3120 # this fails if ballon driver is not loaded, so this must be
3121 # the last commnand (following command are aborted if this fails).
3122 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3124 my $status = 'unknown';
3125 if (!defined($status = $resp->{'return'}->{status
})) {
3126 warn "unable to get VM status\n";
3130 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3133 foreach my $vmid (keys %$list) {
3134 next if $opt_vmid && ($vmid ne $opt_vmid);
3135 next if !$res->{$vmid}->{pid
}; # not running
3136 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3139 $qmpclient->queue_execute(undef, 2);
3141 foreach my $vmid (keys %$list) {
3142 next if $opt_vmid && ($vmid ne $opt_vmid);
3143 next if !$res->{$vmid}->{pid
}; #not running
3145 # we can't use the $qmpclient since it might have already aborted on
3146 # 'query-balloon', but this might also fail for older versions...
3147 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
3148 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3151 foreach my $vmid (keys %$list) {
3152 next if $opt_vmid && ($vmid ne $opt_vmid);
3153 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3159 sub conf_has_serial
{
3162 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3163 if ($conf->{"serial$i"}) {
3171 sub conf_has_audio
{
3172 my ($conf, $id) = @_;
3175 my $audio = $conf->{"audio$id"};
3176 return if !defined($audio);
3178 my $audioproperties = parse_property_string
($audio_fmt, $audio);
3179 my $audiodriver = $audioproperties->{driver
} // 'spice';
3182 dev
=> $audioproperties->{device
},
3183 dev_id
=> "audiodev$id",
3184 backend
=> $audiodriver,
3185 backend_id
=> "$audiodriver-backend${id}",
3190 my ($audio, $audiopciaddr, $machine_version) = @_;
3194 my $id = $audio->{dev_id
};
3196 if (min_version
($machine_version, 4, 2)) {
3197 $audiodev = ",audiodev=$audio->{backend_id}";
3200 if ($audio->{dev
} eq 'AC97') {
3201 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3202 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3203 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3204 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3205 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3207 die "unkown audio device '$audio->{dev}', implement me!";
3210 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3218 socket => "/var/run/qemu-server/$vmid.swtpm",
3219 pid
=> "/var/run/qemu-server/$vmid.swtpm.pid",
3223 sub add_tpm_device
{
3224 my ($vmid, $devices, $conf) = @_;
3226 return if !$conf->{tpmstate0
};
3228 my $paths = get_tpm_paths
($vmid);
3230 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3231 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3232 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3236 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3238 return if !$tpmdrive;
3241 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
3242 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
3244 $state = PVE
::Storage
::map_volume
($storecfg, $tpm->{file
});
3246 $state = $tpm->{file
};
3249 my $paths = get_tpm_paths
($vmid);
3251 # during migration, we will get state from remote
3254 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3261 "--create-platform-cert",
3264 "/etc/swtpm_setup.conf", # do not use XDG configs
3266 "0", # force creation as root, error if not possible
3267 "--not-overwrite", # ignore existing state, do not modify
3270 push @$setup_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3271 # TPM 2.0 supports ECC crypto, use if possible
3272 push @$setup_cmd, "--ecc" if $tpm->{version
} eq 'v2.0';
3274 run_command
($setup_cmd, outfunc
=> sub {
3275 print "swtpm_setup: $1\n";
3279 # Used to distinguish different invocations in the log.
3280 my $log_prefix = "[id=" . int(time()) . "] ";
3282 my $emulator_cmd = [
3286 "backend-uri=file://$state,mode=0600",
3288 "type=unixio,path=$paths->{socket},mode=0600",
3290 "file=$paths->{pid}",
3291 "--terminate", # terminate on QEMU disconnect
3294 "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3296 push @$emulator_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3297 run_command
($emulator_cmd, outfunc
=> sub { print $1; });
3299 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3300 while (! -e
$paths->{pid
}) {
3301 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3305 # return untainted PID of swtpm daemon so it can be killed on error
3306 file_read_firstline
($paths->{pid
}) =~ m/(\d+)/;
3310 sub vga_conf_has_spice
{
3313 my $vgaconf = parse_vga
($vga);
3314 my $vgatype = $vgaconf->{type
};
3315 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3322 return get_host_arch
() eq $arch;
3327 return $conf->{arch
} // get_host_arch
();
3330 my $default_machines = {
3335 sub get_installed_machine_version
{
3336 my ($kvmversion) = @_;
3337 $kvmversion = kvm_user_version
() if !defined($kvmversion);
3338 $kvmversion =~ m/^(\d+\.\d+)/;
3342 sub windows_get_pinned_machine_version
{
3343 my ($machine, $base_version, $kvmversion) = @_;
3345 my $pin_version = $base_version;
3346 if (!defined($base_version) ||
3347 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3349 $pin_version = get_installed_machine_version
($kvmversion);
3351 if (!$machine || $machine eq 'pc') {
3352 $machine = "pc-i440fx-$pin_version";
3353 } elsif ($machine eq 'q35') {
3354 $machine = "pc-q35-$pin_version";
3355 } elsif ($machine eq 'virt') {
3356 $machine = "virt-$pin_version";
3358 warn "unknown machine type '$machine', not touching that!\n";
3364 sub get_vm_machine
{
3365 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3367 my $machine = $forcemachine || $conf->{machine
};
3369 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3370 $kvmversion //= kvm_user_version
();
3371 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3372 # layout which confuses windows quite a bit and may result in various regressions..
3373 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3374 if (windows_version
($conf->{ostype
})) {
3375 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3378 $machine ||= $default_machines->{$arch};
3379 if ($add_pve_version) {
3380 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3381 $machine .= "+pve$pvever";
3385 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3386 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3387 $machine = $1 if $is_pxe;
3389 # for version-pinned machines that do not include a pve-version (e.g.
3390 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3391 $machine .= '+pve0';
3393 $machine .= '.pxe' if $is_pxe;
3399 sub get_ovmf_files
($$$) {
3400 my ($arch, $efidisk, $smm) = @_;
3402 my $types = $OVMF->{$arch}
3403 or die "no OVMF images known for architecture '$arch'\n";
3405 my $type = 'default';
3406 if ($arch ne "aarch64" && defined($efidisk->{efitype
}) && $efidisk->{efitype
} eq '4m') {
3407 $type = $smm ?
"4m" : "4m-no-smm";
3408 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3411 my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3412 die "EFI base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3413 die "EFI vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
3415 return ($ovmf_code, $ovmf_vars);
3419 aarch64
=> '/usr/bin/qemu-system-aarch64',
3420 x86_64
=> '/usr/bin/qemu-system-x86_64',
3422 sub get_command_for_arch
($) {
3424 return '/usr/bin/kvm' if is_native
($arch);
3426 my $cmd = $Arch2Qemu->{$arch}
3427 or die "don't know how to emulate architecture '$arch'\n";
3431 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3432 # to use in a QEMU command line (-cpu element), first array_intersect the result
3433 # of query_supported_ with query_understood_. This is necessary because:
3435 # a) query_understood_ returns flags the host cannot use and
3436 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3437 # flags, but CPU settings - with most of them being flags. Those settings
3438 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3440 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3441 # expensive. If you need the value returned from this, you can get it much
3442 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3443 # $accel being 'kvm' or 'tcg'.
3445 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3446 # changes, automatically populating pmxcfs.
3448 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3449 # since kvm and tcg machines support different flags
3451 sub query_supported_cpu_flags
{
3454 $arch //= get_host_arch
();
3455 my $default_machine = $default_machines->{$arch};
3459 # FIXME: Once this is merged, the code below should work for ARM as well:
3460 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3461 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3464 my $kvm_supported = defined(kvm_version
());
3465 my $qemu_cmd = get_command_for_arch
($arch);
3467 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3469 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3470 my $query_supported_run_qemu = sub {
3476 '-machine', $default_machine,
3478 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3479 '-mon', 'chardev=qmp,mode=control',
3480 '-pidfile', $pidfile,
3485 push @$cmd, '-accel', 'tcg';
3488 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3489 die "QEMU flag querying VM exited with code " . $rc if $rc;
3492 my $cmd_result = mon_cmd
(
3494 'query-cpu-model-expansion',
3496 model
=> { name
=> 'host' }
3499 my $props = $cmd_result->{model
}->{props
};
3500 foreach my $prop (keys %$props) {
3501 next if $props->{$prop} ne '1';
3502 # QEMU returns some flags multiple times, with '_', '.' or '-'
3503 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3504 # We only keep those with underscores, to match /proc/cpuinfo
3505 $prop =~ s/\.|-/_/g;
3506 $flags->{$prop} = 1;
3511 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3512 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3516 return [ sort keys %$flags ];
3519 # We need to query QEMU twice, since KVM and TCG have different supported flags
3520 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3521 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3522 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3524 if ($kvm_supported) {
3525 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3526 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3533 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3534 my $understood_cpu_flag_dir = "/usr/share/kvm";
3535 sub query_understood_cpu_flags
{
3536 my $arch = get_host_arch
();
3537 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3539 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3542 my $raw = file_get_contents
($filepath);
3543 $raw =~ s/^\s+|\s+$//g;
3544 my @flags = split(/\s+/, $raw);
3549 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3550 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3551 my sub should_disable_smm
{
3552 my ($conf, $vga, $machine) = @_;
3554 return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3556 return (!defined($conf->{bios
}) || $conf->{bios
} eq 'seabios') &&
3557 $vga->{type
} && $vga->{type
} =~ m/^(serial\d+|none)$/;
3560 my sub print_ovmf_drive_commandlines
{
3561 my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3563 my $d = $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
3565 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch, $d, $q35);
3567 my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3569 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3570 my ($path, $format) = $d->@{'file', 'format'};
3572 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3573 if (!defined($format)) {
3574 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3575 $format = qemu_img_format
($scfg, $volname);
3577 } elsif (!defined($format)) {
3578 die "efidisk format must be specified\n";
3580 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3581 if ($path =~ m/^rbd:/) {
3582 $var_drive_str .= ',cache=writeback';
3583 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3585 $var_drive_str .= ",format=$format,file=$path";
3587 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3588 $var_drive_str .= ',readonly=on' if drive_is_read_only
($conf, $d);
3590 log_warn
("no efidisk configured! Using temporary efivars disk.");
3591 my $path = "/tmp/$vmid-ovmf.fd";
3592 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3593 $var_drive_str .= ",format=raw,file=$path";
3594 $var_drive_str .= ",size=" . (-s
$ovmf_vars) if $version_guard->(4, 1, 2);
3597 return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3600 sub config_to_command
{
3601 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3604 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3607 my $ostype = $conf->{ostype
};
3608 my $winversion = windows_version
($ostype);
3609 my $kvm = $conf->{kvm
};
3610 my $nodename = nodename
();
3612 my $arch = get_vm_arch
($conf);
3613 my $kvm_binary = get_command_for_arch
($arch);
3614 my $kvmver = kvm_user_version
($kvm_binary);
3616 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3617 $kvmver //= "undefined";
3618 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3621 my $add_pve_version = min_version
($kvmver, 4, 1);
3623 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3624 my $machine_version = extract_version
($machine_type, $kvmver);
3625 $kvm //= 1 if is_native
($arch);
3627 $machine_version =~ m/(\d+)\.(\d+)/;
3628 my ($machine_major, $machine_minor) = ($1, $2);
3630 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3631 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3632 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3633 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3634 ." please upgrade node '$nodename'\n"
3635 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3636 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3637 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3638 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3639 ." node '$nodename'\n";
3642 # if a specific +pve version is required for a feature, use $version_guard
3643 # instead of min_version to allow machines to be run with the minimum
3645 my $required_pve_version = 0;
3646 my $version_guard = sub {
3647 my ($major, $minor, $pve) = @_;
3648 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3649 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3650 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3651 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3655 if ($kvm && !defined kvm_version
()) {
3656 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3657 ." or enable in BIOS.\n";
3660 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3661 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3662 my $use_old_bios_files = undef;
3663 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3666 if ($conf->{affinity
}) {
3667 push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity
};
3670 push @$cmd, $kvm_binary;
3672 push @$cmd, '-id', $vmid;
3674 my $vmname = $conf->{name
} || "vm$vmid";
3676 push @$cmd, '-name', "$vmname,debug-threads=on";
3678 push @$cmd, '-no-shutdown';
3682 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3683 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3684 push @$cmd, '-mon', "chardev=qmp,mode=control";
3686 if (min_version
($machine_version, 2, 12)) {
3687 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3688 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3691 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3693 push @$cmd, '-daemonize';
3695 if ($conf->{smbios1
}) {
3696 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3697 if ($smbios_conf->{base64
}) {
3698 # Do not pass base64 flag to qemu
3699 delete $smbios_conf->{base64
};
3700 my $smbios_string = "";
3701 foreach my $key (keys %$smbios_conf) {
3703 if ($key eq "uuid") {
3704 $value = $smbios_conf->{uuid
}
3706 $value = decode_base64
($smbios_conf->{$key});
3708 # qemu accepts any binary data, only commas need escaping by double comma
3710 $smbios_string .= "," . $key . "=" . $value if $value;
3712 push @$cmd, '-smbios', "type=1" . $smbios_string;
3714 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3718 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3719 my ($code_drive_str, $var_drive_str) =
3720 print_ovmf_drive_commandlines
($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3721 push $cmd->@*, '-drive', $code_drive_str;
3722 push $cmd->@*, '-drive', $var_drive_str;
3725 if ($q35) { # tell QEMU to load q35 config early
3726 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3727 if (min_version
($machine_version, 4, 0)) {
3728 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3730 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3734 if (defined(my $fixups = qemu_created_version_fixups
($conf, $forcemachine, $kvmver))) {
3735 push @$cmd, $fixups->@*;
3738 if ($conf->{vmgenid
}) {
3739 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3742 # add usb controllers
3743 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3744 $conf, $bridges, $arch, $machine_type, $machine_version);
3745 push @$devices, @usbcontrollers if @usbcontrollers;
3746 my $vga = parse_vga
($conf->{vga
});
3748 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3749 $vga->{type
} = 'qxl' if $qxlnum;
3751 if (!$vga->{type
}) {
3752 if ($arch eq 'aarch64') {
3753 $vga->{type
} = 'virtio';
3754 } elsif (min_version
($machine_version, 2, 9)) {
3755 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3757 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3761 # enable absolute mouse coordinates (needed by vnc)
3762 my $tablet = $conf->{tablet
};
3763 if (!defined($tablet)) {
3764 $tablet = $defaults->{tablet
};
3765 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3766 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3770 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3771 my $kbd = print_keyboarddevice_full
($conf, $arch);
3772 push @$devices, '-device', $kbd if defined($kbd);
3775 my $bootorder = device_bootorder
($conf);
3777 # host pci device passthrough
3778 my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3779 $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3782 my $usb_dev_features = {};
3783 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3785 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3786 $conf, $usb_dev_features, $bootorder, $machine_version);
3787 push @$devices, @usbdevices if @usbdevices;
3790 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3791 my $path = $conf->{"serial$i"} or next;
3792 if ($path eq 'socket') {
3793 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3794 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3795 # On aarch64, serial0 is the UART device. QEMU only allows
3796 # connecting UART devices via the '-serial' command line, as
3797 # the device has a fixed slot on the hardware...
3798 if ($arch eq 'aarch64' && $i == 0) {
3799 push @$devices, '-serial', "chardev:serial$i";
3801 push @$devices, '-device', "isa-serial,chardev=serial$i";
3804 die "no such serial device\n" if ! -c
$path;
3805 push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3806 push @$devices, '-device', "isa-serial,chardev=serial$i";
3811 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3812 if (my $path = $conf->{"parallel$i"}) {
3813 die "no such parallel device\n" if ! -c
$path;
3814 my $devtype = $path =~ m!^/dev/usb/lp! ?
'serial' : 'parallel';
3815 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3816 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3820 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3821 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3822 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3823 push @$devices, @$audio_devs;
3826 # Add a TPM only if the VM is not a template,
3827 # to support backing up template VMs even if the TPM disk is write-protected.
3828 add_tpm_device
($vmid, $devices, $conf) if (!PVE
::QemuConfig-
>is_template($conf));
3831 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3832 $sockets = $conf->{sockets
} if $conf->{sockets
};
3834 my $cores = $conf->{cores
} || 1;
3836 my $maxcpus = $sockets * $cores;
3838 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3840 my $allowed_vcpus = $cpuinfo->{cpus
};
3842 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3844 if ($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3845 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3846 for (my $i = 2; $i <= $vcpus; $i++) {
3847 my $cpustr = print_cpu_device
($conf,$i);
3848 push @$cmd, '-device', $cpustr;
3853 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3855 push @$cmd, '-nodefaults';
3857 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3859 push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3861 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3863 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3864 push @$devices, '-device', print_vga_device
(
3865 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3867 push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type
} eq 'virtio-gl'; # VIRGL
3869 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3870 push @$cmd, '-vnc', "unix:$socket,password=on";
3872 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3873 push @$cmd, '-nographic';
3877 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3878 my $useLocaltime = $conf->{localtime};
3880 if ($winversion >= 5) { # windows
3881 $useLocaltime = 1 if !defined($conf->{localtime});
3883 # use time drift fix when acpi is enabled
3884 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3885 $tdf = 1 if !defined($conf->{tdf
});
3889 if ($winversion >= 6) {
3890 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3891 push @$machineFlags, 'hpet=off';
3894 push @$rtcFlags, 'driftfix=slew' if $tdf;
3896 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3897 push @$rtcFlags, "base=$conf->{startdate}";
3898 } elsif ($useLocaltime) {
3899 push @$rtcFlags, 'base=localtime';
3903 push @$cmd, '-cpu', $forcecpu;
3905 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3908 PVE
::QemuServer
::Memory
::config
(
3909 $conf, $vmid, $sockets, $cores, $defaults, $hotplug_features->{memory
}, $cmd);
3911 push @$cmd, '-S' if $conf->{freeze
};
3913 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3915 my $guest_agent = parse_guest_agent
($conf);
3917 if ($guest_agent->{enabled
}) {
3918 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3919 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3921 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3922 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3923 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3924 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3925 } elsif ($guest_agent->{type
} eq 'isa') {
3926 push @$devices, '-device', "isa-serial,chardev=qga0";
3930 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3931 if ($rng && $version_guard->(4, 1, 2)) {
3932 check_rng_source
($rng->{source
});
3934 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3935 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3936 my $limiter_str = "";
3938 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3941 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3942 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3943 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3948 if ($qxlnum || $vga->{type
} =~ /^virtio/) {
3951 for (my $i = 1; $i < $qxlnum; $i++){
3952 push @$devices, '-device', print_vga_device
(
3953 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3956 # assume other OS works like Linux
3957 my ($ram, $vram) = ("134217728", "67108864");
3958 if ($vga->{memory
}) {
3959 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3960 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3962 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3963 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3967 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3969 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3970 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3971 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3973 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3974 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3975 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3977 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3978 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3980 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3981 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3982 if ($spice_enhancement->{foldersharing
}) {
3983 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3984 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3987 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3988 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3989 if $spice_enhancement->{videostreaming
};
3991 push @$devices, '-spice', "$spice_opts";
3994 # enable balloon by default, unless explicitly disabled
3995 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3996 my $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3997 my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3998 $ballooncmd .= ",free-page-reporting=on" if min_version
($machine_version, 6, 2);
3999 push @$devices, '-device', $ballooncmd;
4002 if ($conf->{watchdog
}) {
4003 my $wdopts = parse_watchdog
($conf->{watchdog
});
4004 my $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
4005 my $watchdog = $wdopts->{model
} || 'i6300esb';
4006 push @$devices, '-device', "$watchdog$pciaddr";
4007 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
4011 my $scsicontroller = {};
4012 my $ahcicontroller = {};
4013 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
4015 # Add iscsi initiator name if available
4016 if (my $initiator = get_initiator_name
()) {
4017 push @$devices, '-iscsi', "initiator-name=$initiator";
4020 PVE
::QemuConfig-
>foreach_volume($conf, sub {
4021 my ($ds, $drive) = @_;
4023 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
4024 check_volume_storage_type
($storecfg, $drive->{file
});
4025 push @$vollist, $drive->{file
};
4028 # ignore efidisk here, already added in bios/fw handling code above
4029 return if $drive->{interface
} eq 'efidisk';
4031 return if $drive->{interface
} eq 'tpmstate';
4033 $use_virtio = 1 if $ds =~ m/^virtio/;
4035 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
4037 if ($drive->{interface
} eq 'virtio'){
4038 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
4041 if ($drive->{interface
} eq 'scsi') {
4043 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
4045 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4046 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4048 my $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
4049 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
4052 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
4053 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4054 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4055 } elsif ($drive->{iothread
}) {
4057 "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4062 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
4063 $queues = ",num_queues=$drive->{queues}";
4066 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4067 if !$scsicontroller->{$controller};
4068 $scsicontroller->{$controller}=1;
4071 if ($drive->{interface
} eq 'sata') {
4072 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
4073 my $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
4074 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4075 if !$ahcicontroller->{$controller};
4076 $ahcicontroller->{$controller}=1;
4079 my $pbs_conf = $pbs_backing->{$ds};
4080 my $pbs_name = undef;
4082 $pbs_name = "drive-$ds-pbs";
4083 push @$devices, '-blockdev', print_pbs_blockdev
($pbs_conf, $pbs_name);
4086 my $drive_cmd = print_drive_commandline_full
(
4087 $storecfg, $vmid, $drive, $pbs_name, min_version
($kvmver, 6, 0));
4089 # extra protection for templates, but SATA and IDE don't support it..
4090 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
4092 push @$devices, '-drive',$drive_cmd;
4093 push @$devices, '-device', print_drivedevice_full
(
4094 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4097 for (my $i = 0; $i < $MAX_NETS; $i++) {
4098 my $netname = "net$i";
4100 next if !$conf->{$netname};
4101 my $d = parse_net
($conf->{$netname});
4103 # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4105 $use_virtio = 1 if $d->{model
} eq 'virtio';
4107 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
4109 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
4110 push @$devices, '-netdev', $netdevfull;
4112 my $netdevicefull = print_netdevice_full
(
4113 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4115 push @$devices, '-device', $netdevicefull;
4118 if ($conf->{ivshmem
}) {
4119 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4123 $bus = print_pcie_addr
("ivshmem");
4125 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4128 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4129 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4131 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4132 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4133 .",size=$ivshmem->{size}M";
4136 # pci.4 is nested in pci.1
4137 $bridges->{1} = 1 if $bridges->{4};
4139 if (!$q35) { # add pci bridges
4140 if (min_version
($machine_version, 2, 3)) {
4144 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4147 for my $k (sort {$b cmp $a} keys %$bridges) {
4148 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4151 if ($k == 2 && $legacy_igd) {
4154 my $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
4155 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4157 if ($q35) { # add after -readconfig pve-q35.cfg
4158 splice @$devices, 2, 0, '-device', $devstr;
4160 unshift @$devices, '-device', $devstr if $k > 0;
4165 push @$machineFlags, 'accel=tcg';
4168 push @$machineFlags, 'smm=off' if should_disable_smm
($conf, $vga, $machine_type);
4170 my $machine_type_min = $machine_type;
4171 if ($add_pve_version) {
4172 $machine_type_min =~ s/\+pve\d+$//;
4173 $machine_type_min .= "+pve$required_pve_version";
4175 push @$machineFlags, "type=${machine_type_min}";
4177 push @$cmd, @$devices;
4178 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4179 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4180 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4182 if (my $vmstate = $conf->{vmstate
}) {
4183 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4184 push @$vollist, $vmstate;
4185 push @$cmd, '-loadstate', $statepath;
4186 print "activating and using '$vmstate' as vmstate\n";
4189 if (PVE
::QemuConfig-
>is_template($conf)) {
4190 # needed to workaround base volumes being read-only
4191 push @$cmd, '-snapshot';
4195 if ($conf->{args
}) {
4196 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4200 return wantarray ?
($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4203 sub check_rng_source
{
4206 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4207 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4210 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4211 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
4212 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4213 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4214 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4215 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4223 my $res = mon_cmd
($vmid, 'query-spice');
4225 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4228 sub vm_devices_list
{
4231 my $res = mon_cmd
($vmid, 'query-pci');
4232 my $devices_to_check = [];
4234 foreach my $pcibus (@$res) {
4235 push @$devices_to_check, @{$pcibus->{devices
}},
4238 while (@$devices_to_check) {
4240 for my $d (@$devices_to_check) {
4241 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4242 next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices
};
4244 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4245 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4247 $devices_to_check = $to_check;
4250 my $resblock = mon_cmd
($vmid, 'query-block');
4251 foreach my $block (@$resblock) {
4252 if($block->{device
} =~ m/^drive-(\S+)/){
4257 my $resmice = mon_cmd
($vmid, 'query-mice');
4258 foreach my $mice (@$resmice) {
4259 if ($mice->{name
} eq 'QEMU HID Tablet') {
4260 $devices->{tablet
} = 1;
4265 # for usb devices there is no query-usb
4266 # but we can iterate over the entries in
4267 # qom-list path=/machine/peripheral
4268 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4269 foreach my $per (@$resperipheral) {
4270 if ($per->{name
} =~ m/^usb(?:redirdev)?\d+$/) {
4271 $devices->{$per->{name
}} = 1;
4279 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4281 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4283 my $devices_list = vm_devices_list
($vmid);
4284 return 1 if defined($devices_list->{$deviceid});
4286 # add PCI bridge if we need it for the device
4287 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4289 if ($deviceid eq 'tablet') {
4290 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4291 } elsif ($deviceid eq 'keyboard') {
4292 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4293 } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4295 qemu_spice_usbredir_chardev_add
($vmid, "usbredirchardev$id");
4296 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_spice_usbdevice
($id, "xhci", $id + 1));
4297 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4298 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device, {}, $1 + 1));
4299 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4300 qemu_iothread_add
($vmid, $deviceid, $device);
4302 qemu_driveadd
($storecfg, $vmid, $device);
4303 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4305 qemu_deviceadd
($vmid, $devicefull);
4306 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4308 eval { qemu_drivedel
($vmid, $deviceid); };
4312 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4313 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4314 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4315 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4317 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4319 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4320 qemu_iothread_add
($vmid, $deviceid, $device);
4321 $devicefull .= ",iothread=iothread-$deviceid";
4324 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4325 $devicefull .= ",num_queues=$device->{queues}";
4328 qemu_deviceadd
($vmid, $devicefull);
4329 qemu_deviceaddverify
($vmid, $deviceid);
4330 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4331 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4332 qemu_driveadd
($storecfg, $vmid, $device);
4334 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4335 eval { qemu_deviceadd
($vmid, $devicefull); };
4337 eval { qemu_drivedel
($vmid, $deviceid); };
4341 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4342 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4344 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4345 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type);
4346 my $use_old_bios_files = undef;
4347 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4349 my $netdevicefull = print_netdevice_full
(
4350 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4351 qemu_deviceadd
($vmid, $netdevicefull);
4353 qemu_deviceaddverify
($vmid, $deviceid);
4354 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4357 eval { qemu_netdevdel
($vmid, $deviceid); };
4361 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4363 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4364 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4366 qemu_deviceadd
($vmid, $devicefull);
4367 qemu_deviceaddverify
($vmid, $deviceid);
4369 die "can't hotplug device '$deviceid'\n";
4375 # fixme: this should raise exceptions on error!
4376 sub vm_deviceunplug
{
4377 my ($vmid, $conf, $deviceid) = @_;
4379 my $devices_list = vm_devices_list
($vmid);
4380 return 1 if !defined($devices_list->{$deviceid});
4382 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4383 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4385 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4386 qemu_devicedel
($vmid, $deviceid);
4387 } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4388 qemu_devicedel
($vmid, $deviceid);
4389 qemu_devicedelverify
($vmid, $deviceid);
4390 } elsif ($deviceid =~ m/^usb\d+$/) {
4391 qemu_devicedel
($vmid, $deviceid);
4392 qemu_devicedelverify
($vmid, $deviceid);
4393 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4394 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4396 qemu_devicedel
($vmid, $deviceid);
4397 qemu_devicedelverify
($vmid, $deviceid);
4398 qemu_drivedel
($vmid, $deviceid);
4399 qemu_iothread_del
($vmid, $deviceid, $device);
4400 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4401 qemu_devicedel
($vmid, $deviceid);
4402 qemu_devicedelverify
($vmid, $deviceid);
4403 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4404 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4406 qemu_devicedel
($vmid, $deviceid);
4407 qemu_devicedelverify
($vmid, $deviceid);
4408 qemu_drivedel
($vmid, $deviceid);
4409 qemu_deletescsihw
($conf, $vmid, $deviceid);
4411 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4412 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4413 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4414 qemu_devicedel
($vmid, $deviceid);
4415 qemu_devicedelverify
($vmid, $deviceid);
4416 qemu_netdevdel
($vmid, $deviceid);
4418 die "can't unplug device '$deviceid'\n";
4424 sub qemu_spice_usbredir_chardev_add
{
4425 my ($vmid, $id) = @_;
4427 mon_cmd
($vmid, "chardev-add" , (
4438 sub qemu_deviceadd
{
4439 my ($vmid, $devicefull) = @_;
4441 $devicefull = "driver=".$devicefull;
4442 my %options = split(/[=,]/, $devicefull);
4444 mon_cmd
($vmid, "device_add" , %options);
4447 sub qemu_devicedel
{
4448 my ($vmid, $deviceid) = @_;
4450 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4453 sub qemu_iothread_add
{
4454 my ($vmid, $deviceid, $device) = @_;
4456 if ($device->{iothread
}) {
4457 my $iothreads = vm_iothreads_list
($vmid);
4458 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4462 sub qemu_iothread_del
{
4463 my ($vmid, $deviceid, $device) = @_;
4465 if ($device->{iothread
}) {
4466 my $iothreads = vm_iothreads_list
($vmid);
4467 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4471 sub qemu_objectadd
{
4472 my ($vmid, $objectid, $qomtype) = @_;
4474 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4479 sub qemu_objectdel
{
4480 my ($vmid, $objectid) = @_;
4482 mon_cmd
($vmid, "object-del", id
=> $objectid);
4488 my ($storecfg, $vmid, $device) = @_;
4490 my $kvmver = get_running_qemu_version
($vmid);
4491 my $io_uring = min_version
($kvmver, 6, 0);
4492 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4493 $drive =~ s/\\/\\\\/g;
4494 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4496 # If the command succeeds qemu prints: "OK
"
4497 return 1 if $ret =~ m/OK/s;
4499 die "adding drive failed
: $ret\n";
4503 my ($vmid, $deviceid) = @_;
4505 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4508 return 1 if $ret eq "";
4510 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4511 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4513 die "deleting drive
$deviceid failed
: $ret\n";
4516 sub qemu_deviceaddverify {
4517 my ($vmid, $deviceid) = @_;
4519 for (my $i = 0; $i <= 5; $i++) {
4520 my $devices_list = vm_devices_list($vmid);
4521 return 1 if defined($devices_list->{$deviceid});
4525 die "error on hotplug device
'$deviceid'\n";
4529 sub qemu_devicedelverify {
4530 my ($vmid, $deviceid) = @_;
4532 # need to verify that the device is correctly removed as device_del
4533 # is async and empty return is not reliable
4535 for (my $i = 0; $i <= 5; $i++) {
4536 my $devices_list = vm_devices_list($vmid);
4537 return 1 if !defined($devices_list->{$deviceid});
4541 die "error on hot-unplugging device
'$deviceid'\n";
4544 sub qemu_findorcreatescsihw {
4545 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4547 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4549 my $scsihwid="$controller_prefix$controller";
4550 my $devices_list = vm_devices_list($vmid);
4552 if (!defined($devices_list->{$scsihwid})) {
4553 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4559 sub qemu_deletescsihw {
4560 my ($conf, $vmid, $opt) = @_;
4562 my $device = parse_drive($opt, $conf->{$opt});
4564 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4565 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4569 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4571 my $devices_list = vm_devices_list($vmid);
4572 foreach my $opt (keys %{$devices_list}) {
4573 if (is_valid_drivename($opt)) {
4574 my $drive = parse_drive($opt, $conf->{$opt});
4575 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4581 my $scsihwid="scsihw
$controller";
4583 vm_deviceunplug($vmid, $conf, $scsihwid);
4588 sub qemu_add_pci_bridge {
4589 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4595 print_pci_addr($device, $bridges, $arch, $machine_type);
4597 while (my ($k, $v) = each %$bridges) {
4600 return 1 if !defined($bridgeid) || $bridgeid < 1;
4602 my $bridge = "pci
.$bridgeid";
4603 my $devices_list = vm_devices_list($vmid);
4605 if (!defined($devices_list->{$bridge})) {
4606 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4612 sub qemu_set_link_status {
4613 my ($vmid, $device, $up) = @_;
4615 mon_cmd($vmid, "set_link
", name => $device,
4616 up => $up ? JSON::true : JSON::false);
4619 sub qemu_netdevadd {
4620 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4622 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4623 my %options = split(/[=,]/, $netdev);
4625 if (defined(my $vhost = $options{vhost})) {
4626 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4629 if (defined(my $queues = $options{queues})) {
4630 $options{queues} = $queues + 0;
4633 mon_cmd($vmid, "netdev_add
", %options);
4637 sub qemu_netdevdel {
4638 my ($vmid, $deviceid) = @_;
4640 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4643 sub qemu_usb_hotplug {
4644 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4648 # remove the old one first
4649 vm_deviceunplug($vmid, $conf, $deviceid);
4651 # check if xhci controller is necessary and available
4652 my $devicelist = vm_devices_list($vmid);
4654 if (!$devicelist->{xhci}) {
4655 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4656 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4660 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4663 sub qemu_cpu_hotplug {
4664 my ($vmid, $conf, $vcpus) = @_;
4666 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4669 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4670 $sockets = $conf->{sockets} if $conf->{sockets};
4671 my $cores = $conf->{cores} || 1;
4672 my $maxcpus = $sockets * $cores;
4674 $vcpus = $maxcpus if !$vcpus;
4676 die "you can
't add more vcpus than maxcpus\n"
4677 if $vcpus > $maxcpus;
4679 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4681 if ($vcpus < $currentvcpus) {
4683 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4685 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4686 qemu_devicedel($vmid, "cpu$i");
4688 my $currentrunningvcpus = undef;
4690 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4691 last if scalar(@{$currentrunningvcpus}) == $i-1;
4692 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4696 #update conf after each succesfull cpu unplug
4697 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4698 PVE::QemuConfig->write_config($vmid, $conf);
4701 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4707 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4708 die "vcpus in running vm does not match its configuration\n"
4709 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4711 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4713 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4714 my $cpustr = print_cpu_device($conf, $i);
4715 qemu_deviceadd($vmid, $cpustr);
4718 my $currentrunningvcpus = undef;
4720 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4721 last if scalar(@{$currentrunningvcpus}) == $i;
4722 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4726 #update conf after each succesfull cpu hotplug
4727 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4728 PVE::QemuConfig->write_config($vmid, $conf);
4732 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4733 mon_cmd($vmid, "cpu-add", id => int($i));
4738 sub qemu_block_set_io_throttle {
4739 my ($vmid, $deviceid,
4740 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4741 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4742 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4743 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4745 return if !check_running($vmid) ;
4747 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4749 bps_rd => int($bps_rd),
4750 bps_wr => int($bps_wr),
4752 iops_rd => int($iops_rd),
4753 iops_wr => int($iops_wr),
4754 bps_max => int($bps_max),
4755 bps_rd_max => int($bps_rd_max),
4756 bps_wr_max => int($bps_wr_max),
4757 iops_max => int($iops_max),
4758 iops_rd_max => int($iops_rd_max),
4759 iops_wr_max => int($iops_wr_max),
4760 bps_max_length => int($bps_max_length),
4761 bps_rd_max_length => int($bps_rd_max_length),
4762 bps_wr_max_length => int($bps_wr_max_length),
4763 iops_max_length => int($iops_max_length),
4764 iops_rd_max_length => int($iops_rd_max_length),
4765 iops_wr_max_length => int($iops_wr_max_length),
4770 sub qemu_block_resize {
4771 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4773 my $running = check_running($vmid);
4775 PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4777 return if !$running;
4779 my $padding = (1024 - $size % 1024) % 1024;
4780 $size = $size + $padding;
4785 device => $deviceid,
4791 sub qemu_volume_snapshot {
4792 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4794 my $running = check_running($vmid);
4796 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4797 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4799 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4803 sub qemu_volume_snapshot_delete {
4804 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4806 my $running = check_running($vmid);
4811 my $conf = PVE::QemuConfig->load_config($vmid);
4812 PVE::QemuConfig->foreach_volume($conf, sub {
4813 my ($ds, $drive) = @_;
4814 $running = 1 if $drive->{file} eq $volid;
4818 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4819 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4821 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4825 sub set_migration_caps {
4826 my ($vmid, $savevm) = @_;
4828 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4830 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4831 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4836 "auto-converge" => 1,
4838 "x-rdma-pin-all" => 0,
4841 "dirty-bitmaps" => $dirty_bitmaps,
4844 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4846 for my $supported_capability (@$supported_capabilities) {
4848 capability => $supported_capability->{capability},
4849 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4853 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4857 my ($conf, $func, @param) = @_;
4861 my $test_volid = sub {
4862 my ($key, $drive, $snapname, $pending) = @_;
4864 my $volid = $drive->{file};
4867 $volhash->{$volid}->{cdrom} //= 1;
4868 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4870 my $replicate = $drive->{replicate} // 1;
4871 $volhash->{$volid}->{replicate} //= 0;
4872 $volhash->{$volid}->{replicate} = 1 if $replicate;
4874 $volhash->{$volid}->{shared} //= 0;
4875 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4877 $volhash->{$volid}->{is_unused} //= 0;
4878 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4880 $volhash->{$volid}->{is_attached} //= 0;
4881 $volhash->{$volid}->{is_attached} = 1
4882 if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4884 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4885 if defined($snapname);
4887 $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4889 my $size = $drive->{size};
4890 $volhash->{$volid}->{size} //= $size if $size;
4892 $volhash->{$volid}->{is_vmstate} //= 0;
4893 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4895 $volhash->{$volid}->{is_tpmstate} //= 0;
4896 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0
';
4898 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4901 my $include_opts = {
4902 extra_keys => ['vmstate
'],
4903 include_unused => 1,
4906 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4908 PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4909 if defined($conf->{pending}) && $conf->{pending}->%*;
4911 foreach my $snapname (keys %{$conf->{snapshots}}) {
4912 my $snap = $conf->{snapshots}->{$snapname};
4913 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4916 foreach my $volid (keys %$volhash) {
4917 &$func($volid, $volhash->{$volid}, @param);
4921 my $fast_plug_option = {
4925 'migrate_downtime
' => 1,
4926 'migrate_speed
' => 1,
4933 'vmstatestorage
' => 1,
4936 for my $opt (keys %$confdesc_cloudinit) {
4937 $fast_plug_option->{$opt} = 1;
4940 # hotplug changes in [PENDING]
4941 # $selection hash can be used to only apply specified options, for
4942 # example: { cores => 1 } (only apply changed 'cores
')
4943 # $errors ref is used to return error messages
4944 sub vmconfig_hotplug_pending {
4945 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4947 my $defaults = load_defaults();
4948 my $arch = get_vm_arch($conf);
4949 my $machine_type = get_vm_machine($conf, undef, $arch);
4951 # commit values which do not have any impact on running VM first
4952 # Note: those option cannot raise errors, we we do not care about
4953 # $selection and always apply them.
4955 my $add_error = sub {
4956 my ($opt, $msg) = @_;
4957 $errors->{$opt} = "hotplug problem - $msg";
4960 my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4962 my $cloudinit_record_changed = sub {
4963 my ($conf, $opt, $old, $new) = @_;
4964 return if !$cloudinit_pending_properties->{$opt};
4966 my $ci = ($conf->{cloudinit} //= {});
4968 my $recorded = $ci->{$opt};
4969 my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4971 if (defined($new)) {
4972 if (defined($old)) {
4973 # an existing value is being modified
4974 if (defined($recorded)) {
4975 # the value was already not in sync
4976 if ($new eq $recorded) {
4977 # a value is being reverted to the cloud-init state:
4979 delete $added{$opt};
4981 # the value was changed multiple times, do nothing
4983 } elsif ($added{$opt}) {
4984 # the value had been marked as added and is being changed, do nothing
4986 # the value is new, record it:
4990 # a new value is being added
4991 if (defined($recorded)) {
4992 # it was already not in sync
4993 if ($new eq $recorded) {
4994 # a value is being reverted to the cloud-init state:
4996 delete $added{$opt};
4998 # the value had temporarily been removed, do nothing
5000 } elsif ($added{$opt}) {
5001 # the value had been marked as added already, do nothing
5003 # the value is new, add it
5007 } elsif (!defined($old)) {
5008 # a non-existent value is being removed? ignore...
5010 # a value is being deleted
5011 if (defined($recorded)) {
5012 # a value was already recorded, just keep it
5013 } elsif ($added{$opt}) {
5014 # the value was marked as added, remove it
5015 delete $added{$opt};
5017 # a previously unrecorded value is being removed, record the old value:
5022 my $added = join(',', sort keys %added);
5023 $ci->{added} = $added if length($added);
5027 foreach my $opt (keys %{$conf->{pending}}) { # add/change
5028 if ($fast_plug_option->{$opt}) {
5029 my $new = delete $conf->{pending}->{$opt};
5030 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
5031 $conf->{$opt} = $new;
5037 PVE::QemuConfig->write_config($vmid, $conf);
5040 my $ostype = $conf->{ostype};
5041 my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5042 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5043 my $usb_hotplug = $hotplug_features->{usb}
5044 && min_version($version, 7, 1)
5045 && defined($ostype) && ($ostype eq 'l26
' || windows_version($ostype) > 7);
5047 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5048 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5050 foreach my $opt (sort keys %$pending_delete_hash) {
5051 next if $selection && !$selection->{$opt};
5052 my $force = $pending_delete_hash->{$opt}->{force};
5054 if ($opt eq 'hotplug
') {
5055 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5056 } elsif ($opt eq 'tablet
') {
5057 die "skip\n" if !$hotplug_features->{usb};
5058 if ($defaults->{tablet}) {
5059 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
5060 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
5061 if $arch eq 'aarch64
';
5063 vm_deviceunplug($vmid, $conf, 'tablet
');
5064 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
5066 } elsif ($opt =~ m/^usb(\d+)$/) {
5068 die "skip\n" if !$usb_hotplug;
5069 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5070 vm_deviceunplug
($vmid, $conf, $opt);
5071 } elsif ($opt eq 'vcpus') {
5072 die "skip\n" if !$hotplug_features->{cpu
};
5073 qemu_cpu_hotplug
($vmid, $conf, undef);
5074 } elsif ($opt eq 'balloon') {
5075 # enable balloon device is not hotpluggable
5076 die "skip\n" if defined($conf->{balloon
}) && $conf->{balloon
} == 0;
5077 # here we reset the ballooning value to memory
5078 my $balloon = $conf->{memory
} || $defaults->{memory
};
5079 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5080 } elsif ($fast_plug_option->{$opt}) {
5082 } elsif ($opt =~ m/^net(\d+)$/) {
5083 die "skip\n" if !$hotplug_features->{network
};
5084 vm_deviceunplug
($vmid, $conf, $opt);
5085 } elsif (is_valid_drivename
($opt)) {
5086 die "skip\n" if !$hotplug_features->{disk
} || $opt =~ m/(ide|sata)(\d+)/;
5087 vm_deviceunplug
($vmid, $conf, $opt);
5088 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5089 } elsif ($opt =~ m/^memory$/) {
5090 die "skip\n" if !$hotplug_features->{memory
};
5091 PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $defaults);
5092 } elsif ($opt eq 'cpuunits') {
5093 $cgroup->change_cpu_shares(undef);
5094 } elsif ($opt eq 'cpulimit') {
5095 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5101 &$add_error($opt, $err) if $err ne "skip\n";
5103 my $old = delete $conf->{$opt};
5104 $cloudinit_record_changed->($conf, $opt, $old, undef);
5105 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5110 foreach my $opt (keys %{$conf->{pending
}}) {
5111 next if $selection && !$selection->{$opt};
5112 my $value = $conf->{pending
}->{$opt};
5114 if ($opt eq 'hotplug') {
5115 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug
} =~ /memory/);
5116 } elsif ($opt eq 'tablet') {
5117 die "skip\n" if !$hotplug_features->{usb
};
5119 vm_deviceplug
($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5120 vm_deviceplug
($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5121 if $arch eq 'aarch64';
5122 } elsif ($value == 0) {
5123 vm_deviceunplug
($vmid, $conf, 'tablet');
5124 vm_deviceunplug
($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5126 } elsif ($opt =~ m/^usb(\d+)$/) {
5128 die "skip\n" if !$usb_hotplug;
5129 my $d = eval { parse_property_string
('pve-qm-usb', $value) };
5131 if ($d->{host
} =~ m/^spice$/i) {
5132 $id = "usbredirdev$index";
5134 qemu_usb_hotplug
($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5135 } elsif ($opt eq 'vcpus') {
5136 die "skip\n" if !$hotplug_features->{cpu
};
5137 qemu_cpu_hotplug
($vmid, $conf, $value);
5138 } elsif ($opt eq 'balloon') {
5139 # enable/disable balloning device is not hotpluggable
5140 my $old_balloon_enabled = !!(!defined($conf->{balloon
}) || $conf->{balloon
});
5141 my $new_balloon_enabled = !!(!defined($conf->{pending
}->{balloon
}) || $conf->{pending
}->{balloon
});
5142 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5144 # allow manual ballooning if shares is set to zero
5145 if ((defined($conf->{shares
}) && ($conf->{shares
} == 0))) {
5146 my $balloon = $conf->{pending
}->{balloon
} || $conf->{memory
} || $defaults->{memory
};
5147 mon_cmd
($vmid, "balloon", value
=> $balloon*1024*1024);
5149 } elsif ($opt =~ m/^net(\d+)$/) {
5150 # some changes can be done without hotplug
5151 vmconfig_update_net
($storecfg, $conf, $hotplug_features->{network
},
5152 $vmid, $opt, $value, $arch, $machine_type);
5153 } elsif (is_valid_drivename
($opt)) {
5154 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5155 # some changes can be done without hotplug
5156 my $drive = parse_drive
($opt, $value);
5157 if (drive_is_cloudinit
($drive)) {
5158 $cloudinit_opt = [$opt, $drive];
5159 # apply all the other changes first, then generate the cloudinit disk
5162 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5163 $vmid, $opt, $value, $arch, $machine_type);
5164 } elsif ($opt =~ m/^memory$/) { #dimms
5165 die "skip\n" if !$hotplug_features->{memory
};
5166 $value = PVE
::QemuServer
::Memory
::qemu_memory_hotplug
($vmid, $conf, $defaults, $value);
5167 } elsif ($opt eq 'cpuunits') {
5168 my $new_cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{pending
}->{$opt}); #clamp
5169 $cgroup->change_cpu_shares($new_cpuunits);
5170 } elsif ($opt eq 'cpulimit') {
5171 my $cpulimit = $conf->{pending
}->{$opt} == 0 ?
-1 : int($conf->{pending
}->{$opt} * 100000);
5172 $cgroup->change_cpu_quota($cpulimit, 100000);
5173 } elsif ($opt eq 'agent') {
5174 vmconfig_update_agent
($conf, $opt, $value);
5176 die "skip\n"; # skip non-hot-pluggable options
5180 &$add_error($opt, $err) if $err ne "skip\n";
5182 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5183 $conf->{$opt} = $value;
5184 delete $conf->{pending
}->{$opt};
5188 if (defined($cloudinit_opt)) {
5189 my ($opt, $drive) = @$cloudinit_opt;
5190 my $value = $conf->{pending
}->{$opt};
5192 my $temp = {%$conf, $opt => $value};
5193 PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($temp, $vmid);
5194 vmconfig_update_disk
($storecfg, $conf, $hotplug_features->{disk
},
5195 $vmid, $opt, $value, $arch, $machine_type);
5198 &$add_error($opt, $err) if $err ne "skip\n";
5200 $conf->{$opt} = $value;
5201 delete $conf->{pending
}->{$opt};
5205 # unplug xhci controller if no usb device is left
5208 for (my $i = 0; $i < $PVE::QemuServer
::USB
::MAX_USB_DEVICES
; $i++) {
5209 next if !defined($conf->{"usb$i"});
5214 vm_deviceunplug
($vmid, $conf, 'xhci');
5218 PVE
::QemuConfig-
>write_config($vmid, $conf);
5220 if ($hotplug_features->{cloudinit
} && PVE
::QemuServer
::Cloudinit
::has_changes
($conf)) {
5221 PVE
::QemuServer
::vmconfig_update_cloudinit_drive
($storecfg, $conf, $vmid);
5225 sub try_deallocate_drive
{
5226 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5228 if (($force || $key =~ /^unused/) && !drive_is_cdrom
($drive, 1)) {
5229 my $volid = $drive->{file
};
5230 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
5231 my $sid = PVE
::Storage
::parse_volume_id
($volid);
5232 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5234 # check if the disk is really unused
5235 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5236 if PVE
::QemuServer
::Drive
::is_volume_in_use
($storecfg, $conf, $key, $volid);
5237 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5240 # If vm is not owner of this disk remove from config
5248 sub vmconfig_delete_or_detach_drive
{
5249 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5251 my $drive = parse_drive
($opt, $conf->{$opt});
5253 my $rpcenv = PVE
::RPCEnvironment
::get
();
5254 my $authuser = $rpcenv->get_user();
5257 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5258 try_deallocate_drive
($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5260 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $drive);
5266 sub vmconfig_apply_pending
{
5267 my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5269 return if !scalar(keys %{$conf->{pending
}});
5271 my $add_apply_error = sub {
5272 my ($opt, $msg) = @_;
5273 my $err_msg = "unable to apply pending change $opt : $msg";
5274 $errors->{$opt} = $err_msg;
5280 my $pending_delete_hash = PVE
::QemuConfig-
>parse_pending_delete($conf->{pending
}->{delete});
5281 foreach my $opt (sort keys %$pending_delete_hash) {
5282 my $force = $pending_delete_hash->{$opt}->{force
};
5284 if ($opt =~ m/^unused/) {
5285 die "internal error";
5286 } elsif (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5287 vmconfig_delete_or_detach_drive
($vmid, $storecfg, $conf, $opt, $force);
5291 $add_apply_error->($opt, $err);
5293 PVE
::QemuConfig-
>remove_from_pending_delete($conf, $opt);
5294 delete $conf->{$opt};
5298 PVE
::QemuConfig-
>cleanup_pending($conf);
5300 my $generate_cloudinit = $skip_cloud_init ?
0 : undef;
5302 foreach my $opt (keys %{$conf->{pending
}}) { # add/change
5303 next if $opt eq 'delete'; # just to be sure
5305 if (defined($conf->{$opt}) && is_valid_drivename
($opt)) {
5306 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, parse_drive
($opt, $conf->{$opt}))
5310 $add_apply_error->($opt, $err);
5313 if (is_valid_drivename
($opt)) {
5314 my $drive = parse_drive
($opt, $conf->{pending
}->{$opt});
5315 $generate_cloudinit //= 1 if drive_is_cloudinit
($drive);
5318 $conf->{$opt} = delete $conf->{pending
}->{$opt};
5322 # write all changes at once to avoid unnecessary i/o
5323 PVE
::QemuConfig-
>write_config($vmid, $conf);
5324 if ($generate_cloudinit) {
5325 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5326 # After successful generation and if there were changes to be applied, update the
5327 # config to drop the {cloudinit} entry.
5328 PVE
::QemuConfig-
>write_config($vmid, $conf);
5333 sub vmconfig_update_net
{
5334 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5336 my $newnet = parse_net
($value);
5338 if ($conf->{$opt}) {
5339 my $oldnet = parse_net
($conf->{$opt});
5341 if (safe_string_ne
($oldnet->{model
}, $newnet->{model
}) ||
5342 safe_string_ne
($oldnet->{macaddr
}, $newnet->{macaddr
}) ||
5343 safe_num_ne
($oldnet->{queues
}, $newnet->{queues
}) ||
5344 safe_num_ne
($oldnet->{mtu
}, $newnet->{mtu
}) ||
5345 !($newnet->{bridge
} && $oldnet->{bridge
})) { # bridge/nat mode change
5347 # for non online change, we try to hot-unplug
5348 die "skip\n" if !$hotplug;
5349 vm_deviceunplug
($vmid, $conf, $opt);
5352 die "internal error" if $opt !~ m/net(\d+)/;
5353 my $iface = "tap${vmid}i$1";
5355 if (safe_string_ne
($oldnet->{bridge
}, $newnet->{bridge
}) ||
5356 safe_num_ne
($oldnet->{tag
}, $newnet->{tag
}) ||
5357 safe_string_ne
($oldnet->{trunks
}, $newnet->{trunks
}) ||
5358 safe_num_ne
($oldnet->{firewall
}, $newnet->{firewall
})) {
5359 PVE
::Network
::tap_unplug
($iface);
5362 PVE
::Network
::SDN
::Zones
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5364 PVE
::Network
::tap_plug
($iface, $newnet->{bridge
}, $newnet->{tag
}, $newnet->{firewall
}, $newnet->{trunks
}, $newnet->{rate
});
5366 } elsif (safe_num_ne
($oldnet->{rate
}, $newnet->{rate
})) {
5367 # Rate can be applied on its own but any change above needs to
5368 # include the rate in tap_plug since OVS resets everything.
5369 PVE
::Network
::tap_rate_limit
($iface, $newnet->{rate
});
5372 if (safe_string_ne
($oldnet->{link_down
}, $newnet->{link_down
})) {
5373 qemu_set_link_status
($vmid, $opt, !$newnet->{link_down
});
5381 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5387 sub vmconfig_update_agent
{
5388 my ($conf, $opt, $value) = @_;
5390 die "skip\n" if !$conf->{$opt};
5392 my $hotplug_options = { fstrim_cloned_disks
=> 1 };
5394 my $old_agent = parse_guest_agent
($conf);
5395 my $agent = parse_guest_agent
({$opt => $value});
5397 for my $option (keys %$agent) { # added/changed options
5398 next if defined($hotplug_options->{$option});
5399 die "skip\n" if safe_string_ne
($agent->{$option}, $old_agent->{$option});
5402 for my $option (keys %$old_agent) { # removed options
5403 next if defined($hotplug_options->{$option});
5404 die "skip\n" if safe_string_ne
($old_agent->{$option}, $agent->{$option});
5407 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5410 sub vmconfig_update_disk
{
5411 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5413 my $drive = parse_drive
($opt, $value);
5415 if ($conf->{$opt} && (my $old_drive = parse_drive
($opt, $conf->{$opt}))) {
5416 my $media = $drive->{media
} || 'disk';
5417 my $oldmedia = $old_drive->{media
} || 'disk';
5418 die "unable to change media type\n" if $media ne $oldmedia;
5420 if (!drive_is_cdrom
($old_drive)) {
5422 if ($drive->{file
} ne $old_drive->{file
}) {
5424 die "skip\n" if !$hotplug;
5426 # unplug and register as unused
5427 vm_deviceunplug
($vmid, $conf, $opt);
5428 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive)
5431 # update existing disk
5433 # skip non hotpluggable value
5434 if (safe_string_ne
($drive->{aio
}, $old_drive->{aio
}) ||
5435 safe_string_ne
($drive->{discard
}, $old_drive->{discard
}) ||
5436 safe_string_ne
($drive->{iothread
}, $old_drive->{iothread
}) ||
5437 safe_string_ne
($drive->{queues
}, $old_drive->{queues
}) ||
5438 safe_string_ne
($drive->{cache
}, $old_drive->{cache
}) ||
5439 safe_string_ne
($drive->{ssd
}, $old_drive->{ssd
}) ||
5440 safe_string_ne
($drive->{ro
}, $old_drive->{ro
})) {
5445 if (safe_num_ne
($drive->{mbps
}, $old_drive->{mbps
}) ||
5446 safe_num_ne
($drive->{mbps_rd
}, $old_drive->{mbps_rd
}) ||
5447 safe_num_ne
($drive->{mbps_wr
}, $old_drive->{mbps_wr
}) ||
5448 safe_num_ne
($drive->{iops
}, $old_drive->{iops
}) ||
5449 safe_num_ne
($drive->{iops_rd
}, $old_drive->{iops_rd
}) ||
5450 safe_num_ne
($drive->{iops_wr
}, $old_drive->{iops_wr
}) ||
5451 safe_num_ne
($drive->{mbps_max
}, $old_drive->{mbps_max
}) ||
5452 safe_num_ne
($drive->{mbps_rd_max
}, $old_drive->{mbps_rd_max
}) ||
5453 safe_num_ne
($drive->{mbps_wr_max
}, $old_drive->{mbps_wr_max
}) ||
5454 safe_num_ne
($drive->{iops_max
}, $old_drive->{iops_max
}) ||
5455 safe_num_ne
($drive->{iops_rd_max
}, $old_drive->{iops_rd_max
}) ||
5456 safe_num_ne
($drive->{iops_wr_max
}, $old_drive->{iops_wr_max
}) ||
5457 safe_num_ne
($drive->{bps_max_length
}, $old_drive->{bps_max_length
}) ||
5458 safe_num_ne
($drive->{bps_rd_max_length
}, $old_drive->{bps_rd_max_length
}) ||
5459 safe_num_ne
($drive->{bps_wr_max_length
}, $old_drive->{bps_wr_max_length
}) ||
5460 safe_num_ne
($drive->{iops_max_length
}, $old_drive->{iops_max_length
}) ||
5461 safe_num_ne
($drive->{iops_rd_max_length
}, $old_drive->{iops_rd_max_length
}) ||
5462 safe_num_ne
($drive->{iops_wr_max_length
}, $old_drive->{iops_wr_max_length
})) {
5464 qemu_block_set_io_throttle
(
5466 ($drive->{mbps
} || 0)*1024*1024,
5467 ($drive->{mbps_rd
} || 0)*1024*1024,
5468 ($drive->{mbps_wr
} || 0)*1024*1024,
5469 $drive->{iops
} || 0,
5470 $drive->{iops_rd
} || 0,
5471 $drive->{iops_wr
} || 0,
5472 ($drive->{mbps_max
} || 0)*1024*1024,
5473 ($drive->{mbps_rd_max
} || 0)*1024*1024,
5474 ($drive->{mbps_wr_max
} || 0)*1024*1024,
5475 $drive->{iops_max
} || 0,
5476 $drive->{iops_rd_max
} || 0,
5477 $drive->{iops_wr_max
} || 0,
5478 $drive->{bps_max_length
} || 1,
5479 $drive->{bps_rd_max_length
} || 1,
5480 $drive->{bps_wr_max_length
} || 1,
5481 $drive->{iops_max_length
} || 1,
5482 $drive->{iops_rd_max_length
} || 1,
5483 $drive->{iops_wr_max_length
} || 1,
5493 if ($drive->{file
} eq 'none') {
5494 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5495 if (drive_is_cloudinit
($old_drive)) {
5496 vmconfig_register_unused_drive
($storecfg, $vmid, $conf, $old_drive);
5499 my $path = get_iso_path
($storecfg, $vmid, $drive->{file
});
5501 # force eject if locked
5502 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$opt");
5505 mon_cmd
($vmid, "blockdev-change-medium",
5506 id
=> "$opt", filename
=> "$path");
5514 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5516 PVE
::Storage
::activate_volumes
($storecfg, [$drive->{file
}]) if $drive->{file
} !~ m
|^/dev/.+|;
5517 vm_deviceplug
($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5520 sub vmconfig_update_cloudinit_drive
{
5521 my ($storecfg, $conf, $vmid) = @_;
5523 my $cloudinit_ds = undef;
5524 my $cloudinit_drive = undef;
5526 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5527 my ($ds, $drive) = @_;
5528 if (PVE
::QemuServer
::drive_is_cloudinit
($drive)) {
5529 $cloudinit_ds = $ds;
5530 $cloudinit_drive = $drive;
5534 return if !$cloudinit_drive;
5536 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5537 PVE
::QemuConfig-
>write_config($vmid, $conf);
5540 my $running = PVE
::QemuServer
::check_running
($vmid);
5543 my $path = PVE
::Storage
::path
($storecfg, $cloudinit_drive->{file
});
5545 mon_cmd
($vmid, "eject", force
=> JSON
::true
, id
=> "$cloudinit_ds");
5546 mon_cmd
($vmid, "blockdev-change-medium", id
=> "$cloudinit_ds", filename
=> "$path");
5551 # called in locked context by incoming migration
5552 sub vm_migrate_get_nbd_disks
{
5553 my ($storecfg, $conf, $replicated_volumes) = @_;
5555 my $local_volumes = {};
5556 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5557 my ($ds, $drive) = @_;
5559 return if drive_is_cdrom
($drive);
5560 return if $ds eq 'tpmstate0';
5562 my $volid = $drive->{file
};
5566 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid);
5568 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5569 return if $scfg->{shared
};
5571 my $format = qemu_img_format
($scfg, $volname);
5573 # replicated disks re-use existing state via bitmap
5574 my $use_existing = $replicated_volumes->{$volid} ?
1 : 0;
5575 $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5577 return $local_volumes;
5580 # called in locked context by incoming migration
5581 sub vm_migrate_alloc_nbd_disks
{
5582 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5585 foreach my $opt (sort keys %$source_volumes) {
5586 my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5588 if ($use_existing) {
5589 $nbd->{$opt}->{drivestr
} = print_drive
($drive);
5590 $nbd->{$opt}->{volid
} = $volid;
5591 $nbd->{$opt}->{replicated
} = 1;
5595 $storeid = PVE
::JSONSchema
::map_id
($storagemap, $storeid);
5597 # order of precedence, filtered by whether storage supports it:
5598 # 1. explicit requested format
5599 # 2. default format of storage
5600 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5601 $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5603 my $size = $drive->{size
} / 1024;
5604 my $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $format, undef, $size);
5605 my $newdrive = $drive;
5606 $newdrive->{format
} = $format;
5607 $newdrive->{file
} = $newvolid;
5608 my $drivestr = print_drive
($newdrive);
5609 $nbd->{$opt}->{drivestr
} = $drivestr;
5610 $nbd->{$opt}->{volid
} = $newvolid;
5616 # see vm_start_nolock for parameters, additionally:
5618 # storagemap = parsed storage map for allocating NBD disks
5620 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5622 return PVE
::QemuConfig-
>lock_config($vmid, sub {
5623 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migrate_opts->{migratedfrom
});
5625 die "you can't start a vm if it's a template\n"
5626 if !$params->{skiptemplate
} && PVE
::QemuConfig-
>is_template($conf);
5628 my $has_suspended_lock = PVE
::QemuConfig-
>has_lock($conf, 'suspended');
5629 my $has_backup_lock = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5631 my $running = check_running
($vmid, undef, $migrate_opts->{migratedfrom
});
5633 if ($has_backup_lock && $running) {
5634 # a backup is currently running, attempt to start the guest in the
5635 # existing QEMU instance
5636 return vm_resume
($vmid);
5639 PVE
::QemuConfig-
>check_lock($conf)
5640 if !($params->{skiplock
} || $has_suspended_lock);
5642 $params->{resume
} = $has_suspended_lock || defined($conf->{vmstate
});
5644 die "VM $vmid already running\n" if $running;
5646 if (my $storagemap = $migrate_opts->{storagemap
}) {
5647 my $replicated = $migrate_opts->{replicated_volumes
};
5648 my $disks = vm_migrate_get_nbd_disks
($storecfg, $conf, $replicated);
5649 $migrate_opts->{nbd
} = vm_migrate_alloc_nbd_disks
($storecfg, $vmid, $disks, $storagemap);
5651 foreach my $opt (keys %{$migrate_opts->{nbd
}}) {
5652 $conf->{$opt} = $migrate_opts->{nbd
}->{$opt}->{drivestr
};
5656 return vm_start_nolock
($storecfg, $vmid, $conf, $params, $migrate_opts);
5662 # statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5663 # skiplock => 0/1, skip checking for config lock
5664 # skiptemplate => 0/1, skip checking whether VM is template
5665 # forcemachine => to force QEMU machine (rollback/migration)
5666 # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5667 # timeout => in seconds
5668 # paused => start VM in paused state (backup)
5669 # resume => resume from hibernation
5680 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5681 # migratedfrom => source node
5682 # spice_ticket => used for spice migration, passed via tunnel/stdin
5683 # network => CIDR of migration network
5684 # type => secure/insecure - tunnel over encrypted connection or plain-text
5685 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5686 # replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5687 # offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5688 # contained in config
5689 sub vm_start_nolock
{
5690 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5692 my $statefile = $params->{statefile
};
5693 my $resume = $params->{resume
};
5695 my $migratedfrom = $migrate_opts->{migratedfrom
};
5696 my $migration_type = $migrate_opts->{type
};
5700 # clean up leftover reboot request files
5701 eval { clear_reboot_request
($vmid); };
5704 if (!$statefile && scalar(keys %{$conf->{pending
}})) {
5705 vmconfig_apply_pending
($vmid, $conf, $storecfg);
5706 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5709 # don't regenerate the ISO if the VM is started as part of a live migration
5710 # this way we can reuse the old ISO with the correct config
5711 if (!$migratedfrom) {
5712 if (PVE
::QemuServer
::Cloudinit
::apply_cloudinit_config
($conf, $vmid)) {
5713 # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5714 # $conf->{cloudinit}, so we could just not do this?
5715 # But we do it above, so for now let's be consistent.
5716 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5720 # override offline migrated volumes, conf is out of date still
5721 if (my $offline_volumes = $migrate_opts->{offline_volumes
}) {
5722 for my $key (sort keys $offline_volumes->%*) {
5723 my $parsed = parse_drive
($key, $conf->{$key});
5724 $parsed->{file
} = $offline_volumes->{$key};
5725 $conf->{$key} = print_drive
($parsed);
5729 my $defaults = load_defaults
();
5731 # set environment variable useful inside network script
5732 # for remote migration the config is available on the target node!
5733 if (!$migrate_opts->{remote_node
}) {
5734 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom;
5737 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5739 my $forcemachine = $params->{forcemachine
};
5740 my $forcecpu = $params->{forcecpu
};
5742 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5743 $forcemachine = $conf->{runningmachine
};
5744 $forcecpu = $conf->{runningcpu
};
5745 print "Resuming suspended VM\n";
5748 my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command
($storecfg, $vmid,
5749 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5752 my $get_migration_ip = sub {
5753 my ($nodename) = @_;
5755 return $migration_ip if defined($migration_ip);
5757 my $cidr = $migrate_opts->{network
};
5759 if (!defined($cidr)) {
5760 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5761 $cidr = $dc_conf->{migration
}->{network
};
5764 if (defined($cidr)) {
5765 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5767 die "could not get IP: no address configured on local " .
5768 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5770 die "could not get IP: multiple addresses configured on local " .
5771 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5773 $migration_ip = @$ips[0];
5776 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5777 if !defined($migration_ip);
5779 return $migration_ip;
5783 if ($statefile eq 'tcp') {
5784 my $migrate = $res->{migrate
} = { proto
=> 'tcp' };
5785 $migrate->{addr
} = "localhost";
5786 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5787 my $nodename = nodename
();
5789 if (!defined($migration_type)) {
5790 if (defined($datacenterconf->{migration
}->{type
})) {
5791 $migration_type = $datacenterconf->{migration
}->{type
};
5793 $migration_type = 'secure';
5797 if ($migration_type eq 'insecure') {
5798 $migrate->{addr
} = $get_migration_ip->($nodename);
5799 $migrate->{addr
} = "[$migrate->{addr}]" if Net
::IP
::ip_is_ipv6
($migrate->{addr
});
5802 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5803 $migrate->{port
} = PVE
::Tools
::next_migrate_port
($pfamily);
5804 $migrate->{uri
} = "tcp:$migrate->{addr}:$migrate->{port}";
5805 push @$cmd, '-incoming', $migrate->{uri
};
5808 } elsif ($statefile eq 'unix') {
5809 # should be default for secure migrations as a ssh TCP forward
5810 # tunnel is not deterministic reliable ready and fails regurarly
5811 # to set up in time, so use UNIX socket forwards
5812 my $migrate = $res->{migrate
} = { proto
=> 'unix' };
5813 $migrate->{addr
} = "/run/qemu-server/$vmid.migrate";
5814 unlink $migrate->{addr
};
5816 $migrate->{uri
} = "unix:$migrate->{addr}";
5817 push @$cmd, '-incoming', $migrate->{uri
};
5820 } elsif (-e
$statefile) {
5821 push @$cmd, '-loadstate', $statefile;
5823 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5824 push @$vollist, $statefile;
5825 push @$cmd, '-loadstate', $statepath;
5827 } elsif ($params->{paused
}) {
5831 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $resume);
5833 my $pci_reserve_list = [];
5834 for my $device (values $pci_devices->%*) {
5835 next if $device->{mdev
}; # we don't reserve for mdev devices
5836 push $pci_reserve_list->@*, map { $_->{id
} } $device->{ids
}->@*;
5839 # reserve all PCI IDs before actually doing anything with them
5840 PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, $start_timeout);
5844 for my $id (sort keys %$pci_devices) {
5845 my $d = $pci_devices->{$id};
5846 my ($index) = ($id =~ m/^hostpci(\d+)$/);
5849 for my $dev ($d->{ids
}->@*) {
5850 my $info = eval { PVE
::QemuServer
::PCI
::prepare_pci_device
($vmid, $dev->{id
}, $index, $d->{mdev
}) };
5853 $chosen_mdev = $info;
5854 last if $chosen_mdev; # if successful, we're done
5860 next if !$d->{mdev
};
5861 die "could not create mediated device\n" if !defined($chosen_mdev);
5863 # nvidia grid needs the uuid of the mdev as qemu parameter
5864 if (!defined($uuid) && $chosen_mdev->{vendor
} =~ m/^(0x)?10de$/) {
5865 if (defined($conf->{smbios1
})) {
5866 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
5867 $uuid = $smbios_conf->{uuid
} if defined($smbios_conf->{uuid
});
5869 $uuid = PVE
::QemuServer
::PCI
::generate_mdev_uuid
($vmid, $index) if !defined($uuid);
5872 push @$cmd, '-uuid', $uuid if defined($uuid);
5875 eval { cleanup_pci_devices
($vmid, $conf) };
5880 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5883 my %silence_std_outs = (outfunc
=> sub {}, errfunc
=> sub {});
5884 eval { run_command
(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5885 eval { run_command
(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5886 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5887 # timeout should be more than enough here...
5888 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 20);
5890 my $cpuunits = PVE
::CGroup
::clamp_cpu_shares
($conf->{cpuunits
});
5893 timeout
=> $statefile ?
undef : $start_timeout,
5898 # when migrating, prefix QEMU output so other side can pick up any
5899 # errors that might occur and show the user
5900 if ($migratedfrom) {
5901 $run_params{quiet
} = 1;
5902 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5905 my %systemd_properties = (
5906 Slice
=> 'qemu.slice',
5907 KillMode
=> 'process',
5909 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5912 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5913 $systemd_properties{CPUWeight
} = $cpuunits;
5915 $systemd_properties{CPUShares
} = $cpuunits;
5918 if (my $cpulimit = $conf->{cpulimit
}) {
5919 $systemd_properties{CPUQuota
} = int($cpulimit * 100);
5921 $systemd_properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5923 my $run_qemu = sub {
5924 PVE
::Tools
::run_fork
sub {
5925 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5928 if ((my $tpm = $conf->{tpmstate0
}) && !PVE
::QemuConfig-
>is_template($conf)) {
5929 # start the TPM emulator so QEMU can connect on start
5930 $tpmpid = start_swtpm
($storecfg, $vmid, $tpm, $migratedfrom);
5933 my $exitcode = run_command
($cmd, %run_params);
5936 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5937 kill 'TERM', $tpmpid;
5939 die "QEMU exited with code $exitcode\n";
5944 if ($conf->{hugepages
}) {
5947 my $hotplug_features =
5948 parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
5949 my $hugepages_topology =
5950 PVE
::QemuServer
::Memory
::hugepages_topology
($conf, $hotplug_features->{memory
});
5952 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5954 PVE
::QemuServer
::Memory
::hugepages_mount
();
5955 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5957 eval { $run_qemu->() };
5959 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5960 if !$conf->{keephugepages
};
5964 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5965 if !$conf->{keephugepages
};
5967 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5970 eval { $run_qemu->() };
5974 # deactivate volumes if start fails
5975 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5977 eval { cleanup_pci_devices
($vmid, $conf) };
5980 die "start failed: $err";
5983 # re-reserve all PCI IDs now that we can know the actual VM PID
5984 my $pid = PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
5985 eval { PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_reserve_list, $vmid, undef, $pid) };
5988 if (defined($res->{migrate
})) {
5989 print "migration listens on $res->{migrate}->{uri}\n";
5990 } elsif ($statefile) {
5991 eval { mon_cmd
($vmid, "cont"); };
5995 #start nbd server for storage migration
5996 if (my $nbd = $migrate_opts->{nbd
}) {
5997 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5999 my $migrate_storage_uri;
6000 # nbd_protocol_version > 0 for unix socket support
6001 if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
6002 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
6003 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
6004 $migrate_storage_uri = "nbd:unix:$socket_path";
6005 $res->{migrate
}->{unix_sockets
} = [$socket_path];
6007 my $nodename = nodename
();
6008 my $localip = $get_migration_ip->($nodename);
6009 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
6010 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
6012 mon_cmd
($vmid, "nbd-server-start", addr
=> {
6015 host
=> "${localip}",
6016 port
=> "${storage_migrate_port}",
6019 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
6020 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
6023 my $block_info = mon_cmd
($vmid, "query-block");
6024 $block_info = { map { $_->{device
} => $_ } $block_info->@* };
6026 foreach my $opt (sort keys %$nbd) {
6027 my $drivestr = $nbd->{$opt}->{drivestr
};
6028 my $volid = $nbd->{$opt}->{volid
};
6030 my $block_node = $block_info->{"drive-$opt"}->{inserted
}->{'node-name'};
6036 'node-name' => $block_node,
6037 writable
=> JSON
::true
,
6039 name
=> "drive-$opt", # NBD export name
6042 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6043 print "storage migration listens on $nbd_uri volume:$drivestr\n";
6044 print "re-using replicated volume: $opt - $volid\n"
6045 if $nbd->{$opt}->{replicated
};
6047 $res->{drives
}->{$opt} = $nbd->{$opt};
6048 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
6052 if ($migratedfrom) {
6054 set_migration_caps
($vmid);
6059 print "spice listens on port $spice_port\n";
6060 $res->{spice_port
} = $spice_port;
6061 if ($migrate_opts->{spice_ticket
}) {
6062 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
6063 $migrate_opts->{spice_ticket
});
6064 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
6069 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
6070 if !$statefile && $conf->{balloon
};
6072 foreach my $opt (keys %$conf) {
6073 next if $opt !~ m/^net\d+$/;
6074 my $nicconf = parse_net
($conf->{$opt});
6075 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
6077 add_nets_bridge_fdb
($conf, $vmid);
6080 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
6085 path
=> "machine/peripheral/balloon0",
6086 property
=> "guest-stats-polling-interval",
6090 log_warn
("could not set polling interval for ballooning - $@") if $@;
6094 print "Resumed VM, removing state\n";
6095 if (my $vmstate = $conf->{vmstate
}) {
6096 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6097 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6099 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6100 PVE
::QemuConfig-
>write_config($vmid, $conf);
6103 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
6108 sub vm_commandline
{
6109 my ($storecfg, $vmid, $snapname) = @_;
6111 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6113 my ($forcemachine, $forcecpu);
6115 my $snapshot = $conf->{snapshots
}->{$snapname};
6116 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6118 # check for machine or CPU overrides in snapshot
6119 $forcemachine = $snapshot->{runningmachine
};
6120 $forcecpu = $snapshot->{runningcpu
};
6122 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
6127 my $defaults = load_defaults
();
6129 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6131 return PVE
::Tools
::cmd2string
($cmd);
6135 my ($vmid, $skiplock) = @_;
6137 PVE
::QemuConfig-
>lock_config($vmid, sub {
6139 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6141 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6143 mon_cmd
($vmid, "system_reset");
6147 sub get_vm_volumes
{
6151 foreach_volid
($conf, sub {
6152 my ($volid, $attr) = @_;
6154 return if $volid =~ m
|^/|;
6156 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6159 push @$vollist, $volid;
6165 sub cleanup_pci_devices
{
6166 my ($vmid, $conf) = @_;
6168 foreach my $key (keys %$conf) {
6169 next if $key !~ m/^hostpci(\d+)$/;
6170 my $hostpciindex = $1;
6171 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
6172 my $d = parse_hostpci
($conf->{$key});
6174 # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6175 # don't want to break ABI just for this two liner
6176 my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6178 # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6179 # out when we do it first. so wait for 10 seconds and then try it
6180 if ($d->{ids
}->[0]->[0]->{vendor
} =~ m/^(0x)?10de$/) {
6184 PVE
::SysFSTools
::file_write
("$dev_sysfs_dir/remove", "1") if -e
$dev_sysfs_dir;
6187 PVE
::QemuServer
::PCI
::remove_pci_reservation
($vmid);
6190 sub vm_stop_cleanup
{
6191 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6196 my $vollist = get_vm_volumes
($conf);
6197 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6199 if (my $tpmdrive = $conf->{tpmstate0
}) {
6200 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
6201 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
6203 PVE
::Storage
::unmap_volume
($storecfg, $tpm->{file
});
6208 foreach my $ext (qw(mon qmp pid vnc qga)) {
6209 unlink "/var/run/qemu-server/${vmid}.$ext";
6212 if ($conf->{ivshmem
}) {
6213 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
6214 # just delete it for now, VMs which have this already open do not
6215 # are affected, but new VMs will get a separated one. If this
6216 # becomes an issue we either add some sort of ref-counting or just
6217 # add a "don't delete on stop" flag to the ivshmem format.
6218 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
6221 cleanup_pci_devices
($vmid, $conf);
6223 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
6225 warn $@ if $@; # avoid errors - just warn
6228 # call only in locked context
6230 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6232 my $pid = check_running
($vmid, $nocheck);
6237 $conf = PVE
::QemuConfig-
>load_config($vmid);
6238 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
6239 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
6240 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
6241 $timeout = $opts->{down
} if $opts->{down
};
6243 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
6248 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
6249 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
6251 mon_cmd
($vmid, "system_powerdown");
6254 mon_cmd
($vmid, "quit");
6260 $timeout = 60 if !defined($timeout);
6263 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6268 if ($count >= $timeout) {
6270 warn "VM still running - terminating now with SIGTERM\n";
6273 die "VM quit/powerdown failed - got timeout\n";
6276 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6280 if (!check_running
($vmid, $nocheck)) {
6281 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6285 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6288 die "VM quit/powerdown failed\n";
6296 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
6301 if ($count >= $timeout) {
6302 warn "VM still running - terminating now with SIGKILL\n";
6307 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6310 # Note: use $nocheck to skip tests if VM configuration file exists.
6311 # We need that when migration VMs to other nodes (files already moved)
6312 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6314 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6316 $force = 1 if !defined($force) && !$shutdown;
6319 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
6320 kill 15, $pid if $pid;
6321 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
6322 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
6326 PVE
::QemuConfig-
>lock_config($vmid, sub {
6327 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6332 my ($vmid, $timeout) = @_;
6334 PVE
::QemuConfig-
>lock_config($vmid, sub {
6337 # only reboot if running, as qmeventd starts it again on a stop event
6338 return if !check_running
($vmid);
6340 create_reboot_request
($vmid);
6342 my $storecfg = PVE
::Storage
::config
();
6343 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
6347 # avoid that the next normal shutdown will be confused for a reboot
6348 clear_reboot_request
($vmid);
6354 # note: if using the statestorage parameter, the caller has to check privileges
6356 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6363 PVE
::QemuConfig-
>lock_config($vmid, sub {
6365 $conf = PVE
::QemuConfig-
>load_config($vmid);
6367 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
6368 PVE
::QemuConfig-
>check_lock($conf)
6369 if !($skiplock || $is_backing_up);
6371 die "cannot suspend to disk during backup\n"
6372 if $is_backing_up && $includestate;
6374 if ($includestate) {
6375 $conf->{lock} = 'suspending';
6376 my $date = strftime
("%Y-%m-%d", localtime(time()));
6377 $storecfg = PVE
::Storage
::config
();
6378 if (!$statestorage) {
6379 $statestorage = find_vmstate_storage
($conf, $storecfg);
6380 # check permissions for the storage
6381 my $rpcenv = PVE
::RPCEnvironment
::get
();
6382 if ($rpcenv->{type
} ne 'cli') {
6383 my $authuser = $rpcenv->get_user();
6384 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6389 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
6390 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6391 $path = PVE
::Storage
::path
($storecfg, $vmstate);
6392 PVE
::QemuConfig-
>write_config($vmid, $conf);
6394 mon_cmd
($vmid, "stop");
6398 if ($includestate) {
6400 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
6403 set_migration_caps
($vmid, 1);
6404 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
6406 my $state = mon_cmd
($vmid, "query-savevm");
6407 if (!$state->{status
}) {
6408 die "savevm not active\n";
6409 } elsif ($state->{status
} eq 'active') {
6412 } elsif ($state->{status
} eq 'completed') {
6413 print "State saved, quitting\n";
6415 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
6416 die "query-savevm failed with error '$state->{error}'\n"
6418 die "query-savevm returned status '$state->{status}'\n";
6424 PVE
::QemuConfig-
>lock_config($vmid, sub {
6425 $conf = PVE
::QemuConfig-
>load_config($vmid);
6427 # cleanup, but leave suspending lock, to indicate something went wrong
6429 mon_cmd
($vmid, "savevm-end");
6430 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6431 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6432 delete $conf->@{qw(vmstate runningmachine runningcpu)};
6433 PVE
::QemuConfig-
>write_config($vmid, $conf);
6439 die "lock changed unexpectedly\n"
6440 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
6442 mon_cmd
($vmid, "quit");
6443 $conf->{lock} = 'suspended';
6444 PVE
::QemuConfig-
>write_config($vmid, $conf);
6449 # $nocheck is set when called as part of a migration - in this context the
6450 # location of the config file (source or target node) is not deterministic,
6451 # since migration cannot wait for pmxcfs to process the rename
6453 my ($vmid, $skiplock, $nocheck) = @_;
6455 PVE
::QemuConfig-
>lock_config($vmid, sub {
6456 my $res = mon_cmd
($vmid, 'query-status');
6457 my $resume_cmd = 'cont';
6461 $conf = eval { PVE
::QemuConfig-
>load_config($vmid) }; # try on target node
6463 my $vmlist = PVE
::Cluster
::get_vmlist
();
6464 if (exists($vmlist->{ids
}->{$vmid})) {
6465 my $node = $vmlist->{ids
}->{$vmid}->{node
};
6466 $conf = eval { PVE
::QemuConfig-
>load_config($vmid, $node) }; # try on source node
6469 PVE
::Cluster
::cfs_update
(); # vmlist was wrong, invalidate cache
6470 $conf = PVE
::QemuConfig-
>load_config($vmid); # last try on target node again
6474 $conf = PVE
::QemuConfig-
>load_config($vmid);
6477 if ($res->{status
}) {
6478 return if $res->{status
} eq 'running'; # job done, go home
6479 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
6480 $reset = 1 if $res->{status
} eq 'shutdown';
6484 PVE
::QemuConfig-
>check_lock($conf)
6485 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
6489 # required if a VM shuts down during a backup and we get a resume
6490 # request before the backup finishes for example
6491 mon_cmd
($vmid, "system_reset");
6494 add_nets_bridge_fdb
($conf, $vmid) if $resume_cmd eq 'cont';
6496 mon_cmd
($vmid, $resume_cmd);
6501 my ($vmid, $skiplock, $key) = @_;
6503 PVE
::QemuConfig-
>lock_config($vmid, sub {
6505 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6507 # there is no qmp command, so we use the human monitor command
6508 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
6509 die $res if $res ne '';
6513 sub check_bridge_access
{
6514 my ($rpcenv, $authuser, $conf) = @_;
6516 return 1 if $authuser eq 'root@pam';
6518 for my $opt (sort keys $conf->%*) {
6519 next if $opt !~ m/^net\d+$/;
6520 my $net = parse_net
($conf->{$opt});
6521 my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6522 PVE
::GuestHelpers
::check_vnet_access
($rpcenv, $authuser, $bridge, $tag, $trunks);
6527 sub check_mapping_access
{
6528 my ($rpcenv, $user, $conf) = @_;
6530 for my $opt (keys $conf->%*) {
6531 if ($opt =~ m/^usb\d+$/) {
6532 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-usb', $conf->{$opt});
6533 if (my $host = $device->{host
}) {
6534 die "only root can set '$opt' config for real devices\n"
6535 if $host !~ m/^spice$/i && $user ne 'root@pam';
6536 } elsif ($device->{mapping
}) {
6537 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6539 die "either 'host' or 'mapping' must be set.\n";
6541 } elsif ($opt =~ m/^hostpci\d+$/) {
6542 my $device = PVE
::JSONSchema
::parse_property_string
('pve-qm-hostpci', $conf->{$opt});
6543 if ($device->{host
}) {
6544 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6545 } elsif ($device->{mapping
}) {
6546 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6548 die "either 'host' or 'mapping' must be set.\n";
6554 sub check_restore_permissions
{
6555 my ($rpcenv, $user, $conf) = @_;
6557 check_bridge_access
($rpcenv, $user, $conf);
6558 check_mapping_access
($rpcenv, $user, $conf);
6560 # vzdump restore implementaion
6562 sub tar_archive_read_firstfile
{
6563 my $archive = shift;
6565 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
6567 # try to detect archive type first
6568 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6569 die "unable to open file '$archive'\n";
6570 my $firstfile = <$fh>;
6574 die "ERROR: archive contaions no data\n" if !$firstfile;
6580 sub tar_restore_cleanup
{
6581 my ($storecfg, $statfile) = @_;
6583 print STDERR
"starting cleanup\n";
6585 if (my $fd = IO
::File-
>new($statfile, "r")) {
6586 while (defined(my $line = <$fd>)) {
6587 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6590 if ($volid =~ m
|^/|) {
6591 unlink $volid || die 'unlink failed\n';
6593 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6595 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6597 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6599 print STDERR
"unable to parse line in statfile - $line";
6606 sub restore_file_archive
{
6607 my ($archive, $vmid, $user, $opts) = @_;
6609 return restore_vma_archive
($archive, $vmid, $user, $opts)
6612 my $info = PVE
::Storage
::archive_info
($archive);
6613 my $format = $opts->{format
} // $info->{format
};
6614 my $comp = $info->{compression
};
6616 # try to detect archive format
6617 if ($format eq 'tar') {
6618 return restore_tar_archive
($archive, $vmid, $user, $opts);
6620 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
6624 # hepler to remove disks that will not be used after restore
6625 my $restore_cleanup_oldconf = sub {
6626 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6628 my $kept_disks = {};
6630 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
6631 my ($ds, $drive) = @_;
6633 return if drive_is_cdrom
($drive, 1);
6635 my $volid = $drive->{file
};
6636 return if !$volid || $volid =~ m
|^/|;
6638 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
6639 return if !$path || !$owner || ($owner != $vmid);
6641 # Note: only delete disk we want to restore
6642 # other volumes will become unused
6643 if ($virtdev_hash->{$ds}) {
6644 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
6649 $kept_disks->{$volid} = 1;
6653 # after the restore we have no snapshots anymore
6654 for my $snapname (keys $oldconf->{snapshots
}->%*) {
6655 my $snap = $oldconf->{snapshots
}->{$snapname};
6656 if ($snap->{vmstate
}) {
6657 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
6663 for my $volid (keys $kept_disks->%*) {
6664 eval { PVE
::Storage
::volume_snapshot_delete
($storecfg, $volid, $snapname); };
6670 # Helper to parse vzdump backup device hints
6672 # $rpcenv: Environment, used to ckeck storage permissions
6673 # $user: User ID, to check storage permissions
6674 # $storecfg: Storage configuration
6675 # $fh: the file handle for reading the configuration
6676 # $devinfo: should contain device sizes for all backu-up'ed devices
6677 # $options: backup options (pool, default storage)
6679 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6680 my $parse_backup_hints = sub {
6681 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6683 my $check_storage = sub { # assert if an image can be allocate
6684 my ($storeid, $scfg) = @_;
6685 die "Content type 'images' is not available on storage '$storeid'\n"
6686 if !$scfg->{content
}->{images
};
6687 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6688 if $user ne 'root@pam';
6691 my $virtdev_hash = {};
6692 while (defined(my $line = <$fh>)) {
6693 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6694 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6695 die "archive does not contain data for drive '$virtdev'\n"
6696 if !$devinfo->{$devname};
6698 if (defined($options->{storage
})) {
6699 $storeid = $options->{storage
} || 'local';
6700 } elsif (!$storeid) {
6703 $format = 'raw' if !$format;
6704 $devinfo->{$devname}->{devname
} = $devname;
6705 $devinfo->{$devname}->{virtdev
} = $virtdev;
6706 $devinfo->{$devname}->{format
} = $format;
6707 $devinfo->{$devname}->{storeid
} = $storeid;
6709 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6710 $check_storage->($storeid, $scfg); # permission and content type check
6712 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6713 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6715 my $drive = parse_drive
($virtdev, $2);
6717 if (drive_is_cloudinit
($drive)) {
6718 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6719 $storeid = $options->{storage
} if defined ($options->{storage
});
6720 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6721 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6723 $check_storage->($storeid, $scfg); # permission and content type check
6725 $virtdev_hash->{$virtdev} = {
6727 storeid
=> $storeid,
6728 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6735 return $virtdev_hash;
6738 # Helper to allocate and activate all volumes required for a restore
6740 # $storecfg: Storage configuration
6741 # $virtdev_hash: as returned by parse_backup_hints()
6743 # Returns: { $virtdev => $volid }
6744 my $restore_allocate_devices = sub {
6745 my ($storecfg, $virtdev_hash, $vmid) = @_;
6748 foreach my $virtdev (sort keys %$virtdev_hash) {
6749 my $d = $virtdev_hash->{$virtdev};
6750 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6751 my $storeid = $d->{storeid
};
6752 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6754 # test if requested format is supported
6755 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6756 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6757 $d->{format
} = $defFormat if !$supported;
6760 if ($d->{is_cloudinit
}) {
6761 $name = "vm-$vmid-cloudinit";
6762 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6763 if ($scfg->{path
}) {
6764 $name .= ".$d->{format}";
6768 my $volid = PVE
::Storage
::vdisk_alloc
(
6769 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6771 print STDERR
"new volume ID is '$volid'\n";
6772 $d->{volid
} = $volid;
6774 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6776 $map->{$virtdev} = $volid;
6782 sub restore_update_config_line
{
6783 my ($cookie, $map, $line, $unique) = @_;
6785 return '' if $line =~ m/^\#qmdump\#/;
6786 return '' if $line =~ m/^\#vzdump\#/;
6787 return '' if $line =~ m/^lock:/;
6788 return '' if $line =~ m/^unused\d+:/;
6789 return '' if $line =~ m/^parent:/;
6793 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6794 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6795 # try to convert old 1.X settings
6796 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6797 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6798 my ($model, $macaddr) = split(/\=/, $devconfig);
6799 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6802 bridge
=> "vmbr$ind",
6803 macaddr
=> $macaddr,
6805 my $netstr = print_net
($net);
6807 $res .= "net$cookie->{netcount}: $netstr\n";
6808 $cookie->{netcount
}++;
6810 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6811 my ($id, $netstr) = ($1, $2);
6812 my $net = parse_net
($netstr);
6813 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6814 $netstr = print_net
($net);
6815 $res .= "$id: $netstr\n";
6816 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6819 my $di = parse_drive
($virtdev, $value);
6820 if (defined($di->{backup
}) && !$di->{backup
}) {
6822 } elsif ($map->{$virtdev}) {
6823 delete $di->{format
}; # format can change on restore
6824 $di->{file
} = $map->{$virtdev};
6825 $value = print_drive
($di);
6826 $res .= "$virtdev: $value\n";
6830 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6832 if ($vmgenid ne '0') {
6833 # always generate a new vmgenid if there was a valid one setup
6834 $vmgenid = generate_uuid
();
6836 $res .= "vmgenid: $vmgenid\n";
6837 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6838 my ($uuid, $uuid_str);
6839 UUID
::generate
($uuid);
6840 UUID
::unparse
($uuid, $uuid_str);
6841 my $smbios1 = parse_smbios1
($2);
6842 $smbios1->{uuid
} = $uuid_str;
6843 $res .= $1.print_smbios1
($smbios1)."\n";
6851 my $restore_deactivate_volumes = sub {
6852 my ($storecfg, $virtdev_hash) = @_;
6855 for my $dev (values $virtdev_hash->%*) {
6856 push $vollist->@*, $dev->{volid
} if $dev->{volid
};
6859 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
6860 print STDERR
$@ if $@;
6863 my $restore_destroy_volumes = sub {
6864 my ($storecfg, $virtdev_hash) = @_;
6866 for my $dev (values $virtdev_hash->%*) {
6867 my $volid = $dev->{volid
} or next;
6869 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6870 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6872 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6876 sub restore_merge_config
{
6877 my ($filename, $backup_conf_raw, $override_conf) = @_;
6879 my $backup_conf = parse_vm_config
($filename, $backup_conf_raw);
6880 for my $key (keys $override_conf->%*) {
6881 $backup_conf->{$key} = $override_conf->{$key};
6884 return $backup_conf;
6888 my ($cfg, $vmid) = @_;
6890 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6892 my $volid_hash = {};
6893 foreach my $storeid (keys %$info) {
6894 foreach my $item (@{$info->{$storeid}}) {
6895 next if !($item->{volid
} && $item->{size
});
6896 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6897 $volid_hash->{$item->{volid
}} = $item;
6904 sub update_disk_config
{
6905 my ($vmid, $conf, $volid_hash) = @_;
6908 my $prefix = "VM $vmid";
6910 # used and unused disks
6911 my $referenced = {};
6913 # Note: it is allowed to define multiple storages with same path (alias), so
6914 # we need to check both 'volid' and real 'path' (two different volid can point
6915 # to the same path).
6917 my $referencedpath = {};
6920 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6921 my ($opt, $drive) = @_;
6923 my $volid = $drive->{file
};
6925 my $volume = $volid_hash->{$volid};
6927 # mark volid as "in-use" for next step
6928 $referenced->{$volid} = 1;
6929 if ($volume && (my $path = $volume->{path
})) {
6930 $referencedpath->{$path} = 1;
6933 return if drive_is_cdrom
($drive);
6936 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6937 if (defined($updated)) {
6939 $conf->{$opt} = print_drive
($updated);
6940 print "$prefix ($opt): $msg\n";
6944 # remove 'unusedX' entry if volume is used
6945 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6946 my ($opt, $drive) = @_;
6948 my $volid = $drive->{file
};
6952 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6953 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6954 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6956 delete $conf->{$opt};
6959 $referenced->{$volid} = 1;
6960 $referencedpath->{$path} = 1 if $path;
6963 foreach my $volid (sort keys %$volid_hash) {
6964 next if $volid =~ m/vm-$vmid-state-/;
6965 next if $referenced->{$volid};
6966 my $path = $volid_hash->{$volid}->{path
};
6967 next if !$path; # just to be sure
6968 next if $referencedpath->{$path};
6970 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6971 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6972 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6979 my ($vmid, $nolock, $dryrun) = @_;
6981 my $cfg = PVE
::Storage
::config
();
6983 print "rescan volumes...\n";
6984 my $volid_hash = scan_volids
($cfg, $vmid);
6986 my $updatefn = sub {
6989 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6991 PVE
::QemuConfig-
>check_lock($conf);
6994 foreach my $volid (keys %$volid_hash) {
6995 my $info = $volid_hash->{$volid};
6996 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6999 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
7001 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
7004 if (defined($vmid)) {
7008 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
7011 my $vmlist = config_list
();
7012 foreach my $vmid (keys %$vmlist) {
7016 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
7022 sub restore_proxmox_backup_archive
{
7023 my ($archive, $vmid, $user, $options) = @_;
7025 my $storecfg = PVE
::Storage
::config
();
7027 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
7028 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7030 my $fingerprint = $scfg->{fingerprint
};
7031 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
7033 my $repo = PVE
::PBSClient
::get_repository
($scfg);
7034 my $namespace = $scfg->{namespace
};
7036 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7037 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
7038 local $ENV{PBS_PASSWORD
} = $password;
7039 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
7041 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7042 PVE
::Storage
::parse_volname
($storecfg, $archive);
7044 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7046 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7048 my $tmpdir = "/var/tmp/vzdumptmp$$";
7052 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7053 # disable interrupts (always do cleanups)
7057 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7059 # Note: $oldconf is undef if VM does not exists
7060 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7061 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7062 my $new_conf_raw = '';
7064 my $rpcenv = PVE
::RPCEnvironment
::get
();
7065 my $devinfo = {}; # info about drives included in backup
7066 my $virtdev_hash = {}; # info about allocated drives
7074 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7076 my $cfgfn = "$tmpdir/qemu-server.conf";
7077 my $firewall_config_fn = "$tmpdir/fw.conf";
7078 my $index_fn = "$tmpdir/index.json";
7080 my $cmd = "restore";
7082 my $param = [$pbs_backup_name, "index.json", $index_fn];
7083 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7084 my $index = PVE
::Tools
::file_get_contents
($index_fn);
7085 $index = decode_json
($index);
7087 foreach my $info (@{$index->{files
}}) {
7088 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
7090 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
7091 $devinfo->{$devname}->{size
} = $1;
7093 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7098 my $is_qemu_server_backup = scalar(
7099 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
7101 if (!$is_qemu_server_backup) {
7102 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7104 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
7106 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7107 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7109 if ($has_firewall_config) {
7110 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7111 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
7113 my $pve_firewall_dir = '/etc/pve/firewall';
7114 mkdir $pve_firewall_dir; # make sure the dir exists
7115 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7118 my $fh = IO
::File-
>new($cfgfn, "r") ||
7119 die "unable to read qemu-server.conf - $!\n";
7121 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7123 # fixme: rate limit?
7125 # create empty/temp config
7126 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
7128 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7131 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7133 foreach my $virtdev (sort keys %$virtdev_hash) {
7134 my $d = $virtdev_hash->{$virtdev};
7135 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7137 # this fails if storage is unavailable
7138 my $volid = $d->{volid
};
7139 my $path = PVE
::Storage
::path
($storecfg, $volid);
7141 # for live-restore we only want to preload the efidisk and TPM state
7142 next if $options->{live
} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7145 if (defined(my $ns = $scfg->{namespace
})) {
7146 @ns_arg = ('--ns', $ns);
7149 my $pbs_restore_cmd = [
7150 '/usr/bin/pbs-restore',
7151 '--repository', $repo,
7154 "$d->{devname}.img.fidx",
7159 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
7160 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
7162 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
7163 push @$pbs_restore_cmd, '--skip-zero';
7166 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
7167 print "restore proxmox backup image: $dbg_cmdstring\n";
7168 run_command
($pbs_restore_cmd);
7171 $fh->seek(0, 0) || die "seek failed - $!\n";
7173 my $cookie = { netcount
=> 0 };
7174 while (defined(my $line = <$fh>)) {
7175 $new_conf_raw .= restore_update_config_line
(
7187 if ($err || !$options->{live
}) {
7188 $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7194 $restore_destroy_volumes->($storecfg, $virtdev_hash);
7198 if ($options->{live
}) {
7199 # keep lock during live-restore
7200 $new_conf_raw .= "\nlock: create";
7203 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $options->{override_conf
});
7204 check_restore_permissions
($rpcenv, $user, $new_conf);
7205 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7207 eval { rescan
($vmid, 1); };
7210 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
7212 if ($options->{live
}) {
7218 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
7220 my $conf = PVE
::QemuConfig-
>load_config($vmid);
7221 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
7223 # these special drives are already restored before start
7224 delete $devinfo->{'drive-efidisk0'};
7225 delete $devinfo->{'drive-tpmstate0-backup'};
7229 keyfile
=> $keyfile,
7230 snapshot
=> $pbs_backup_name,
7231 namespace
=> $namespace,
7233 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7235 PVE
::QemuConfig-
>remove_lock($vmid, "create");
7239 sub pbs_live_restore
{
7240 my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7242 print "starting VM for live-restore\n";
7243 print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7245 my $pbs_backing = {};
7246 for my $ds (keys %$restored_disks) {
7247 $ds =~ m/^drive-(.*)$/;
7249 $pbs_backing->{$confname} = {
7250 repository
=> $opts->{repo
},
7251 snapshot
=> $opts->{snapshot
},
7252 archive
=> "$ds.img.fidx",
7254 $pbs_backing->{$confname}->{keyfile
} = $opts->{keyfile
} if -e
$opts->{keyfile
};
7255 $pbs_backing->{$confname}->{namespace
} = $opts->{namespace
} if defined($opts->{namespace
});
7257 my $drive = parse_drive
($confname, $conf->{$confname});
7258 print "restoring '$ds' to '$drive->{file}'\n";
7261 my $drives_streamed = 0;
7263 # make sure HA doesn't interrupt our restore by stopping the VM
7264 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
7265 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7268 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7269 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7270 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'pbs-backing' => $pbs_backing}, {});
7272 my $qmeventd_fd = register_qmeventd_handle
($vmid);
7274 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7275 # this will effectively collapse the backing image chain consisting of
7276 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7277 # removes itself once all backing images vanish with 'auto-remove=on')
7279 for my $ds (sort keys %$restored_disks) {
7280 my $job_id = "restore-$ds";
7281 mon_cmd
($vmid, 'block-stream',
7282 'job-id' => $job_id,
7285 $jobs->{$job_id} = {};
7288 mon_cmd
($vmid, 'cont');
7289 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
7291 print "restore-drive jobs finished successfully, removing all tracking block devices"
7292 ." to disconnect from Proxmox Backup Server\n";
7294 for my $ds (sort keys %$restored_disks) {
7295 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7298 close($qmeventd_fd);
7304 warn "An error occurred during live-restore: $err\n";
7305 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
7306 die "live-restore failed\n";
7310 sub restore_vma_archive
{
7311 my ($archive, $vmid, $user, $opts, $comp) = @_;
7313 my $readfrom = $archive;
7315 my $cfg = PVE
::Storage
::config
();
7317 my $bwlimit = $opts->{bwlimit
};
7319 my $dbg_cmdstring = '';
7320 my $add_pipe = sub {
7322 push @$commands, $cmd;
7323 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7324 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
7329 if ($archive eq '-') {
7332 # If we use a backup from a PVE defined storage we also consider that
7333 # storage's rate limit:
7334 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
7335 if (defined($volid)) {
7336 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
7337 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
7339 print STDERR
"applying read rate limit: $readlimit\n";
7340 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7341 $add_pipe->($cstream);
7347 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
7348 my $cmd = $info->{decompressor
};
7349 push @$cmd, $readfrom;
7353 my $tmpdir = "/var/tmp/vzdumptmp$$";
7356 # disable interrupts (always do cleanups)
7360 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
7362 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7363 POSIX
::mkfifo
($mapfifo, 0600);
7365 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7367 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7372 my $devinfo = {}; # info about drives included in backup
7373 my $virtdev_hash = {}; # info about allocated drives
7375 my $rpcenv = PVE
::RPCEnvironment
::get
();
7377 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7379 # Note: $oldconf is undef if VM does not exist
7380 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
7381 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
7382 my $new_conf_raw = '';
7386 my $print_devmap = sub {
7387 my $cfgfn = "$tmpdir/qemu-server.conf";
7389 # we can read the config - that is already extracted
7390 my $fh = IO
::File-
>new($cfgfn, "r") ||
7391 die "unable to read qemu-server.conf - $!\n";
7393 my $fwcfgfn = "$tmpdir/qemu-server.fw";
7395 my $pve_firewall_dir = '/etc/pve/firewall';
7396 mkdir $pve_firewall_dir; # make sure the dir exists
7397 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7400 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7402 foreach my $info (values %{$virtdev_hash}) {
7403 my $storeid = $info->{storeid
};
7404 next if defined($storage_limits{$storeid});
7406 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
7407 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7408 $storage_limits{$storeid} = $limit * 1024;
7411 foreach my $devname (keys %$devinfo) {
7412 die "found no device mapping information for device '$devname'\n"
7413 if !$devinfo->{$devname}->{virtdev
};
7416 # create empty/temp config
7418 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
7419 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7423 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7425 # print restore information to $fifofh
7426 foreach my $virtdev (sort keys %$virtdev_hash) {
7427 my $d = $virtdev_hash->{$virtdev};
7428 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7430 my $storeid = $d->{storeid
};
7431 my $volid = $d->{volid
};
7434 if (my $limit = $storage_limits{$storeid}) {
7435 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7438 my $write_zeros = 1;
7439 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
7443 my $path = PVE
::Storage
::path
($cfg, $volid);
7445 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7447 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7450 $fh->seek(0, 0) || die "seek failed - $!\n";
7452 my $cookie = { netcount
=> 0 };
7453 while (defined(my $line = <$fh>)) {
7454 $new_conf_raw .= restore_update_config_line
(
7471 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7472 local $SIG{ALRM
} = sub { die "got timeout\n"; };
7474 $oldtimeout = alarm($timeout);
7481 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7482 my ($dev_id, $size, $devname) = ($1, $2, $3);
7483 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
7484 } elsif ($line =~ m/^CTIME: /) {
7485 # we correctly received the vma config, so we can disable
7486 # the timeout now for disk allocation
7487 alarm($oldtimeout || 0);
7488 $oldtimeout = undef;
7490 print $fifofh "done\n";
7496 print "restore vma archive: $dbg_cmdstring\n";
7497 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
7501 alarm($oldtimeout) if $oldtimeout;
7503 $restore_deactivate_volumes->($cfg, $virtdev_hash);
7505 close($fifofh) if $fifofh;
7510 $restore_destroy_volumes->($cfg, $virtdev_hash);
7514 my $new_conf = restore_merge_config
($conffile, $new_conf_raw, $opts->{override_conf
});
7515 check_restore_permissions
($rpcenv, $user, $new_conf);
7516 PVE
::QemuConfig-
>write_config($vmid, $new_conf);
7518 eval { rescan
($vmid, 1); };
7521 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
7524 sub restore_tar_archive
{
7525 my ($archive, $vmid, $user, $opts) = @_;
7527 if (scalar(keys $opts->{override_conf
}->%*) > 0) {
7528 my $keystring = join(' ', keys $opts->{override_conf
}->%*);
7529 die "cannot pass along options ($keystring) when restoring from tar archive\n";
7532 if ($archive ne '-') {
7533 my $firstfile = tar_archive_read_firstfile
($archive);
7534 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7535 if $firstfile ne 'qemu-server.conf';
7538 my $storecfg = PVE
::Storage
::config
();
7540 # avoid zombie disks when restoring over an existing VM -> cleanup first
7541 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7542 # skiplock=1 because qmrestore has set the 'create' lock itself already
7543 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
7544 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
7546 my $tocmd = "/usr/lib/qemu-server/qmextract";
7548 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
7549 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
7550 $tocmd .= ' --prealloc' if $opts->{prealloc
};
7551 $tocmd .= ' --info' if $opts->{info
};
7553 # tar option "xf" does not autodetect compression when read from STDIN,
7554 # so we pipe to zcat
7555 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
7556 PVE
::Tools
::shellquote
("--to-command=$tocmd");
7558 my $tmpdir = "/var/tmp/vzdumptmp$$";
7561 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
7562 local $ENV{VZDUMP_VMID
} = $vmid;
7563 local $ENV{VZDUMP_USER
} = $user;
7565 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7566 my $new_conf_raw = '';
7568 # disable interrupts (always do cleanups)
7572 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7580 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7582 if ($archive eq '-') {
7583 print "extracting archive from STDIN\n";
7584 run_command
($cmd, input
=> "<&STDIN");
7586 print "extracting archive '$archive'\n";
7590 return if $opts->{info
};
7594 my $statfile = "$tmpdir/qmrestore.stat";
7595 if (my $fd = IO
::File-
>new($statfile, "r")) {
7596 while (defined (my $line = <$fd>)) {
7597 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7598 $map->{$1} = $2 if $1;
7600 print STDERR
"unable to parse line in statfile - $line\n";
7606 my $confsrc = "$tmpdir/qemu-server.conf";
7608 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7610 my $cookie = { netcount
=> 0 };
7611 while (defined (my $line = <$srcfd>)) {
7612 $new_conf_raw .= restore_update_config_line
(
7623 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
7629 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7631 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7633 eval { rescan
($vmid, 1); };
7637 sub foreach_storage_used_by_vm
{
7638 my ($conf, $func) = @_;
7642 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7643 my ($ds, $drive) = @_;
7644 return if drive_is_cdrom
($drive);
7646 my $volid = $drive->{file
};
7648 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
7649 $sidhash->{$sid} = $sid if $sid;
7652 foreach my $sid (sort keys %$sidhash) {
7657 my $qemu_snap_storage = {
7660 sub do_snapshots_with_qemu
{
7661 my ($storecfg, $volid, $deviceid) = @_;
7663 return if $deviceid =~ m/tpmstate0/;
7665 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
7666 my $scfg = $storecfg->{ids
}->{$storage_name};
7667 die "could not find storage '$storage_name'\n" if !defined($scfg);
7669 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
7673 if ($volid =~ m/\.(qcow2|qed)$/){
7680 sub qga_check_running
{
7681 my ($vmid, $nowarn) = @_;
7683 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
7685 warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7691 sub template_create
{
7692 my ($vmid, $conf, $disk) = @_;
7694 my $storecfg = PVE
::Storage
::config
();
7696 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7697 my ($ds, $drive) = @_;
7699 return if drive_is_cdrom
($drive);
7700 return if $disk && $ds ne $disk;
7702 my $volid = $drive->{file
};
7703 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
7705 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
7706 $drive->{file
} = $voliddst;
7707 $conf->{$ds} = print_drive
($drive);
7708 PVE
::QemuConfig-
>write_config($vmid, $conf);
7712 sub convert_iscsi_path
{
7715 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
7720 my $initiator_name = get_initiator_name
();
7722 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7723 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7726 die "cannot convert iscsi path '$path', unkown format\n";
7729 sub qemu_img_convert
{
7730 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7732 my $storecfg = PVE
::Storage
::config
();
7733 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
7734 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
7736 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7740 my $src_is_iscsi = 0;
7744 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
7745 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7746 $src_format = qemu_img_format
($src_scfg, $src_volname);
7747 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
7748 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
7749 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
7750 } elsif (-f
$src_volid || -b
$src_volid) {
7751 $src_path = $src_volid;
7752 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7757 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7759 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7760 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7761 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7762 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7765 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7766 push @$cmd, '-l', "snapshot.name=$snapname"
7767 if $snapname && $src_format && $src_format eq "qcow2";
7768 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7769 push @$cmd, '-T', $cachemode if defined($cachemode);
7770 push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7772 if ($src_is_iscsi) {
7773 push @$cmd, '--image-opts';
7774 $src_path = convert_iscsi_path
($src_path);
7775 } elsif ($src_format) {
7776 push @$cmd, '-f', $src_format;
7779 if ($dst_is_iscsi) {
7780 push @$cmd, '--target-image-opts';
7781 $dst_path = convert_iscsi_path
($dst_path);
7783 push @$cmd, '-O', $dst_format;
7786 push @$cmd, $src_path;
7788 if (!$dst_is_iscsi && $is_zero_initialized) {
7789 push @$cmd, "zeroinit:$dst_path";
7791 push @$cmd, $dst_path;
7796 if($line =~ m/\((\S+)\/100\
%\)/){
7798 my $transferred = int($size * $percent / 100);
7799 my $total_h = render_bytes
($size, 1);
7800 my $transferred_h = render_bytes
($transferred, 1);
7802 print "transferred $transferred_h of $total_h ($percent%)\n";
7807 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7809 die "copy failed: $err" if $err;
7812 sub qemu_img_format
{
7813 my ($scfg, $volname) = @_;
7815 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7822 sub qemu_drive_mirror
{
7823 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7825 $jobs = {} if !$jobs;
7829 $jobs->{"drive-$drive"} = {};
7831 if ($dst_volid =~ /^nbd:/) {
7832 $qemu_target = $dst_volid;
7835 my $storecfg = PVE
::Storage
::config
();
7836 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7838 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7840 $format = qemu_img_format
($dst_scfg, $dst_volname);
7842 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7844 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7847 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7848 $opts->{format
} = $format if $format;
7850 if (defined($src_bitmap)) {
7851 $opts->{sync
} = 'incremental';
7852 $opts->{bitmap
} = $src_bitmap;
7853 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7856 if (defined($bwlimit)) {
7857 $opts->{speed
} = $bwlimit * 1024;
7858 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7860 print "drive mirror is starting for drive-$drive\n";
7863 # if a job already runs for this device we get an error, catch it for cleanup
7864 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7866 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7868 die "mirroring error: $err\n";
7871 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7874 # $completion can be either
7875 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7876 # 'cancel': wait until all jobs are ready, block-job-cancel them
7877 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7878 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7879 sub qemu_drive_mirror_monitor
{
7880 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7882 $completion //= 'complete';
7886 my $err_complete = 0;
7888 my $starttime = time ();
7890 die "block job ('$op') timed out\n" if $err_complete > 300;
7892 my $stats = mon_cmd
($vmid, "query-block-jobs");
7895 my $running_jobs = {};
7896 for my $stat (@$stats) {
7897 next if $stat->{type
} ne $op;
7898 $running_jobs->{$stat->{device
}} = $stat;
7901 my $readycounter = 0;
7903 for my $job_id (sort keys %$jobs) {
7904 my $job = $running_jobs->{$job_id};
7906 my $vanished = !defined($job);
7907 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7908 if($complete || ($vanished && $completion eq 'auto')) {
7909 print "$job_id: $op-job finished\n";
7910 delete $jobs->{$job_id};
7914 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7916 my $busy = $job->{busy
};
7917 my $ready = $job->{ready
};
7918 if (my $total = $job->{len
}) {
7919 my $transferred = $job->{offset
} || 0;
7920 my $remaining = $total - $transferred;
7921 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7923 my $duration = $ctime - $starttime;
7924 my $total_h = render_bytes
($total, 1);
7925 my $transferred_h = render_bytes
($transferred, 1);
7927 my $status = sprintf(
7928 "transferred $transferred_h of $total_h ($percent%%) in %s",
7929 render_duration
($duration),
7934 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7936 $status .= ", ready";
7939 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
7940 $jobs->{$job_id}->{ready
} = $ready;
7943 $readycounter++ if $job->{ready
};
7946 last if scalar(keys %$jobs) == 0;
7948 if ($readycounter == scalar(keys %$jobs)) {
7949 print "all '$op' jobs are ready\n";
7951 # do the complete later (or has already been done)
7952 last if $completion eq 'skip' || $completion eq 'auto';
7954 if ($vmiddst && $vmiddst != $vmid) {
7955 my $agent_running = $qga && qga_check_running
($vmid);
7956 if ($agent_running) {
7957 print "freeze filesystem\n";
7958 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
7961 print "suspend vm\n";
7962 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
7966 # if we clone a disk for a new target vm, we don't switch the disk
7967 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
7969 if ($agent_running) {
7970 print "unfreeze filesystem\n";
7971 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
7974 print "resume vm\n";
7975 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
7982 for my $job_id (sort keys %$jobs) {
7983 # try to switch the disk if source and destination are on the same guest
7984 print "$job_id: Completing block job_id...\n";
7987 if ($completion eq 'complete') {
7988 $op = 'block-job-complete';
7989 } elsif ($completion eq 'cancel') {
7990 $op = 'block-job-cancel';
7992 die "invalid completion value: $completion\n";
7994 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
7995 if ($@ =~ m/cannot be completed/) {
7996 print "$job_id: block job cannot be completed, trying again.\n";
7999 print "$job_id: Completed successfully.\n";
8000 $jobs->{$job_id}->{complete
} = 1;
8011 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
8012 die "block job ($op) error: $err";
8016 sub qemu_blockjobs_cancel
{
8017 my ($vmid, $jobs) = @_;
8019 foreach my $job (keys %$jobs) {
8020 print "$job: Cancelling block job\n";
8021 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
8022 $jobs->{$job}->{cancel
} = 1;
8026 my $stats = mon_cmd
($vmid, "query-block-jobs");
8028 my $running_jobs = {};
8029 foreach my $stat (@$stats) {
8030 $running_jobs->{$stat->{device
}} = $stat;
8033 foreach my $job (keys %$jobs) {
8035 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
8036 print "$job: Done.\n";
8037 delete $jobs->{$job};
8041 last if scalar(keys %$jobs) == 0;
8047 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8048 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8049 my sub clone_disk_check_io_uring
{
8050 my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8052 return if !$use_drive_mirror;
8054 # Don't complain when not changing storage.
8055 # Assume if it works for the source, it'll work for the target too.
8056 return if $src_storeid eq $dst_storeid;
8058 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
8059 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
8061 my $cache_direct = drive_uses_cache_direct
($src_drive);
8063 my $src_uses_io_uring;
8064 if ($src_drive->{aio
}) {
8065 $src_uses_io_uring = $src_drive->{aio
} eq 'io_uring';
8067 $src_uses_io_uring = storage_allows_io_uring_default
($src_scfg, $cache_direct);
8070 die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8071 if $src_uses_io_uring && !storage_allows_io_uring_default
($dst_scfg, $cache_direct);
8075 my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8077 my ($vmid, $running) = $source->@{qw(vmid running)};
8078 my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8080 my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8081 my ($storage, $format) = $dest->@{qw(storage format)};
8083 my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8085 if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8086 die "cloning from/to EFI disk requires EFI disk\n"
8087 if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8088 die "cloning from/to TPM state requires TPM state\n"
8089 if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8091 # This would lead to two device nodes in QEMU pointing to the same backing image!
8092 die "cannot change drive name when cloning disk from/to the same VM\n"
8093 if $use_drive_mirror && $vmid == $newvmid;
8096 die "cannot move TPM state while VM is running\n"
8097 if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8101 print "create " . ($full ?
'full' : 'linked') . " clone of drive ";
8102 print "$src_drivename " if $src_drivename;
8103 print "($drive->{file})\n";
8106 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
8107 push @$newvollist, $newvolid;
8109 my ($src_storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
8110 my $storeid = $storage || $src_storeid;
8112 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
8116 if (drive_is_cloudinit
($drive)) {
8117 $name = "vm-$newvmid-cloudinit";
8118 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8119 if ($scfg->{path
}) {
8120 $name .= ".$dst_format";
8123 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
8124 } elsif ($dst_drivename eq 'efidisk0') {
8125 $size = $efisize or die "internal error - need to specify EFI disk size\n";
8126 } elsif ($dst_drivename eq 'tpmstate0') {
8127 $dst_format = 'raw';
8128 $size = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8130 clone_disk_check_io_uring
($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8132 $size = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
8134 $newvolid = PVE
::Storage
::vdisk_alloc
(
8135 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8137 push @$newvollist, $newvolid;
8139 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
8141 if (drive_is_cloudinit
($drive)) {
8142 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8143 # if this is the case, we have to complete any block-jobs still there from
8144 # previous drive-mirrors
8145 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8146 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
8151 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
8152 if ($use_drive_mirror) {
8153 qemu_drive_mirror
($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8154 $completion, $qga, $bwlimit);
8156 if ($dst_drivename eq 'efidisk0') {
8157 # the relevant data on the efidisk may be smaller than the source
8158 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8159 # that is given by the OVMF_VARS.fd
8160 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
8161 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
8163 my $src_format = (PVE
::Storage
::parse_volname
($storecfg, $drive->{file
}))[6];
8165 # better for Ceph if block size is not too small, see bug #3324
8168 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8170 if ($src_format eq 'qcow2' && $snapname) {
8171 die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8172 if !min_version
(kvm_user_version
(), 6, 2);
8173 push $cmd->@*, '-l', $snapname;
8175 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8178 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8184 my $size = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
8186 my $disk = dclone
($drive);
8187 delete $disk->{format
};
8188 $disk->{file
} = $newvolid;
8189 $disk->{size
} = $size if defined($size);
8194 sub get_running_qemu_version
{
8196 my $res = mon_cmd
($vmid, "query-version");
8197 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8200 sub qemu_use_old_bios_files
{
8201 my ($machine_type) = @_;
8203 return if !$machine_type;
8205 my $use_old_bios_files = undef;
8207 if ($machine_type =~ m/^(\S+)\.pxe$/) {
8209 $use_old_bios_files = 1;
8211 my $version = extract_version
($machine_type, kvm_user_version
());
8212 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8213 # load new efi bios files on migration. So this hack is required to allow
8214 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8215 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8216 $use_old_bios_files = !min_version
($version, 2, 4);
8219 return ($use_old_bios_files, $machine_type);
8222 sub get_efivars_size
{
8223 my ($conf, $efidisk) = @_;
8225 my $arch = get_vm_arch
($conf);
8226 $efidisk //= $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
8227 my $smm = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
8228 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8229 return -s
$ovmf_vars;
8232 sub update_efidisk_size
{
8235 return if !defined($conf->{efidisk0
});
8237 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
8238 $disk->{size
} = get_efivars_size
($conf);
8239 $conf->{efidisk0
} = print_drive
($disk);
8244 sub update_tpmstate_size
{
8247 my $disk = PVE
::QemuServer
::parse_drive
('tpmstate0', $conf->{tpmstate0
});
8248 $disk->{size
} = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
8249 $conf->{tpmstate0
} = print_drive
($disk);
8252 sub create_efidisk
($$$$$$$) {
8253 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8255 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
8257 my $vars_size_b = -s
$ovmf_vars;
8258 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
8259 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8260 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
8262 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
8263 my $size = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
8265 return ($volid, $size/1024);
8268 sub vm_iothreads_list
{
8271 my $res = mon_cmd
($vmid, 'query-iothreads');
8274 foreach my $iothread (@$res) {
8275 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
8282 my ($conf, $drive) = @_;
8286 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
8288 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
8294 my $controller = int($drive->{index} / $maxdev);
8295 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
8299 return ($maxdev, $controller, $controller_prefix);
8302 sub resolve_dst_disk_format
{
8303 my ($storecfg, $storeid, $src_volname, $format) = @_;
8304 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
8307 # if no target format is specified, use the source disk format as hint
8309 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8310 $format = qemu_img_format
($scfg, $src_volname);
8316 # test if requested format is supported - else use default
8317 my $supported = grep { $_ eq $format } @$validFormats;
8318 $format = $defFormat if !$supported;
8322 # NOTE: if this logic changes, please update docs & possibly gui logic
8323 sub find_vmstate_storage
{
8324 my ($conf, $storecfg) = @_;
8326 # first, return storage from conf if set
8327 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
8329 my ($target, $shared, $local);
8331 foreach_storage_used_by_vm
($conf, sub {
8333 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
8334 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
8335 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
8338 # second, use shared storage where VM has at least one disk
8339 # third, use local storage where VM has at least one disk
8340 # fall back to local storage
8341 $target = $shared // $local // 'local';
8347 my ($uuid, $uuid_str);
8348 UUID
::generate
($uuid);
8349 UUID
::unparse
($uuid, $uuid_str);
8353 sub generate_smbios1_uuid
{
8354 return "uuid=".generate_uuid
();
8360 mon_cmd
($vmid, 'nbd-server-stop');
8363 sub create_reboot_request
{
8365 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8366 or die "failed to create reboot trigger file: $!\n";
8370 sub clear_reboot_request
{
8372 my $path = "/run/qemu-server/$vmid.reboot";
8375 $res = unlink($path);
8376 die "could not remove reboot request for $vmid: $!"
8377 if !$res && $! != POSIX
::ENOENT
;
8382 sub bootorder_from_legacy
{
8383 my ($conf, $bootcfg) = @_;
8385 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
8386 my $bootindex_hash = {};
8388 foreach my $o (split(//, $boot)) {
8389 $bootindex_hash->{$o} = $i*100;
8395 PVE
::QemuConfig-
>foreach_volume($conf, sub {
8396 my ($ds, $drive) = @_;
8398 if (drive_is_cdrom
($drive, 1)) {
8399 if ($bootindex_hash->{d
}) {
8400 $bootorder->{$ds} = $bootindex_hash->{d
};
8401 $bootindex_hash->{d
} += 1;
8403 } elsif ($bootindex_hash->{c
}) {
8404 $bootorder->{$ds} = $bootindex_hash->{c
}
8405 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
8406 $bootindex_hash->{c
} += 1;
8410 if ($bootindex_hash->{n
}) {
8411 for (my $i = 0; $i < $MAX_NETS; $i++) {
8412 my $netname = "net$i";
8413 next if !$conf->{$netname};
8414 $bootorder->{$netname} = $bootindex_hash->{n
};
8415 $bootindex_hash->{n
} += 1;
8422 # Generate default device list for 'boot: order=' property. Matches legacy
8423 # default boot order, but with explicit device names. This is important, since
8424 # the fallback for when neither 'order' nor the old format is specified relies
8425 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8426 sub get_default_bootdevices
{
8432 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
8433 push @ret, $first if $first;
8436 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
8437 push @ret, $first if $first;
8440 for (my $i = 0; $i < $MAX_NETS; $i++) {
8441 my $netname = "net$i";
8442 next if !$conf->{$netname};
8443 push @ret, $netname;
8450 sub device_bootorder
{
8453 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
8455 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
8458 if (!defined($boot) || $boot->{legacy
}) {
8459 $bootorder = bootorder_from_legacy
($conf, $boot);
8460 } elsif ($boot->{order
}) {
8461 my $i = 100; # start at 100 to allow user to insert devices before us with -args
8462 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
8463 $bootorder->{$dev} = $i++;
8470 sub register_qmeventd_handle
{
8474 my $peer = "/var/run/qmeventd.sock";
8479 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
8481 if ($! != EINTR
&& $! != EAGAIN
) {
8482 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8485 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8486 . "after $count retries\n";
8491 # send handshake to mark VM as backing up
8492 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
8494 # return handle to be closed later when inhibit is no longer required
8498 # bash completion helper
8500 sub complete_backup_archives
{
8501 my ($cmdname, $pname, $cvalue) = @_;
8503 my $cfg = PVE
::Storage
::config
();
8507 if ($cvalue =~ m/^([^:]+):/) {
8511 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
8514 foreach my $id (keys %$data) {
8515 foreach my $item (@{$data->{$id}}) {
8516 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8517 push @$res, $item->{volid
} if defined($item->{volid
});
8524 my $complete_vmid_full = sub {
8527 my $idlist = vmstatus
();
8531 foreach my $id (keys %$idlist) {
8532 my $d = $idlist->{$id};
8533 if (defined($running)) {
8534 next if $d->{template
};
8535 next if $running && $d->{status
} ne 'running';
8536 next if !$running && $d->{status
} eq 'running';
8545 return &$complete_vmid_full();
8548 sub complete_vmid_stopped
{
8549 return &$complete_vmid_full(0);
8552 sub complete_vmid_running
{
8553 return &$complete_vmid_full(1);
8556 sub complete_storage
{
8558 my $cfg = PVE
::Storage
::config
();
8559 my $ids = $cfg->{ids
};
8562 foreach my $sid (keys %$ids) {
8563 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
8564 next if !$ids->{$sid}->{content
}->{images
};
8571 sub complete_migration_storage
{
8572 my ($cmd, $param, $current_value, $all_args) = @_;
8574 my $targetnode = @$all_args[1];
8576 my $cfg = PVE
::Storage
::config
();
8577 my $ids = $cfg->{ids
};
8580 foreach my $sid (keys %$ids) {
8581 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
8582 next if !$ids->{$sid}->{content
}->{images
};
8591 my $qmpstatus = eval {
8592 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
8593 mon_cmd
($vmid, "query-status");
8596 return $qmpstatus && $qmpstatus->{status
} eq "paused";
8599 sub check_volume_storage_type
{
8600 my ($storecfg, $vol) = @_;
8602 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
8603 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8604 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
8606 die "storage '$storeid' does not support content-type '$vtype'\n"
8607 if !$scfg->{content
}->{$vtype};
8612 sub add_nets_bridge_fdb
{
8613 my ($conf, $vmid) = @_;
8615 for my $opt (keys %$conf) {
8616 next if $opt !~ m/^net(\d+)$/;
8617 my $iface = "tap${vmid}i$1";
8618 # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8619 my $net = parse_net
($conf->{$opt}, 1) or next;
8621 my $mac = $net->{macaddr
};
8623 log_warn
("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8624 if !file_read_firstline
("/sys/class/net/$iface/brport/learning");
8628 my $bridge = $net->{bridge
};
8630 log_warn
("Interface '$iface' not attached to any bridge.");
8634 PVE
::Network
::SDN
::Zones
::add_bridge_fdb
($iface, $mac, $bridge, $net->{firewall
});
8635 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8636 PVE
::Network
::add_bridge_fdb
($iface, $mac, $net->{firewall
});
8641 sub del_nets_bridge_fdb
{
8642 my ($conf, $vmid) = @_;
8644 for my $opt (keys %$conf) {
8645 next if $opt !~ m/^net(\d+)$/;
8646 my $iface = "tap${vmid}i$1";
8648 my $net = parse_net
($conf->{$opt}) or next;
8649 my $mac = $net->{macaddr
} or next;
8651 my $bridge = $net->{bridge
};
8653 PVE
::Network
::SDN
::Zones
::del_bridge_fdb
($iface, $mac, $bridge, $net->{firewall
});
8654 } elsif (-d
"/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8655 PVE
::Network
::del_bridge_fdb
($iface, $mac, $net->{firewall
});