1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
31 use PVE
::DataCenterConfig
;
32 use PVE
::Exception
qw(raise raise_param_exc);
33 use PVE
::Format
qw(render_duration render_bytes);
34 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
36 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
39 use PVE
::RPCEnvironment
;
43 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
47 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
48 use PVE
::QemuServer
::Cloudinit
;
49 use PVE
::QemuServer
::CGroup
;
50 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
51 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
52 use PVE
::QemuServer
::Machine
;
53 use PVE
::QemuServer
::Memory
;
54 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
55 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
56 use PVE
::QemuServer
::USB
qw(parse_usb_device);
60 require PVE
::Network
::SDN
::Zones
;
64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
67 "$EDK2_FW_BASE/OVMF_CODE.fd",
68 "$EDK2_FW_BASE/OVMF_VARS.fd"
71 "$EDK2_FW_BASE/AAVMF_CODE.fd",
72 "$EDK2_FW_BASE/AAVMF_VARS.fd"
76 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
78 # Note about locking: we use flock on the config file protect
79 # against concurent actions.
80 # Aditionaly, we have a 'lock' setting in the config file. This
81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
82 # allowed when such lock is set. But you can ignore this kind of
83 # lock with the --skiplock flag.
85 cfs_register_file
('/qemu-server/',
89 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
90 description
=> "Some command save/restore state from this location.",
96 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
97 description
=> "Specifies the Qemu machine type.",
99 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
106 my ($map, $source) = @_;
108 return $source if !defined($map);
110 return $map->{entries
}->{$source}
111 if $map->{entries
} && defined($map->{entries
}->{$source});
113 return $map->{default} if $map->{default};
115 # identity (fallback)
119 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
120 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
122 format
=> 'storagepair-list',
126 #no warnings 'redefine';
130 $nodename_cache //= PVE
::INotify
::nodename
();
131 return $nodename_cache;
138 enum
=> [qw(i6300esb ib700)],
139 description
=> "Watchdog type to emulate.",
140 default => 'i6300esb',
145 enum
=> [qw(reset shutdown poweroff pause debug none)],
146 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
150 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
154 description
=> "Enable/disable Qemu GuestAgent.",
159 fstrim_cloned_disks
=> {
160 description
=> "Run fstrim after moving a disk or migrating the VM.",
166 description
=> "Select the agent type",
170 enum
=> [qw(virtio isa)],
176 description
=> "Select the VGA type.",
181 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
184 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
196 description
=> "The size of the file in MB.",
200 pattern
=> '[a-zA-Z0-9\-]+',
202 format_description
=> 'string',
203 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
210 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
211 description
=> "Configure an audio device."
215 enum
=> ['spice', 'none'],
218 description
=> "Driver backend for the audio device."
222 my $spice_enhancements_fmt = {
227 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
231 enum
=> ['off', 'all', 'filter'],
234 description
=> "Enable video streaming. Uses compression for detected video streams."
241 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
243 description
=> "The file on the host to gather entropy from. In most"
244 . " cases /dev/urandom should be preferred over /dev/random"
245 . " to avoid entropy-starvation issues on the host. Using"
246 . " urandom does *not* decrease security in any meaningful"
247 . " way, as it's still seeded from real entropy, and the"
248 . " bytes provided will most likely be mixed with real"
249 . " entropy on the guest as well. /dev/hwrng can be used"
250 . " to pass through a hardware RNG from the host.",
254 description
=> "Maximum bytes of entropy injected into the guest every"
255 . " 'period' milliseconds. Prefer a lower value when using"
256 . " /dev/random as source. Use 0 to disable limiting"
257 . " (potentially dangerous!).",
260 # default is 1 KiB/s, provides enough entropy to the guest to avoid
261 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
262 # of overwhelming the host, provided we're reading from /dev/urandom
267 description
=> "Every 'period' milliseconds the entropy-injection quota"
268 . " is reset, allowing the guest to retrieve another"
269 . " 'max_bytes' of entropy.",
279 description
=> "Specifies whether a VM will be started during system bootup.",
285 description
=> "Automatic restart after crash (currently ignored).",
290 type
=> 'string', format
=> 'pve-hotplug-features',
291 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
292 default => 'network,disk,usb',
297 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
303 description
=> "Lock/unlock the VM.",
304 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
309 description
=> "Limit of CPU usage.",
310 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
318 description
=> "CPU weight for a VM.",
319 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
327 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
334 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
340 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
348 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
349 "It should not be necessary to set it.",
350 enum
=> PVE
::Tools
::kvmkeymaplist
(),
355 type
=> 'string', format
=> 'dns-name',
356 description
=> "Set a name for the VM. Only used on the configuration web interface.",
361 description
=> "SCSI controller model",
362 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
368 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
373 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
374 description
=> "Specify guest operating system.",
375 verbose_description
=> <<EODESC,
376 Specify guest operating system. This is used to enable special
377 optimization/features for specific operating systems:
380 other;; unspecified OS
381 wxp;; Microsoft Windows XP
382 w2k;; Microsoft Windows 2000
383 w2k3;; Microsoft Windows 2003
384 w2k8;; Microsoft Windows 2008
385 wvista;; Microsoft Windows Vista
386 win7;; Microsoft Windows 7
387 win8;; Microsoft Windows 8/2012/2012r2
388 win10;; Microsoft Windows 10/2016/2019
389 l24;; Linux 2.4 Kernel
390 l26;; Linux 2.6 - 5.X Kernel
391 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
396 type
=> 'string', format
=> 'pve-qm-boot',
397 description
=> "Specify guest boot order. Use with 'order=', usage with"
398 . " no key or 'legacy=' is deprecated.",
402 type
=> 'string', format
=> 'pve-qm-bootdisk',
403 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
404 pattern
=> '(ide|sata|scsi|virtio)\d+',
409 description
=> "The number of CPUs. Please use option -sockets instead.",
416 description
=> "The number of CPU sockets.",
423 description
=> "The number of cores per socket.",
430 description
=> "Enable/disable NUMA.",
436 description
=> "Enable/disable hugepages memory.",
437 enum
=> [qw(any 2 1024)],
443 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
444 ." after VM shutdown and can be used for subsequent starts.",
449 description
=> "Number of hotplugged vcpus.",
456 description
=> "Enable/disable ACPI.",
461 description
=> "Enable/disable Qemu GuestAgent and its properties.",
463 format
=> $agent_fmt,
468 description
=> "Enable/disable KVM hardware virtualization.",
474 description
=> "Enable/disable time drift fix.",
480 description
=> "Set the real time clock to local time. This is enabled by default if ostype"
481 ." indicates a Microsoft OS.",
486 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
490 type
=> 'string', format
=> $vga_fmt,
491 description
=> "Configure the VGA hardware.",
492 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
493 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
494 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
495 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
496 ." display server. For win* OS you can select how many independent displays you want,"
497 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
498 ." using a serial device as terminal.",
502 type
=> 'string', format
=> 'pve-qm-watchdog',
503 description
=> "Create a virtual hardware watchdog device.",
504 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
505 ." action), the watchdog must be periodically polled by an agent inside the guest or"
506 ." else the watchdog will reset the guest (or execute the respective action specified)",
511 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
512 description
=> "Set the initial date of the real time clock. Valid format for date are:"
513 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
514 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
517 startup
=> get_standard_option
('pve-startup-order'),
521 description
=> "Enable/disable Template.",
527 description
=> "Arbitrary arguments passed to kvm.",
528 verbose_description
=> <<EODESCR,
529 Arbitrary arguments passed to kvm, for example:
531 args: -no-reboot -no-hpet
533 NOTE: this option is for experts only.
540 description
=> "Enable/disable the USB tablet device.",
541 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
542 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
543 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
544 ." may consider disabling this to save some context switches. This is turned off by"
545 ." default if you use spice (`qm set <vmid> --vga qxl`).",
550 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
554 migrate_downtime
=> {
557 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
563 type
=> 'string', format
=> 'pve-qm-ide',
564 typetext
=> '<volume>',
565 description
=> "This is an alias for option -ide2",
569 description
=> "Emulated CPU type.",
571 format
=> 'pve-vm-cpu-conf',
573 parent
=> get_standard_option
('pve-snapshot-name', {
575 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
579 description
=> "Timestamp for snapshots.",
585 type
=> 'string', format
=> 'pve-volume-id',
586 description
=> "Reference to a volume which stores the VM state. This is used internally"
589 vmstatestorage
=> get_standard_option
('pve-storage-id', {
590 description
=> "Default storage for VM state volumes/files.",
593 runningmachine
=> get_standard_option
('pve-qemu-machine', {
594 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
598 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
599 ." internally for snapshots.",
602 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
603 format_description
=> 'QEMU -cpu parameter'
605 machine
=> get_standard_option
('pve-qemu-machine'),
607 description
=> "Virtual processor architecture. Defaults to the host.",
610 enum
=> [qw(x86_64 aarch64)],
613 description
=> "Specify SMBIOS type 1 fields.",
614 type
=> 'string', format
=> 'pve-qm-smbios1',
621 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
622 ." remove disk operations.",
628 enum
=> [ qw(seabios ovmf) ],
629 description
=> "Select BIOS implementation.",
630 default => 'seabios',
634 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
635 format_description
=> 'UUID',
636 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
637 ." to disable explicitly.",
638 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
639 ." value identifier to the guest OS. This allows to notify the guest operating system"
640 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
641 ." execution or creation from a template). The guest operating system notices the"
642 ." change, and is then able to react as appropriate by marking its copies of"
643 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
644 ."Note that auto-creation only works when done through API/CLI create or update methods"
645 .", but not when manually editing the config file.",
646 default => "1 (autogenerated)",
651 format
=> 'pve-volume-id',
653 description
=> "Script that will be executed during various steps in the vms lifetime.",
657 format
=> $ivshmem_fmt,
658 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
664 format
=> $audio_fmt,
665 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
668 spice_enhancements
=> {
670 format
=> $spice_enhancements_fmt,
671 description
=> "Configure additional enhancements for SPICE.",
675 type
=> 'string', format
=> 'pve-tag-list',
676 description
=> 'Tags of the VM. This is only meta information.',
682 description
=> "Configure a VirtIO-based Random Number Generator.",
691 description
=> 'Specify a custom file containing all meta data passed to the VM via"
692 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
693 format
=> 'pve-volume-id',
694 format_description
=> 'volume',
699 description
=> 'Specify a custom file containing all network data passed to the VM via'
701 format
=> 'pve-volume-id',
702 format_description
=> 'volume',
707 description
=> 'Specify a custom file containing all user data passed to the VM via'
709 format
=> 'pve-volume-id',
710 format_description
=> 'volume',
713 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
715 my $confdesc_cloudinit = {
719 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
720 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
721 .' and `configdrive2` for windows.',
722 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
727 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
728 ." image's configured default user.",
733 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
734 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
735 .' support hashed passwords.',
740 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
742 format
=> 'pve-qm-cicustom',
747 description
=> "cloud-init: Sets DNS search domains for a container. Create will'
748 .' automatically use the setting from the host if neither searchdomain nor nameserver'
753 type
=> 'string', format
=> 'address-list',
754 description
=> "cloud-init: Sets DNS server IP address for a container. Create will'
755 .' automatically use the setting from the host if neither searchdomain nor nameserver'
761 format
=> 'urlencoded',
762 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
766 # what about other qemu settings ?
768 #machine => 'string',
781 ##soundhw => 'string',
783 while (my ($k, $v) = each %$confdesc) {
784 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
787 my $MAX_USB_DEVICES = 5;
789 my $MAX_SERIAL_PORTS = 4;
790 my $MAX_PARALLEL_PORTS = 3;
796 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
797 description
=> "CPUs accessing this NUMA node.",
798 format_description
=> "id[-id];...",
802 description
=> "Amount of memory this NUMA node provides.",
807 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
808 description
=> "Host NUMA nodes to use.",
809 format_description
=> "id[-id];...",
814 enum
=> [qw(preferred bind interleave)],
815 description
=> "NUMA allocation policy.",
819 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
822 type
=> 'string', format
=> $numa_fmt,
823 description
=> "NUMA topology.",
825 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
827 for (my $i = 0; $i < $MAX_NUMA; $i++) {
828 $confdesc->{"numa$i"} = $numadesc;
831 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
832 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
833 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
834 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
836 my $net_fmt_bridge_descr = <<__EOD__;
837 Bridge to attach the network device to. The Proxmox VE standard bridge
840 If you do not specify a bridge, we create a kvm user (NATed) network
841 device, which provides DHCP and DNS services. The following addresses
848 The DHCP server assign addresses to the guest starting from 10.0.2.15.
852 macaddr
=> get_standard_option
('mac-addr', {
853 description
=> "MAC address. That address must be unique withing your network. This is"
854 ." automatically generated if not specified.",
858 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
859 ." very low CPU overhead. If your guest does not support this driver, it is usually"
860 ." best to use 'e1000'.",
861 enum
=> $nic_model_list,
864 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
867 description
=> $net_fmt_bridge_descr,
868 format_description
=> 'bridge',
869 pattern
=> '[-_.\w\d]+',
874 minimum
=> 0, maximum
=> 16,
875 description
=> 'Number of packet queues to be used on the device.',
881 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
886 minimum
=> 1, maximum
=> 4094,
887 description
=> 'VLAN tag to apply to packets on this interface.',
892 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
893 description
=> 'VLAN trunks to pass through this interface.',
894 format_description
=> 'vlanid[;vlanid...]',
899 description
=> 'Whether this interface should be protected by the firewall.',
904 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
909 minimum
=> 1, maximum
=> 65520,
910 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
917 type
=> 'string', format
=> $net_fmt,
918 description
=> "Specify network devices.",
921 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
926 format
=> 'pve-ipv4-config',
927 format_description
=> 'IPv4Format/CIDR',
928 description
=> 'IPv4 address in CIDR format.',
935 format_description
=> 'GatewayIPv4',
936 description
=> 'Default gateway for IPv4 traffic.',
942 format
=> 'pve-ipv6-config',
943 format_description
=> 'IPv6Format/CIDR',
944 description
=> 'IPv6 address in CIDR format.',
951 format_description
=> 'GatewayIPv6',
952 description
=> 'Default gateway for IPv6 traffic.',
957 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
960 type
=> 'string', format
=> 'pve-qm-ipconfig',
961 description
=> <<'EODESCR',
962 cloud-init: Specify IP addresses and gateways for the corresponding interface.
964 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
966 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
967 gateway should be provided.
968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
969 cloud-init 19.4 or newer.
971 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
975 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
977 for (my $i = 0; $i < $MAX_NETS; $i++) {
978 $confdesc->{"net$i"} = $netdesc;
979 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
982 foreach my $key (keys %$confdesc_cloudinit) {
983 $confdesc->{$key} = $confdesc_cloudinit->{$key};
986 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
987 sub verify_volume_id_or_qm_path
{
988 my ($volid, $noerr) = @_;
990 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
994 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
995 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1006 type
=> 'string', format
=> 'pve-qm-usb-device',
1007 format_description
=> 'HOSTUSBDEVICE|spice',
1008 description
=> <<EODESCR,
1009 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1011 'bus-port(.port)*' (decimal numbers) or
1012 'vendor_id:product_id' (hexadeciaml numbers) or
1015 You can use the 'lsusb -t' command to list existing usb devices.
1017 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1018 machines - use with special care.
1020 The value 'spice' can be used to add a usb redirection devices for spice.
1026 description
=> "Specifies whether if given host option is a USB3 device or port.",
1033 type
=> 'string', format
=> $usb_fmt,
1034 description
=> "Configure an USB device (n is 0 to 4).",
1036 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1041 pattern
=> '(/dev/.+|socket)',
1042 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1043 verbose_description
=> <<EODESCR,
1044 Create a serial device inside the VM (n is 0 to 3), and pass through a
1045 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1046 host side (use 'qm terminal' to open a terminal connection).
1048 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1049 use with special care.
1051 CAUTION: Experimental! User reported problems with this option.
1058 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1059 description
=> "Map host parallel devices (n is 0 to 2).",
1060 verbose_description
=> <<EODESCR,
1061 Map host parallel devices (n is 0 to 2).
1063 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1064 machines - use with special care.
1066 CAUTION: Experimental! User reported problems with this option.
1070 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1071 $confdesc->{"parallel$i"} = $paralleldesc;
1074 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1075 $confdesc->{"serial$i"} = $serialdesc;
1078 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1079 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1082 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1083 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1086 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1087 $confdesc->{"usb$i"} = $usbdesc;
1095 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1096 . " Deprecated, use 'order=' instead.",
1097 pattern
=> '[acdn]{1,4}',
1098 format_description
=> "[acdn]{1,4}",
1100 # note: this is also the fallback if boot: is not given at all
1106 format
=> 'pve-qm-bootdev-list',
1107 format_description
=> "device[;device...]",
1108 description
=> <<EODESC,
1109 The guest will attempt to boot from devices in the order they appear here.
1111 Disks, optical drives and passed-through storage USB devices will be directly
1112 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1113 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1115 Note that only devices in this list will be marked as bootable and thus loaded
1116 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1117 (e.g. software-raid), you need to specify all of them here.
1119 Overrides the deprecated 'legacy=[acdn]*' value when given.
1123 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1125 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1126 sub verify_bootdev
{
1127 my ($dev, $noerr) = @_;
1129 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && $dev !~ m/^efidisk/;
1133 return 0 if $dev !~ m/^$base\d+$/;
1134 return 0 if !$confdesc->{$dev};
1138 return $dev if $check->("net");
1139 return $dev if $check->("usb");
1140 return $dev if $check->("hostpci");
1143 die "invalid boot device '$dev'\n";
1146 sub print_bootorder
{
1148 return "" if !@$devs;
1149 my $data = { order
=> join(';', @$devs) };
1150 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1153 my $kvm_api_version = 0;
1156 return $kvm_api_version if $kvm_api_version;
1158 open my $fh, '<', '/dev/kvm' or return;
1160 # 0xae00 => KVM_GET_API_VERSION
1161 $kvm_api_version = ioctl($fh, 0xae00, 0);
1164 return $kvm_api_version;
1167 my $kvm_user_version = {};
1170 sub kvm_user_version
{
1173 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1174 my $st = stat($binary);
1176 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1177 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1178 $cachedmtime == $st->mtime;
1180 $kvm_user_version->{$binary} = 'unknown';
1181 $kvm_mtime->{$binary} = $st->mtime;
1185 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1186 $kvm_user_version->{$binary} = $2;
1190 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1193 return $kvm_user_version->{$binary};
1196 my sub extract_version
{
1197 my ($machine_type, $version) = @_;
1198 $version = kvm_user_version
() if !defined($version);
1199 PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1202 sub kernel_has_vhost_net
{
1203 return -c
'/dev/vhost-net';
1208 return defined($confdesc->{$key});
1212 sub get_cdrom_path
{
1214 return $cdrom_path if $cdrom_path;
1216 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1217 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1218 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1222 my ($storecfg, $vmid, $cdrom) = @_;
1224 if ($cdrom eq 'cdrom') {
1225 return get_cdrom_path
();
1226 } elsif ($cdrom eq 'none') {
1228 } elsif ($cdrom =~ m
|^/|) {
1231 return PVE
::Storage
::path
($storecfg, $cdrom);
1235 # try to convert old style file names to volume IDs
1236 sub filename_to_volume_id
{
1237 my ($vmid, $file, $media) = @_;
1239 if (!($file eq 'none' || $file eq 'cdrom' ||
1240 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1242 return if $file =~ m
|/|;
1244 if ($media && $media eq 'cdrom') {
1245 $file = "local:iso/$file";
1247 $file = "local:$vmid/$file";
1254 sub verify_media_type
{
1255 my ($opt, $vtype, $media) = @_;
1260 if ($media eq 'disk') {
1262 } elsif ($media eq 'cdrom') {
1265 die "internal error";
1268 return if ($vtype eq $etype);
1270 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1273 sub cleanup_drive_path
{
1274 my ($opt, $storecfg, $drive) = @_;
1276 # try to convert filesystem paths to volume IDs
1278 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1279 ($drive->{file
} !~ m
|^/dev/.+|) &&
1280 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1281 ($drive->{file
} !~ m/^\d+$/)) {
1282 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1283 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1285 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1286 verify_media_type
($opt, $vtype, $drive->{media
});
1287 $drive->{file
} = $volid;
1290 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1293 sub parse_hotplug_features
{
1298 return $res if $data eq '0';
1300 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1302 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1303 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1306 die "invalid hotplug feature '$feature'\n";
1312 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1313 sub pve_verify_hotplug_features
{
1314 my ($value, $noerr) = @_;
1316 return $value if parse_hotplug_features
($value);
1320 die "unable to parse hotplug option\n";
1324 my($fh, $noerr) = @_;
1327 my $SG_GET_VERSION_NUM = 0x2282;
1329 my $versionbuf = "\x00" x
8;
1330 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1332 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1335 my $version = unpack("I", $versionbuf);
1336 if ($version < 30000) {
1337 die "scsi generic interface too old\n" if !$noerr;
1341 my $buf = "\x00" x
36;
1342 my $sensebuf = "\x00" x
8;
1343 my $cmd = pack("C x3 C x1", 0x12, 36);
1345 # see /usr/include/scsi/sg.h
1346 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1348 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1349 length($sensebuf), 0, length($buf), $buf,
1350 $cmd, $sensebuf, 6000);
1352 $ret = ioctl($fh, $SG_IO, $packet);
1354 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1358 my @res = unpack($sg_io_hdr_t, $packet);
1359 if ($res[17] || $res[18]) {
1360 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1365 (my $byte0, my $byte1, $res->{vendor
},
1366 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1368 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1369 $res->{type
} = $byte0 & 31;
1377 my $fh = IO
::File-
>new("+<$path") || return;
1378 my $res = scsi_inquiry
($fh, 1);
1384 sub print_tabletdevice_full
{
1385 my ($conf, $arch) = @_;
1387 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1389 # we use uhci for old VMs because tablet driver was buggy in older qemu
1391 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1397 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1400 sub print_keyboarddevice_full
{
1401 my ($conf, $arch, $machine) = @_;
1403 return if $arch ne 'aarch64';
1405 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1408 my sub get_drive_id
{
1410 return "$drive->{interface}$drive->{index}";
1413 sub print_drivedevice_full
{
1414 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1419 my $drive_id = get_drive_id
($drive);
1420 if ($drive->{interface
} eq 'virtio') {
1421 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1422 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1423 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1424 } elsif ($drive->{interface
} eq 'scsi') {
1426 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1427 my $unit = $drive->{index} % $maxdev;
1428 my $devicetype = 'hd';
1430 if (drive_is_cdrom
($drive)) {
1433 if ($drive->{file
} =~ m
|^/|) {
1434 $path = $drive->{file
};
1435 if (my $info = path_is_scsi
($path)) {
1436 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1437 $devicetype = 'block';
1438 } elsif ($info->{type
} == 1) { # tape
1439 $devicetype = 'generic';
1443 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1446 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1447 my $version = extract_version
($machine_type, kvm_user_version
());
1448 if ($path =~ m/^iscsi\:\/\
// &&
1449 !min_version
($version, 4, 1)) {
1450 $devicetype = 'generic';
1454 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1455 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1457 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1458 .",lun=$drive->{index}";
1460 $device .= ",drive=drive-$drive_id,id=$drive_id";
1462 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1463 $device .= ",rotation_rate=1";
1465 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1467 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1468 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1469 my $controller = int($drive->{index} / $maxdev);
1470 my $unit = $drive->{index} % $maxdev;
1471 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1473 $device = "ide-$devicetype";
1474 if ($drive->{interface
} eq 'ide') {
1475 $device .= ",bus=ide.$controller,unit=$unit";
1477 $device .= ",bus=ahci$controller.$unit";
1479 $device .= ",drive=drive-$drive_id,id=$drive_id";
1481 if ($devicetype eq 'hd') {
1482 if (my $model = $drive->{model
}) {
1483 $model = URI
::Escape
::uri_unescape
($model);
1484 $device .= ",model=$model";
1486 if ($drive->{ssd
}) {
1487 $device .= ",rotation_rate=1";
1490 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1491 } elsif ($drive->{interface
} eq 'usb') {
1493 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1495 die "unsupported interface type";
1498 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1500 if (my $serial = $drive->{serial
}) {
1501 $serial = URI
::Escape
::uri_unescape
($serial);
1502 $device .= ",serial=$serial";
1509 sub get_initiator_name
{
1512 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1513 while (defined(my $line = <$fh>)) {
1514 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1523 sub print_drive_commandline_full
{
1524 my ($storecfg, $vmid, $drive, $pbs_name) = @_;
1527 my $volid = $drive->{file
};
1528 my $format = $drive->{format
};
1529 my $drive_id = get_drive_id
($drive);
1531 if (drive_is_cdrom
($drive)) {
1532 $path = get_iso_path
($storecfg, $vmid, $volid);
1533 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1535 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1537 $path = PVE
::Storage
::path
($storecfg, $volid);
1538 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1539 $format //= qemu_img_format
($scfg, $volname);
1546 my $is_rbd = $path =~ m/^rbd:/;
1549 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1550 foreach my $o (@qemu_drive_options) {
1551 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1554 # snapshot only accepts on|off
1555 if (defined($drive->{snapshot
})) {
1556 my $v = $drive->{snapshot
} ?
'on' : 'off';
1557 $opts .= ",snapshot=$v";
1560 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1561 my ($dir, $qmpname) = @$type;
1562 if (my $v = $drive->{"mbps$dir"}) {
1563 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1565 if (my $v = $drive->{"mbps${dir}_max"}) {
1566 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1568 if (my $v = $drive->{"bps${dir}_max_length"}) {
1569 $opts .= ",throttling.bps$qmpname-max-length=$v";
1571 if (my $v = $drive->{"iops${dir}"}) {
1572 $opts .= ",throttling.iops$qmpname=$v";
1574 if (my $v = $drive->{"iops${dir}_max"}) {
1575 $opts .= ",throttling.iops$qmpname-max=$v";
1577 if (my $v = $drive->{"iops${dir}_max_length"}) {
1578 $opts .= ",throttling.iops$qmpname-max-length=$v";
1583 $format = "rbd" if $is_rbd;
1584 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1586 $opts .= ",format=alloc-track,file.driver=$format";
1588 $opts .= ",format=$format";
1591 my $cache_direct = 0;
1593 if (my $cache = $drive->{cache
}) {
1594 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1595 } elsif (!drive_is_cdrom
($drive)) {
1596 $opts .= ",cache=none";
1600 # aio native works only with O_DIRECT
1601 if (!$drive->{aio
}) {
1603 $opts .= ",aio=native";
1605 $opts .= ",aio=threads";
1609 if (!drive_is_cdrom
($drive)) {
1611 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1612 $detectzeroes = 'off';
1613 } elsif ($drive->{discard
}) {
1614 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1616 # This used to be our default with discard not being specified:
1617 $detectzeroes = 'on';
1620 # note: 'detect-zeroes' works per blockdev and we want it to persist
1621 # after the alloc-track is removed, so put it on 'file' directly
1622 my $dz_param = $pbs_name ?
"file.detect-zeroes" : "detect-zeroes";
1623 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1627 $opts .= ",backing=$pbs_name";
1628 $opts .= ",auto-remove=on";
1631 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1632 my $file_param = "file";
1634 # non-rbd drivers require the underlying file to be a seperate block
1635 # node, so add a second .file indirection
1636 $file_param .= ".file" if !$is_rbd;
1637 $file_param .= ".filename";
1639 my $pathinfo = $path ?
"$file_param=$path," : '';
1641 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1644 sub print_pbs_blockdev
{
1645 my ($pbs_conf, $pbs_name) = @_;
1646 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1647 $blockdev .= ",repository=$pbs_conf->{repository}";
1648 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1649 $blockdev .= ",archive=$pbs_conf->{archive}";
1650 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1654 sub print_netdevice_full
{
1655 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1657 my $device = $net->{model
};
1658 if ($net->{model
} eq 'virtio') {
1659 $device = 'virtio-net-pci';
1662 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1663 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1664 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1665 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1666 # and out of each queue plus one config interrupt and control vector queue
1667 my $vectors = $net->{queues
} * 2 + 2;
1668 $tmpstr .= ",vectors=$vectors,mq=on";
1670 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1672 if (my $mtu = $net->{mtu
}) {
1673 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1674 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1677 } elsif ($mtu < 576) {
1678 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1679 } elsif ($mtu > $bridge_mtu) {
1680 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1682 $tmpstr .= ",host_mtu=$mtu";
1684 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1688 if ($use_old_bios_files) {
1690 if ($device eq 'virtio-net-pci') {
1691 $romfile = 'pxe-virtio.rom';
1692 } elsif ($device eq 'e1000') {
1693 $romfile = 'pxe-e1000.rom';
1694 } elsif ($device eq 'ne2k') {
1695 $romfile = 'pxe-ne2k_pci.rom';
1696 } elsif ($device eq 'pcnet') {
1697 $romfile = 'pxe-pcnet.rom';
1698 } elsif ($device eq 'rtl8139') {
1699 $romfile = 'pxe-rtl8139.rom';
1701 $tmpstr .= ",romfile=$romfile" if $romfile;
1707 sub print_netdev_full
{
1708 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1711 if ($netid =~ m/^net(\d+)$/) {
1715 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1717 my $ifname = "tap${vmid}i$i";
1719 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1720 die "interface name '$ifname' is too long (max 15 character)\n"
1721 if length($ifname) >= 16;
1723 my $vhostparam = '';
1724 if (is_native
($arch)) {
1725 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1728 my $vmname = $conf->{name
} || "vm$vmid";
1731 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1733 if ($net->{bridge
}) {
1734 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1735 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1737 $netdev = "type=user,id=$netid,hostname=$vmname";
1740 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1746 'cirrus' => 'cirrus-vga',
1748 'vmware' => 'vmware-svga',
1749 'virtio' => 'virtio-vga',
1752 sub print_vga_device
{
1753 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1755 my $type = $vga_map->{$vga->{type
}};
1756 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1757 $type = 'virtio-gpu';
1759 my $vgamem_mb = $vga->{memory
};
1761 my $max_outputs = '';
1763 $type = $id ?
'qxl' : 'qxl-vga';
1765 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1766 # set max outputs so linux can have up to 4 qxl displays with one device
1767 if (min_version
($machine_version, 4, 1)) {
1768 $max_outputs = ",max_outputs=4";
1773 die "no devicetype for $vga->{type}\n" if !$type;
1777 if ($vga->{type
} eq 'virtio') {
1778 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1779 $memory = ",max_hostmem=$bytes";
1781 # from https://www.spice-space.org/multiple-monitors.html
1782 $memory = ",vgamem_mb=$vga->{memory}";
1783 my $ram = $vgamem_mb * 4;
1784 my $vram = $vgamem_mb * 2;
1785 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1787 $memory = ",vgamem_mb=$vga->{memory}";
1789 } elsif ($qxlnum && $id) {
1790 $memory = ",ram_size=67108864,vram_size=33554432";
1794 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1795 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1798 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1799 my $vgaid = "vga" . ($id // '');
1802 if ($q35 && $vgaid eq 'vga') {
1803 # the first display uses pcie.0 bus on q35 machines
1804 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1806 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1809 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1812 sub parse_number_sets
{
1815 foreach my $part (split(/;/, $set)) {
1816 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1817 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1818 push @$res, [ $1, $2 ];
1820 die "invalid range: $part\n";
1829 my $res = parse_property_string
($numa_fmt, $data);
1830 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1831 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1835 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1839 my $res = eval { parse_property_string
($net_fmt, $data) };
1844 if (!defined($res->{macaddr
})) {
1845 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1846 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1851 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1852 sub parse_ipconfig
{
1855 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1861 if ($res->{gw
} && !$res->{ip
}) {
1862 warn 'gateway specified without specifying an IP address';
1865 if ($res->{gw6
} && !$res->{ip6
}) {
1866 warn 'IPv6 gateway specified without specifying an IPv6 address';
1869 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1870 warn 'gateway specified together with DHCP';
1873 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1875 warn "IPv6 gateway specified together with $res->{ip6} address";
1879 if (!$res->{ip
} && !$res->{ip6
}) {
1880 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1889 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1892 sub add_random_macs
{
1893 my ($settings) = @_;
1895 foreach my $opt (keys %$settings) {
1896 next if $opt !~ m/^net(\d+)$/;
1897 my $net = parse_net
($settings->{$opt});
1899 $settings->{$opt} = print_net
($net);
1903 sub vm_is_volid_owner
{
1904 my ($storecfg, $vmid, $volid) = @_;
1906 if ($volid !~ m
|^/|) {
1908 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1909 if ($owner && ($owner == $vmid)) {
1917 sub vmconfig_register_unused_drive
{
1918 my ($storecfg, $vmid, $conf, $drive) = @_;
1920 if (drive_is_cloudinit
($drive)) {
1921 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1923 } elsif (!drive_is_cdrom
($drive)) {
1924 my $volid = $drive->{file
};
1925 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1926 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1931 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1935 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1936 format_description
=> 'UUID',
1937 description
=> "Set SMBIOS1 UUID.",
1942 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1943 format_description
=> 'Base64 encoded string',
1944 description
=> "Set SMBIOS1 version.",
1949 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1950 format_description
=> 'Base64 encoded string',
1951 description
=> "Set SMBIOS1 serial number.",
1956 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1957 format_description
=> 'Base64 encoded string',
1958 description
=> "Set SMBIOS1 manufacturer.",
1963 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1964 format_description
=> 'Base64 encoded string',
1965 description
=> "Set SMBIOS1 product ID.",
1970 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1971 format_description
=> 'Base64 encoded string',
1972 description
=> "Set SMBIOS1 SKU string.",
1977 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1978 format_description
=> 'Base64 encoded string',
1979 description
=> "Set SMBIOS1 family string.",
1984 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
1992 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
1999 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2002 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2004 sub parse_watchdog
{
2009 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2014 sub parse_guest_agent
{
2017 return {} if !defined($conf->{agent
});
2019 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2022 # if the agent is disabled ignore the other potentially set properties
2023 return {} if !$res->{enabled
};
2028 my ($conf, $key) = @_;
2029 return undef if !defined($conf->{agent
});
2031 my $agent = parse_guest_agent
($conf);
2032 return $agent->{$key};
2038 return {} if !$value;
2039 my $res = eval { parse_property_string
($vga_fmt, $value) };
2049 my $res = eval { parse_property_string
($rng_fmt, $value) };
2054 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2055 sub verify_usb_device
{
2056 my ($value, $noerr) = @_;
2058 return $value if parse_usb_device
($value);
2062 die "unable to parse usb device\n";
2065 # add JSON properties for create and set function
2066 sub json_config_properties
{
2069 foreach my $opt (keys %$confdesc) {
2070 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2071 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2072 $prop->{$opt} = $confdesc->{$opt};
2078 # return copy of $confdesc_cloudinit to generate documentation
2079 sub cloudinit_config_properties
{
2081 return dclone
($confdesc_cloudinit);
2085 my ($key, $value) = @_;
2087 die "unknown setting '$key'\n" if !$confdesc->{$key};
2089 my $type = $confdesc->{$key}->{type
};
2091 if (!defined($value)) {
2092 die "got undefined value\n";
2095 if ($value =~ m/[\n\r]/) {
2096 die "property contains a line feed\n";
2099 if ($type eq 'boolean') {
2100 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2101 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2102 die "type check ('boolean') failed - got '$value'\n";
2103 } elsif ($type eq 'integer') {
2104 return int($1) if $value =~ m/^(\d+)$/;
2105 die "type check ('integer') failed - got '$value'\n";
2106 } elsif ($type eq 'number') {
2107 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2108 die "type check ('number') failed - got '$value'\n";
2109 } elsif ($type eq 'string') {
2110 if (my $fmt = $confdesc->{$key}->{format
}) {
2111 PVE
::JSONSchema
::check_format
($fmt, $value);
2114 $value =~ s/^\"(.*)\"$/$1/;
2117 die "internal error"
2122 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2124 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2126 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2128 if ($conf->{template
}) {
2129 # check if any base image is still used by a linked clone
2130 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2131 my ($ds, $drive) = @_;
2132 return if drive_is_cdrom
($drive);
2134 my $volid = $drive->{file
};
2135 return if !$volid || $volid =~ m
|^/|;
2137 die "base volume '$volid' is still in use by linked cloned\n"
2138 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2143 my $remove_owned_drive = sub {
2144 my ($ds, $drive) = @_;
2145 return if drive_is_cdrom
($drive, 1);
2147 my $volid = $drive->{file
};
2148 return if !$volid || $volid =~ m
|^/|;
2150 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2151 return if !$path || !$owner || ($owner != $vmid);
2153 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2154 warn "Could not remove disk '$volid', check manually: $@" if $@;
2157 # only remove disks owned by this VM (referenced in the config)
2158 my $include_opts = {
2159 include_unused
=> 1,
2160 extra_keys
=> ['vmstate'],
2162 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2164 for my $snap (values %{$conf->{snapshots
}}) {
2165 next if !defined($snap->{vmstate
});
2166 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2167 next if !defined($drive);
2168 $remove_owned_drive->('vmstate', $drive);
2171 if ($purge_unreferenced) { # also remove unreferenced disk
2172 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2173 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2174 my ($volid, $sid, $volname, $d) = @_;
2175 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2180 if (defined $replacement_conf) {
2181 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2183 PVE
::QemuConfig-
>destroy_config($vmid);
2187 sub parse_vm_config
{
2188 my ($filename, $raw) = @_;
2190 return if !defined($raw);
2193 digest
=> Digest
::SHA
::sha1_hex
($raw),
2198 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2199 || die "got strange filename '$filename'";
2207 my @lines = split(/\n/, $raw);
2208 foreach my $line (@lines) {
2209 next if $line =~ m/^\s*$/;
2211 if ($line =~ m/^\[PENDING\]\s*$/i) {
2212 $section = 'pending';
2213 if (defined($descr)) {
2215 $conf->{description
} = $descr;
2218 $conf = $res->{$section} = {};
2221 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2223 if (defined($descr)) {
2225 $conf->{description
} = $descr;
2228 $conf = $res->{snapshots
}->{$section} = {};
2232 if ($line =~ m/^\#(.*)\s*$/) {
2233 $descr = '' if !defined($descr);
2234 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2238 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2239 $descr = '' if !defined($descr);
2240 $descr .= PVE
::Tools
::decode_text
($2);
2241 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2242 $conf->{snapstate
} = $1;
2243 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2246 $conf->{$key} = $value;
2247 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2249 if ($section eq 'pending') {
2250 $conf->{delete} = $value; # we parse this later
2252 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2254 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2257 eval { $value = check_type
($key, $value); };
2259 warn "vm $vmid - unable to parse value of '$key' - $@";
2261 $key = 'ide2' if $key eq 'cdrom';
2262 my $fmt = $confdesc->{$key}->{format
};
2263 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2264 my $v = parse_drive
($key, $value);
2265 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2266 $v->{file
} = $volid;
2267 $value = print_drive
($v);
2269 warn "vm $vmid - unable to parse value of '$key'\n";
2274 $conf->{$key} = $value;
2277 warn "vm $vmid - unable to parse config: $line\n";
2281 if (defined($descr)) {
2283 $conf->{description
} = $descr;
2285 delete $res->{snapstate
}; # just to be sure
2290 sub write_vm_config
{
2291 my ($filename, $conf) = @_;
2293 delete $conf->{snapstate
}; # just to be sure
2295 if ($conf->{cdrom
}) {
2296 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2297 $conf->{ide2
} = $conf->{cdrom
};
2298 delete $conf->{cdrom
};
2301 # we do not use 'smp' any longer
2302 if ($conf->{sockets
}) {
2303 delete $conf->{smp
};
2304 } elsif ($conf->{smp
}) {
2305 $conf->{sockets
} = $conf->{smp
};
2306 delete $conf->{cores
};
2307 delete $conf->{smp
};
2310 my $used_volids = {};
2312 my $cleanup_config = sub {
2313 my ($cref, $pending, $snapname) = @_;
2315 foreach my $key (keys %$cref) {
2316 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2317 $key eq 'snapstate' || $key eq 'pending';
2318 my $value = $cref->{$key};
2319 if ($key eq 'delete') {
2320 die "propertry 'delete' is only allowed in [PENDING]\n"
2322 # fixme: check syntax?
2325 eval { $value = check_type
($key, $value); };
2326 die "unable to parse value of '$key' - $@" if $@;
2328 $cref->{$key} = $value;
2330 if (!$snapname && is_valid_drivename
($key)) {
2331 my $drive = parse_drive
($key, $value);
2332 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2337 &$cleanup_config($conf);
2339 &$cleanup_config($conf->{pending
}, 1);
2341 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2342 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2343 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2346 # remove 'unusedX' settings if we re-add a volume
2347 foreach my $key (keys %$conf) {
2348 my $value = $conf->{$key};
2349 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2350 delete $conf->{$key};
2354 my $generate_raw_config = sub {
2355 my ($conf, $pending) = @_;
2359 # add description as comment to top of file
2360 if (defined(my $descr = $conf->{description
})) {
2362 foreach my $cl (split(/\n/, $descr)) {
2363 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2366 $raw .= "#\n" if $pending;
2370 foreach my $key (sort keys %$conf) {
2371 next if $key =~ /^(digest|description|pending|snapshots)$/;
2372 $raw .= "$key: $conf->{$key}\n";
2377 my $raw = &$generate_raw_config($conf);
2379 if (scalar(keys %{$conf->{pending
}})){
2380 $raw .= "\n[PENDING]\n";
2381 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2384 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2385 $raw .= "\n[$snapname]\n";
2386 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2396 # we use static defaults from our JSON schema configuration
2397 foreach my $key (keys %$confdesc) {
2398 if (defined(my $default = $confdesc->{$key}->{default})) {
2399 $res->{$key} = $default;
2407 my $vmlist = PVE
::Cluster
::get_vmlist
();
2409 return $res if !$vmlist || !$vmlist->{ids
};
2410 my $ids = $vmlist->{ids
};
2411 my $nodename = nodename
();
2413 foreach my $vmid (keys %$ids) {
2414 my $d = $ids->{$vmid};
2415 next if !$d->{node
} || $d->{node
} ne $nodename;
2416 next if !$d->{type
} || $d->{type
} ne 'qemu';
2417 $res->{$vmid}->{exists} = 1;
2422 # test if VM uses local resources (to prevent migration)
2423 sub check_local_resources
{
2424 my ($conf, $noerr) = @_;
2428 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2429 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2431 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2433 foreach my $k (keys %$conf) {
2434 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2435 # sockets are safe: they will recreated be on the target side post-migrate
2436 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2437 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2440 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2445 # check if used storages are available on all nodes (use by migrate)
2446 sub check_storage_availability
{
2447 my ($storecfg, $conf, $node) = @_;
2449 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2450 my ($ds, $drive) = @_;
2452 my $volid = $drive->{file
};
2455 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2458 # check if storage is available on both nodes
2459 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2460 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2464 # list nodes where all VM images are available (used by has_feature API)
2466 my ($conf, $storecfg) = @_;
2468 my $nodelist = PVE
::Cluster
::get_nodelist
();
2469 my $nodehash = { map { $_ => 1 } @$nodelist };
2470 my $nodename = nodename
();
2472 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2473 my ($ds, $drive) = @_;
2475 my $volid = $drive->{file
};
2478 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2480 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2481 if ($scfg->{disable
}) {
2483 } elsif (my $avail = $scfg->{nodes
}) {
2484 foreach my $node (keys %$nodehash) {
2485 delete $nodehash->{$node} if !$avail->{$node};
2487 } elsif (!$scfg->{shared
}) {
2488 foreach my $node (keys %$nodehash) {
2489 delete $nodehash->{$node} if $node ne $nodename
2498 sub check_local_storage_availability
{
2499 my ($conf, $storecfg) = @_;
2501 my $nodelist = PVE
::Cluster
::get_nodelist
();
2502 my $nodehash = { map { $_ => {} } @$nodelist };
2504 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2505 my ($ds, $drive) = @_;
2507 my $volid = $drive->{file
};
2510 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2512 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2514 if ($scfg->{disable
}) {
2515 foreach my $node (keys %$nodehash) {
2516 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2518 } elsif (my $avail = $scfg->{nodes
}) {
2519 foreach my $node (keys %$nodehash) {
2520 if (!$avail->{$node}) {
2521 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2528 foreach my $node (values %$nodehash) {
2529 if (my $unavail = $node->{unavailable_storages
}) {
2530 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2537 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2539 my ($vmid, $nocheck, $node) = @_;
2541 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2542 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2547 my $vzlist = config_list
();
2549 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2551 while (defined(my $de = $fd->read)) {
2552 next if $de !~ m/^(\d+)\.pid$/;
2554 next if !defined($vzlist->{$vmid});
2555 if (my $pid = check_running
($vmid)) {
2556 $vzlist->{$vmid}->{pid
} = $pid;
2563 our $vmstatus_return_properties = {
2564 vmid
=> get_standard_option
('pve-vmid'),
2566 description
=> "Qemu process status.",
2568 enum
=> ['stopped', 'running'],
2571 description
=> "Maximum memory in bytes.",
2574 renderer
=> 'bytes',
2577 description
=> "Root disk size in bytes.",
2580 renderer
=> 'bytes',
2583 description
=> "VM name.",
2588 description
=> "Qemu QMP agent status.",
2593 description
=> "PID of running qemu process.",
2598 description
=> "Uptime.",
2601 renderer
=> 'duration',
2604 description
=> "Maximum usable CPUs.",
2609 description
=> "The current config lock, if any.",
2614 description
=> "The current configured tags, if any",
2618 'running-machine' => {
2619 description
=> "The currently running machine type (if running).",
2624 description
=> "The currently running QEMU version (if running).",
2630 my $last_proc_pid_stat;
2632 # get VM status information
2633 # This must be fast and should not block ($full == false)
2634 # We only query KVM using QMP if $full == true (this can be slow)
2636 my ($opt_vmid, $full) = @_;
2640 my $storecfg = PVE
::Storage
::config
();
2642 my $list = vzlist
();
2643 my $defaults = load_defaults
();
2645 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2647 my $cpucount = $cpuinfo->{cpus
} || 1;
2649 foreach my $vmid (keys %$list) {
2650 next if $opt_vmid && ($vmid ne $opt_vmid);
2652 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2654 my $d = { vmid
=> int($vmid) };
2655 $d->{pid
} = int($list->{$vmid}->{pid
});
2657 # fixme: better status?
2658 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2660 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2661 if (defined($size)) {
2662 $d->{disk
} = 0; # no info available
2663 $d->{maxdisk
} = $size;
2669 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2670 * ($conf->{cores
} || $defaults->{cores
});
2671 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2672 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2674 $d->{name
} = $conf->{name
} || "VM $vmid";
2675 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2676 : $defaults->{memory
}*(1024*1024);
2678 if ($conf->{balloon
}) {
2679 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2680 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2681 : $defaults->{shares
};
2692 $d->{diskwrite
} = 0;
2694 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2696 $d->{serial
} = 1 if conf_has_serial
($conf);
2697 $d->{lock} = $conf->{lock} if $conf->{lock};
2698 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2703 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2704 foreach my $dev (keys %$netdev) {
2705 next if $dev !~ m/^tap([1-9]\d*)i/;
2707 my $d = $res->{$vmid};
2710 $d->{netout
} += $netdev->{$dev}->{receive
};
2711 $d->{netin
} += $netdev->{$dev}->{transmit
};
2714 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2715 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2720 my $ctime = gettimeofday
;
2722 foreach my $vmid (keys %$list) {
2724 my $d = $res->{$vmid};
2725 my $pid = $d->{pid
};
2728 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2729 next if !$pstat; # not running
2731 my $used = $pstat->{utime} + $pstat->{stime
};
2733 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2735 if ($pstat->{vsize
}) {
2736 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2739 my $old = $last_proc_pid_stat->{$pid};
2741 $last_proc_pid_stat->{$pid} = {
2749 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2751 if ($dtime > 1000) {
2752 my $dutime = $used - $old->{used
};
2754 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2755 $last_proc_pid_stat->{$pid} = {
2761 $d->{cpu
} = $old->{cpu
};
2765 return $res if !$full;
2767 my $qmpclient = PVE
::QMPClient-
>new();
2769 my $ballooncb = sub {
2770 my ($vmid, $resp) = @_;
2772 my $info = $resp->{'return'};
2773 return if !$info->{max_mem
};
2775 my $d = $res->{$vmid};
2777 # use memory assigned to VM
2778 $d->{maxmem
} = $info->{max_mem
};
2779 $d->{balloon
} = $info->{actual
};
2781 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2782 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2783 $d->{freemem
} = $info->{free_mem
};
2786 $d->{ballooninfo
} = $info;
2789 my $blockstatscb = sub {
2790 my ($vmid, $resp) = @_;
2791 my $data = $resp->{'return'} || [];
2792 my $totalrdbytes = 0;
2793 my $totalwrbytes = 0;
2795 for my $blockstat (@$data) {
2796 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2797 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2799 $blockstat->{device
} =~ s/drive-//;
2800 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2802 $res->{$vmid}->{diskread
} = $totalrdbytes;
2803 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2806 my $machinecb = sub {
2807 my ($vmid, $resp) = @_;
2808 my $data = $resp->{'return'} || [];
2810 $res->{$vmid}->{'running-machine'} =
2811 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
2814 my $versioncb = sub {
2815 my ($vmid, $resp) = @_;
2816 my $data = $resp->{'return'} // {};
2817 my $version = 'unknown';
2819 if (my $v = $data->{qemu
}) {
2820 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
2823 $res->{$vmid}->{'running-qemu'} = $version;
2826 my $statuscb = sub {
2827 my ($vmid, $resp) = @_;
2829 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2830 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2831 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2832 # this fails if ballon driver is not loaded, so this must be
2833 # the last commnand (following command are aborted if this fails).
2834 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2836 my $status = 'unknown';
2837 if (!defined($status = $resp->{'return'}->{status
})) {
2838 warn "unable to get VM status\n";
2842 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2845 foreach my $vmid (keys %$list) {
2846 next if $opt_vmid && ($vmid ne $opt_vmid);
2847 next if !$res->{$vmid}->{pid
}; # not running
2848 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2851 $qmpclient->queue_execute(undef, 2);
2853 foreach my $vmid (keys %$list) {
2854 next if $opt_vmid && ($vmid ne $opt_vmid);
2855 next if !$res->{$vmid}->{pid
}; #not running
2857 # we can't use the $qmpclient since it might have already aborted on
2858 # 'query-balloon', but this might also fail for older versions...
2859 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
2860 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2863 foreach my $vmid (keys %$list) {
2864 next if $opt_vmid && ($vmid ne $opt_vmid);
2865 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2871 sub conf_has_serial
{
2874 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2875 if ($conf->{"serial$i"}) {
2883 sub conf_has_audio
{
2884 my ($conf, $id) = @_;
2887 my $audio = $conf->{"audio$id"};
2888 return if !defined($audio);
2890 my $audioproperties = parse_property_string
($audio_fmt, $audio);
2891 my $audiodriver = $audioproperties->{driver
} // 'spice';
2894 dev
=> $audioproperties->{device
},
2895 dev_id
=> "audiodev$id",
2896 backend
=> $audiodriver,
2897 backend_id
=> "$audiodriver-backend${id}",
2902 my ($audio, $audiopciaddr, $machine_version) = @_;
2906 my $id = $audio->{dev_id
};
2908 if (min_version
($machine_version, 4, 2)) {
2909 $audiodev = ",audiodev=$audio->{backend_id}";
2912 if ($audio->{dev
} eq 'AC97') {
2913 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2914 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
2915 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2916 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2917 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2919 die "unkown audio device '$audio->{dev}', implement me!";
2922 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2927 sub vga_conf_has_spice
{
2930 my $vgaconf = parse_vga
($vga);
2931 my $vgatype = $vgaconf->{type
};
2932 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2939 return get_host_arch
() eq $arch;
2944 return $conf->{arch
} // get_host_arch
();
2947 my $default_machines = {
2952 sub get_installed_machine_version
{
2953 my ($kvmversion) = @_;
2954 $kvmversion = kvm_user_version
() if !defined($kvmversion);
2955 $kvmversion =~ m/^(\d+\.\d+)/;
2959 sub windows_get_pinned_machine_version
{
2960 my ($machine, $base_version, $kvmversion) = @_;
2962 my $pin_version = $base_version;
2963 if (!defined($base_version) ||
2964 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
2966 $pin_version = get_installed_machine_version
($kvmversion);
2968 if (!$machine || $machine eq 'pc') {
2969 $machine = "pc-i440fx-$pin_version";
2970 } elsif ($machine eq 'q35') {
2971 $machine = "pc-q35-$pin_version";
2972 } elsif ($machine eq 'virt') {
2973 $machine = "virt-$pin_version";
2975 warn "unknown machine type '$machine', not touching that!\n";
2981 sub get_vm_machine
{
2982 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2984 my $machine = $forcemachine || $conf->{machine
};
2986 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2987 $kvmversion //= kvm_user_version
();
2988 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2989 # layout which confuses windows quite a bit and may result in various regressions..
2990 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2991 if (windows_version
($conf->{ostype
})) {
2992 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
2995 $machine ||= $default_machines->{$arch};
2996 if ($add_pve_version) {
2997 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
2998 $machine .= "+pve$pvever";
3002 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3003 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3004 $machine = $1 if $is_pxe;
3006 # for version-pinned machines that do not include a pve-version (e.g.
3007 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3008 $machine .= '+pve0';
3010 $machine .= '.pxe' if $is_pxe;
3016 sub get_ovmf_files
($) {
3019 my $ovmf = $OVMF->{$arch}
3020 or die "no OVMF images known for architecture '$arch'\n";
3026 aarch64
=> '/usr/bin/qemu-system-aarch64',
3027 x86_64
=> '/usr/bin/qemu-system-x86_64',
3029 sub get_command_for_arch
($) {
3031 return '/usr/bin/kvm' if is_native
($arch);
3033 my $cmd = $Arch2Qemu->{$arch}
3034 or die "don't know how to emulate architecture '$arch'\n";
3038 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3039 # to use in a QEMU command line (-cpu element), first array_intersect the result
3040 # of query_supported_ with query_understood_. This is necessary because:
3042 # a) query_understood_ returns flags the host cannot use and
3043 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3044 # flags, but CPU settings - with most of them being flags. Those settings
3045 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3047 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3048 # expensive. If you need the value returned from this, you can get it much
3049 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3050 # $accel being 'kvm' or 'tcg'.
3052 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3053 # changes, automatically populating pmxcfs.
3055 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3056 # since kvm and tcg machines support different flags
3058 sub query_supported_cpu_flags
{
3061 $arch //= get_host_arch
();
3062 my $default_machine = $default_machines->{$arch};
3066 # FIXME: Once this is merged, the code below should work for ARM as well:
3067 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3068 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3071 my $kvm_supported = defined(kvm_version
());
3072 my $qemu_cmd = get_command_for_arch
($arch);
3074 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3076 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3077 my $query_supported_run_qemu = sub {
3083 '-machine', $default_machine,
3085 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3086 '-mon', 'chardev=qmp,mode=control',
3087 '-pidfile', $pidfile,
3092 push @$cmd, '-accel', 'tcg';
3095 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3096 die "QEMU flag querying VM exited with code " . $rc if $rc;
3099 my $cmd_result = mon_cmd
(
3101 'query-cpu-model-expansion',
3103 model
=> { name
=> 'host' }
3106 my $props = $cmd_result->{model
}->{props
};
3107 foreach my $prop (keys %$props) {
3108 next if $props->{$prop} ne '1';
3109 # QEMU returns some flags multiple times, with '_', '.' or '-'
3110 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3111 # We only keep those with underscores, to match /proc/cpuinfo
3112 $prop =~ s/\.|-/_/g;
3113 $flags->{$prop} = 1;
3118 # force stop with 10 sec timeout and 'nocheck'
3119 # always stop, even if QMP failed
3120 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3124 return [ sort keys %$flags ];
3127 # We need to query QEMU twice, since KVM and TCG have different supported flags
3128 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3129 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3130 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3132 if ($kvm_supported) {
3133 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3134 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3141 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3142 my $understood_cpu_flag_dir = "/usr/share/kvm";
3143 sub query_understood_cpu_flags
{
3144 my $arch = get_host_arch
();
3145 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3147 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3150 my $raw = file_get_contents
($filepath);
3151 $raw =~ s/^\s+|\s+$//g;
3152 my @flags = split(/\s+/, $raw);
3157 sub config_to_command
{
3158 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3162 my $globalFlags = [];
3163 my $machineFlags = [];
3168 my $ostype = $conf->{ostype
};
3169 my $winversion = windows_version
($ostype);
3170 my $kvm = $conf->{kvm
};
3171 my $nodename = nodename
();
3173 my $arch = get_vm_arch
($conf);
3174 my $kvm_binary = get_command_for_arch
($arch);
3175 my $kvmver = kvm_user_version
($kvm_binary);
3177 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3178 $kvmver //= "undefined";
3179 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3182 my $add_pve_version = min_version
($kvmver, 4, 1);
3184 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3185 my $machine_version = extract_version
($machine_type, $kvmver);
3186 $kvm //= 1 if is_native
($arch);
3188 $machine_version =~ m/(\d+)\.(\d+)/;
3189 my ($machine_major, $machine_minor) = ($1, $2);
3191 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3192 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3193 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3194 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3195 ." please upgrade node '$nodename'\n"
3196 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3197 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3198 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3199 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3200 ." node '$nodename'\n";
3203 # if a specific +pve version is required for a feature, use $version_guard
3204 # instead of min_version to allow machines to be run with the minimum
3206 my $required_pve_version = 0;
3207 my $version_guard = sub {
3208 my ($major, $minor, $pve) = @_;
3209 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3210 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3211 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3212 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3216 if ($kvm && !defined kvm_version
()) {
3217 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3218 ." or enable in BIOS.\n";
3221 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3222 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3223 my $use_old_bios_files = undef;
3224 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3226 my $cpuunits = defined($conf->{cpuunits
}) ?
3227 $conf->{cpuunits
} : $defaults->{cpuunits
};
3229 push @$cmd, $kvm_binary;
3231 push @$cmd, '-id', $vmid;
3233 my $vmname = $conf->{name
} || "vm$vmid";
3235 push @$cmd, '-name', $vmname;
3237 push @$cmd, '-no-shutdown';
3241 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3242 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3243 push @$cmd, '-mon', "chardev=qmp,mode=control";
3245 if (min_version
($machine_version, 2, 12)) {
3246 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3247 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3250 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3252 push @$cmd, '-daemonize';
3254 if ($conf->{smbios1
}) {
3255 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3256 if ($smbios_conf->{base64
}) {
3257 # Do not pass base64 flag to qemu
3258 delete $smbios_conf->{base64
};
3259 my $smbios_string = "";
3260 foreach my $key (keys %$smbios_conf) {
3262 if ($key eq "uuid") {
3263 $value = $smbios_conf->{uuid
}
3265 $value = decode_base64
($smbios_conf->{$key});
3267 # qemu accepts any binary data, only commas need escaping by double comma
3269 $smbios_string .= "," . $key . "=" . $value if $value;
3271 push @$cmd, '-smbios', "type=1" . $smbios_string;
3273 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3277 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3278 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3279 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3281 my ($path, $format);
3282 if (my $efidisk = $conf->{efidisk0
}) {
3283 my $d = parse_drive
('efidisk0', $efidisk);
3284 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3285 $format = $d->{format
};
3287 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3288 if (!defined($format)) {
3289 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3290 $format = qemu_img_format
($scfg, $volname);
3294 die "efidisk format must be specified\n"
3295 if !defined($format);
3298 warn "no efidisk configured! Using temporary efivars disk.\n";
3299 $path = "/tmp/$vmid-ovmf.fd";
3300 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3306 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3307 $size_str = ",size=" . (-s
$ovmf_vars);
3310 # on slower ceph clusters, booting without cache on efidisk can take a while, see #3329
3311 my $cache = $path =~ m/^rbd:/ ?
',cache=writeback' : '';
3313 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3314 push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=$path";
3319 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3320 if (min_version
($machine_version, 4, 0)) {
3321 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3323 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3327 if ($conf->{vmgenid
}) {
3328 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3331 # add usb controllers
3332 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3333 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3334 push @$devices, @usbcontrollers if @usbcontrollers;
3335 my $vga = parse_vga
($conf->{vga
});
3337 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3338 $vga->{type
} = 'qxl' if $qxlnum;
3340 if (!$vga->{type
}) {
3341 if ($arch eq 'aarch64') {
3342 $vga->{type
} = 'virtio';
3343 } elsif (min_version
($machine_version, 2, 9)) {
3344 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3346 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3350 # enable absolute mouse coordinates (needed by vnc)
3352 if (defined($conf->{tablet
})) {
3353 $tablet = $conf->{tablet
};
3355 $tablet = $defaults->{tablet
};
3356 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3357 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3361 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3362 my $kbd = print_keyboarddevice_full
($conf, $arch);
3363 push @$devices, '-device', $kbd if defined($kbd);
3366 my $bootorder = device_bootorder
($conf);
3368 # host pci device passthrough
3369 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3370 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3373 my $usb_dev_features = {};
3374 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3376 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3377 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3378 push @$devices, @usbdevices if @usbdevices;
3381 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3382 if (my $path = $conf->{"serial$i"}) {
3383 if ($path eq 'socket') {
3384 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3385 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3386 # On aarch64, serial0 is the UART device. Qemu only allows
3387 # connecting UART devices via the '-serial' command line, as
3388 # the device has a fixed slot on the hardware...
3389 if ($arch eq 'aarch64' && $i == 0) {
3390 push @$devices, '-serial', "chardev:serial$i";
3392 push @$devices, '-device', "isa-serial,chardev=serial$i";
3395 die "no such serial device\n" if ! -c
$path;
3396 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3397 push @$devices, '-device', "isa-serial,chardev=serial$i";
3403 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3404 if (my $path = $conf->{"parallel$i"}) {
3405 die "no such parallel device\n" if ! -c
$path;
3406 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3407 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3408 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3412 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3413 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3414 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3415 push @$devices, @$audio_devs;
3419 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3420 $sockets = $conf->{sockets
} if $conf->{sockets
};
3422 my $cores = $conf->{cores
} || 1;
3424 my $maxcpus = $sockets * $cores;
3426 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3428 my $allowed_vcpus = $cpuinfo->{cpus
};
3430 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3431 if ($allowed_vcpus < $maxcpus);
3433 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3435 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3436 for (my $i = 2; $i <= $vcpus; $i++) {
3437 my $cpustr = print_cpu_device
($conf,$i);
3438 push @$cmd, '-device', $cpustr;
3443 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3445 push @$cmd, '-nodefaults';
3447 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3449 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3451 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3453 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3454 push @$devices, '-device', print_vga_device
(
3455 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3456 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3457 push @$cmd, '-vnc', "unix:$socket,password=on";
3459 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3460 push @$cmd, '-nographic';
3464 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3465 my $useLocaltime = $conf->{localtime};
3467 if ($winversion >= 5) { # windows
3468 $useLocaltime = 1 if !defined($conf->{localtime});
3470 # use time drift fix when acpi is enabled
3471 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3472 $tdf = 1 if !defined($conf->{tdf
});
3476 if ($winversion >= 6) {
3477 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3478 push @$cmd, '-no-hpet';
3481 push @$rtcFlags, 'driftfix=slew' if $tdf;
3483 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3484 push @$rtcFlags, "base=$conf->{startdate}";
3485 } elsif ($useLocaltime) {
3486 push @$rtcFlags, 'base=localtime';
3490 push @$cmd, '-cpu', $forcecpu;
3492 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3495 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3497 push @$cmd, '-S' if $conf->{freeze
};
3499 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3501 my $guest_agent = parse_guest_agent
($conf);
3503 if ($guest_agent->{enabled
}) {
3504 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3505 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3507 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3508 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3509 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3510 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3511 } elsif ($guest_agent->{type
} eq 'isa') {
3512 push @$devices, '-device', "isa-serial,chardev=qga0";
3516 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3517 if ($rng && $version_guard->(4, 1, 2)) {
3518 check_rng_source
($rng->{source
});
3520 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3521 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3522 my $limiter_str = "";
3524 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3527 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3528 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3529 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3537 for (my $i = 1; $i < $qxlnum; $i++){
3538 push @$devices, '-device', print_vga_device
(
3539 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3542 # assume other OS works like Linux
3543 my ($ram, $vram) = ("134217728", "67108864");
3544 if ($vga->{memory
}) {
3545 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3546 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3548 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3549 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3553 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3555 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3556 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3557 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3559 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3560 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3561 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3563 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3564 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3566 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3567 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3568 if ($spice_enhancement->{foldersharing
}) {
3569 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3570 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3573 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3574 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3575 if $spice_enhancement->{videostreaming
};
3577 push @$devices, '-spice', "$spice_opts";
3580 # enable balloon by default, unless explicitly disabled
3581 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3582 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3583 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3586 if ($conf->{watchdog
}) {
3587 my $wdopts = parse_watchdog
($conf->{watchdog
});
3588 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3589 my $watchdog = $wdopts->{model
} || 'i6300esb';
3590 push @$devices, '-device', "$watchdog$pciaddr";
3591 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3595 my $scsicontroller = {};
3596 my $ahcicontroller = {};
3597 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3599 # Add iscsi initiator name if available
3600 if (my $initiator = get_initiator_name
()) {
3601 push @$devices, '-iscsi', "initiator-name=$initiator";
3604 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3605 my ($ds, $drive) = @_;
3607 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3608 push @$vollist, $drive->{file
};
3611 # ignore efidisk here, already added in bios/fw handling code above
3612 return if $drive->{interface
} eq 'efidisk';
3614 $use_virtio = 1 if $ds =~ m/^virtio/;
3616 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3618 if ($drive->{interface
} eq 'virtio'){
3619 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3622 if ($drive->{interface
} eq 'scsi') {
3624 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3626 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3627 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3629 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3630 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3633 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3634 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3635 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3636 } elsif ($drive->{iothread
}) {
3637 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3641 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3642 $queues = ",num_queues=$drive->{queues}";
3645 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3646 if !$scsicontroller->{$controller};
3647 $scsicontroller->{$controller}=1;
3650 if ($drive->{interface
} eq 'sata') {
3651 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3652 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3653 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3654 if !$ahcicontroller->{$controller};
3655 $ahcicontroller->{$controller}=1;
3658 my $pbs_conf = $pbs_backing->{$ds};
3659 my $pbs_name = undef;
3661 $pbs_name = "drive-$ds-pbs";
3662 push @$devices, '-blockdev', print_pbs_blockdev
($pbs_conf, $pbs_name);
3665 my $drive_cmd = print_drive_commandline_full
($storecfg, $vmid, $drive, $pbs_name);
3667 # extra protection for templates, but SATA and IDE don't support it..
3668 my $read_only = PVE
::QemuConfig-
>is_template($conf)
3669 && $drive->{interface
} ne 'sata'
3670 && $drive->{interface
} ne 'ide';
3672 $drive_cmd .= ',readonly=on' if $read_only;
3674 push @$devices, '-drive',$drive_cmd;
3675 push @$devices, '-device', print_drivedevice_full
(
3676 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3679 for (my $i = 0; $i < $MAX_NETS; $i++) {
3680 my $netname = "net$i";
3682 next if !$conf->{$netname};
3683 my $d = parse_net
($conf->{$netname});
3686 $use_virtio = 1 if $d->{model
} eq 'virtio';
3688 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
3690 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
3691 push @$devices, '-netdev', $netdevfull;
3693 my $netdevicefull = print_netdevice_full
(
3694 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3696 push @$devices, '-device', $netdevicefull;
3699 if ($conf->{ivshmem
}) {
3700 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3704 $bus = print_pcie_addr
("ivshmem");
3706 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3709 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3710 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3712 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3713 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3714 .",size=$ivshmem->{size}M";
3717 # pci.4 is nested in pci.1
3718 $bridges->{1} = 1 if $bridges->{4};
3722 if (min_version
($machine_version, 2, 3)) {
3727 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3731 for my $k (sort {$b cmp $a} keys %$bridges) {
3732 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3735 if ($k == 2 && $legacy_igd) {
3738 $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
3740 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3742 # add after -readconfig pve-q35.cfg
3743 splice @$devices, 2, 0, '-device', $devstr;
3745 unshift @$devices, '-device', $devstr if $k > 0;
3750 push @$machineFlags, 'accel=tcg';
3753 my $machine_type_min = $machine_type;
3754 if ($add_pve_version) {
3755 $machine_type_min =~ s/\+pve\d+$//;
3756 $machine_type_min .= "+pve$required_pve_version";
3758 push @$machineFlags, "type=${machine_type_min}";
3760 push @$cmd, @$devices;
3761 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3762 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3763 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3765 if (my $vmstate = $conf->{vmstate
}) {
3766 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3767 push @$vollist, $vmstate;
3768 push @$cmd, '-loadstate', $statepath;
3769 print "activating and using '$vmstate' as vmstate\n";
3773 if ($conf->{args
}) {
3774 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3778 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3781 sub check_rng_source
{
3784 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3785 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3788 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3789 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
3790 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3791 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3792 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3793 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3801 my $res = mon_cmd
($vmid, 'query-spice');
3803 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3806 sub vm_devices_list
{
3809 my $res = mon_cmd
($vmid, 'query-pci');
3810 my $devices_to_check = [];
3812 foreach my $pcibus (@$res) {
3813 push @$devices_to_check, @{$pcibus->{devices
}},
3816 while (@$devices_to_check) {
3818 for my $d (@$devices_to_check) {
3819 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3820 next if !$d->{'pci_bridge'};
3822 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3823 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3825 $devices_to_check = $to_check;
3828 my $resblock = mon_cmd
($vmid, 'query-block');
3829 foreach my $block (@$resblock) {
3830 if($block->{device
} =~ m/^drive-(\S+)/){
3835 my $resmice = mon_cmd
($vmid, 'query-mice');
3836 foreach my $mice (@$resmice) {
3837 if ($mice->{name
} eq 'QEMU HID Tablet') {
3838 $devices->{tablet
} = 1;
3843 # for usb devices there is no query-usb
3844 # but we can iterate over the entries in
3845 # qom-list path=/machine/peripheral
3846 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3847 foreach my $per (@$resperipheral) {
3848 if ($per->{name
} =~ m/^usb\d+$/) {
3849 $devices->{$per->{name
}} = 1;
3857 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3859 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3861 my $devices_list = vm_devices_list
($vmid);
3862 return 1 if defined($devices_list->{$deviceid});
3864 # add PCI bridge if we need it for the device
3865 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3867 if ($deviceid eq 'tablet') {
3869 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
3871 } elsif ($deviceid eq 'keyboard') {
3873 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
3875 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3877 die "usb hotplug currently not reliable\n";
3878 # since we can't reliably hot unplug all added usb devices and usb
3879 # passthrough breaks live migration we disable usb hotplugging for now
3880 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3882 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3884 qemu_iothread_add
($vmid, $deviceid, $device);
3886 qemu_driveadd
($storecfg, $vmid, $device);
3887 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3889 qemu_deviceadd
($vmid, $devicefull);
3890 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3892 eval { qemu_drivedel
($vmid, $deviceid); };
3897 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3900 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3901 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3902 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3904 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3906 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3907 qemu_iothread_add
($vmid, $deviceid, $device);
3908 $devicefull .= ",iothread=iothread-$deviceid";
3911 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3912 $devicefull .= ",num_queues=$device->{queues}";
3915 qemu_deviceadd
($vmid, $devicefull);
3916 qemu_deviceaddverify
($vmid, $deviceid);
3918 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3920 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3921 qemu_driveadd
($storecfg, $vmid, $device);
3923 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3924 eval { qemu_deviceadd
($vmid, $devicefull); };
3926 eval { qemu_drivedel
($vmid, $deviceid); };
3931 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3933 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
3935 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
3936 my $use_old_bios_files = undef;
3937 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3939 my $netdevicefull = print_netdevice_full
(
3940 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3941 qemu_deviceadd
($vmid, $netdevicefull);
3943 qemu_deviceaddverify
($vmid, $deviceid);
3944 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
3947 eval { qemu_netdevdel
($vmid, $deviceid); };
3952 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3955 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3956 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3958 qemu_deviceadd
($vmid, $devicefull);
3959 qemu_deviceaddverify
($vmid, $deviceid);
3962 die "can't hotplug device '$deviceid'\n";
3968 # fixme: this should raise exceptions on error!
3969 sub vm_deviceunplug
{
3970 my ($vmid, $conf, $deviceid) = @_;
3972 my $devices_list = vm_devices_list
($vmid);
3973 return 1 if !defined($devices_list->{$deviceid});
3975 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
3976 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3978 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3980 qemu_devicedel
($vmid, $deviceid);
3982 } elsif ($deviceid =~ m/^usb\d+$/) {
3984 die "usb hotplug currently not reliable\n";
3985 # when unplugging usb devices this way, there may be remaining usb
3986 # controllers/hubs so we disable it for now
3987 #qemu_devicedel($vmid, $deviceid);
3988 #qemu_devicedelverify($vmid, $deviceid);
3990 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3992 qemu_devicedel
($vmid, $deviceid);
3993 qemu_devicedelverify
($vmid, $deviceid);
3994 qemu_drivedel
($vmid, $deviceid);
3995 qemu_iothread_del
($conf, $vmid, $deviceid);
3997 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3999 qemu_devicedel
($vmid, $deviceid);
4000 qemu_devicedelverify
($vmid, $deviceid);
4001 qemu_iothread_del
($conf, $vmid, $deviceid);
4003 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4005 qemu_devicedel
($vmid, $deviceid);
4006 qemu_drivedel
($vmid, $deviceid);
4007 qemu_deletescsihw
($conf, $vmid, $deviceid);
4009 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4011 qemu_devicedel
($vmid, $deviceid);
4012 qemu_devicedelverify
($vmid, $deviceid);
4013 qemu_netdevdel
($vmid, $deviceid);
4016 die "can't unplug device '$deviceid'\n";
4022 sub qemu_deviceadd
{
4023 my ($vmid, $devicefull) = @_;
4025 $devicefull = "driver=".$devicefull;
4026 my %options = split(/[=,]/, $devicefull);
4028 mon_cmd
($vmid, "device_add" , %options);
4031 sub qemu_devicedel
{
4032 my ($vmid, $deviceid) = @_;
4034 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4037 sub qemu_iothread_add
{
4038 my($vmid, $deviceid, $device) = @_;
4040 if ($device->{iothread
}) {
4041 my $iothreads = vm_iothreads_list
($vmid);
4042 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4046 sub qemu_iothread_del
{
4047 my($conf, $vmid, $deviceid) = @_;
4049 my $confid = $deviceid;
4050 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4051 $confid = 'scsi' . $1;
4053 my $device = parse_drive
($confid, $conf->{$confid});
4054 if ($device->{iothread
}) {
4055 my $iothreads = vm_iothreads_list
($vmid);
4056 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4060 sub qemu_objectadd
{
4061 my($vmid, $objectid, $qomtype) = @_;
4063 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4068 sub qemu_objectdel
{
4069 my($vmid, $objectid) = @_;
4071 mon_cmd
($vmid, "object-del", id
=> $objectid);
4077 my ($storecfg, $vmid, $device) = @_;
4079 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device);
4080 $drive =~ s/\\/\\\\/g;
4081 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4083 # If the command succeeds qemu prints: "OK
"
4084 return 1 if $ret =~ m/OK/s;
4086 die "adding drive failed
: $ret\n";
4090 my($vmid, $deviceid) = @_;
4092 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4095 return 1 if $ret eq "";
4097 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4098 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4100 die "deleting drive
$deviceid failed
: $ret\n";
4103 sub qemu_deviceaddverify {
4104 my ($vmid, $deviceid) = @_;
4106 for (my $i = 0; $i <= 5; $i++) {
4107 my $devices_list = vm_devices_list($vmid);
4108 return 1 if defined($devices_list->{$deviceid});
4112 die "error on hotplug device
'$deviceid'\n";
4116 sub qemu_devicedelverify {
4117 my ($vmid, $deviceid) = @_;
4119 # need to verify that the device is correctly removed as device_del
4120 # is async and empty return is not reliable
4122 for (my $i = 0; $i <= 5; $i++) {
4123 my $devices_list = vm_devices_list($vmid);
4124 return 1 if !defined($devices_list->{$deviceid});
4128 die "error on hot-unplugging device
'$deviceid'\n";
4131 sub qemu_findorcreatescsihw {
4132 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4134 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4136 my $scsihwid="$controller_prefix$controller";
4137 my $devices_list = vm_devices_list($vmid);
4139 if(!defined($devices_list->{$scsihwid})) {
4140 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4146 sub qemu_deletescsihw {
4147 my ($conf, $vmid, $opt) = @_;
4149 my $device = parse_drive($opt, $conf->{$opt});
4151 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4152 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4156 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4158 my $devices_list = vm_devices_list($vmid);
4159 foreach my $opt (keys %{$devices_list}) {
4160 if (is_valid_drivename($opt)) {
4161 my $drive = parse_drive($opt, $conf->{$opt});
4162 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4168 my $scsihwid="scsihw
$controller";
4170 vm_deviceunplug($vmid, $conf, $scsihwid);
4175 sub qemu_add_pci_bridge {
4176 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4182 print_pci_addr($device, $bridges, $arch, $machine_type);
4184 while (my ($k, $v) = each %$bridges) {
4187 return 1 if !defined($bridgeid) || $bridgeid < 1;
4189 my $bridge = "pci
.$bridgeid";
4190 my $devices_list = vm_devices_list($vmid);
4192 if (!defined($devices_list->{$bridge})) {
4193 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4199 sub qemu_set_link_status {
4200 my ($vmid, $device, $up) = @_;
4202 mon_cmd($vmid, "set_link
", name => $device,
4203 up => $up ? JSON::true : JSON::false);
4206 sub qemu_netdevadd {
4207 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4209 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4210 my %options = split(/[=,]/, $netdev);
4212 if (defined(my $vhost = $options{vhost})) {
4213 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4216 if (defined(my $queues = $options{queues})) {
4217 $options{queues} = $queues + 0;
4220 mon_cmd($vmid, "netdev_add
", %options);
4224 sub qemu_netdevdel {
4225 my ($vmid, $deviceid) = @_;
4227 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4230 sub qemu_usb_hotplug {
4231 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4235 # remove the old one first
4236 vm_deviceunplug($vmid, $conf, $deviceid);
4238 # check if xhci controller is necessary and available
4239 if ($device->{usb3}) {
4241 my $devicelist = vm_devices_list($vmid);
4243 if (!$devicelist->{xhci}) {
4244 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4245 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4248 my $d = parse_usb_device($device->{host});
4249 $d->{usb3} = $device->{usb3};
4252 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4255 sub qemu_cpu_hotplug {
4256 my ($vmid, $conf, $vcpus) = @_;
4258 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4261 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4262 $sockets = $conf->{sockets} if $conf->{sockets};
4263 my $cores = $conf->{cores} || 1;
4264 my $maxcpus = $sockets * $cores;
4266 $vcpus = $maxcpus if !$vcpus;
4268 die "you can
't add more vcpus than maxcpus\n"
4269 if $vcpus > $maxcpus;
4271 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4273 if ($vcpus < $currentvcpus) {
4275 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4277 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4278 qemu_devicedel($vmid, "cpu$i");
4280 my $currentrunningvcpus = undef;
4282 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4283 last if scalar(@{$currentrunningvcpus}) == $i-1;
4284 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4288 #update conf after each succesfull cpu unplug
4289 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4290 PVE::QemuConfig->write_config($vmid, $conf);
4293 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4299 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4300 die "vcpus in running vm does not match its configuration\n"
4301 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4303 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4305 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4306 my $cpustr = print_cpu_device($conf, $i);
4307 qemu_deviceadd($vmid, $cpustr);
4310 my $currentrunningvcpus = undef;
4312 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4313 last if scalar(@{$currentrunningvcpus}) == $i;
4314 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4318 #update conf after each succesfull cpu hotplug
4319 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4320 PVE::QemuConfig->write_config($vmid, $conf);
4324 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4325 mon_cmd($vmid, "cpu-add", id => int($i));
4330 sub qemu_block_set_io_throttle {
4331 my ($vmid, $deviceid,
4332 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4333 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4334 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4335 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4337 return if !check_running($vmid) ;
4339 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4341 bps_rd => int($bps_rd),
4342 bps_wr => int($bps_wr),
4344 iops_rd => int($iops_rd),
4345 iops_wr => int($iops_wr),
4346 bps_max => int($bps_max),
4347 bps_rd_max => int($bps_rd_max),
4348 bps_wr_max => int($bps_wr_max),
4349 iops_max => int($iops_max),
4350 iops_rd_max => int($iops_rd_max),
4351 iops_wr_max => int($iops_wr_max),
4352 bps_max_length => int($bps_max_length),
4353 bps_rd_max_length => int($bps_rd_max_length),
4354 bps_wr_max_length => int($bps_wr_max_length),
4355 iops_max_length => int($iops_max_length),
4356 iops_rd_max_length => int($iops_rd_max_length),
4357 iops_wr_max_length => int($iops_wr_max_length),
4362 sub qemu_block_resize {
4363 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4365 my $running = check_running($vmid);
4367 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4369 return if !$running;
4371 my $padding = (1024 - $size % 1024) % 1024;
4372 $size = $size + $padding;
4377 device => $deviceid,
4383 sub qemu_volume_snapshot {
4384 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4386 my $running = check_running($vmid);
4388 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4389 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4391 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4395 sub qemu_volume_snapshot_delete {
4396 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4398 my $running = check_running($vmid);
4403 my $conf = PVE::QemuConfig->load_config($vmid);
4404 PVE::QemuConfig->foreach_volume($conf, sub {
4405 my ($ds, $drive) = @_;
4406 $running = 1 if $drive->{file} eq $volid;
4410 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4411 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4413 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4417 sub set_migration_caps {
4418 my ($vmid, $savevm) = @_;
4420 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4422 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4423 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4428 "auto-converge" => 1,
4430 "x-rdma-pin-all" => 0,
4433 "dirty-bitmaps" => $dirty_bitmaps,
4436 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4438 for my $supported_capability (@$supported_capabilities) {
4440 capability => $supported_capability->{capability},
4441 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4445 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4449 my ($conf, $func, @param) = @_;
4453 my $test_volid = sub {
4454 my ($key, $drive, $snapname) = @_;
4456 my $volid = $drive->{file};
4459 $volhash->{$volid}->{cdrom} //= 1;
4460 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4462 my $replicate = $drive->{replicate} // 1;
4463 $volhash->{$volid}->{replicate} //= 0;
4464 $volhash->{$volid}->{replicate} = 1 if $replicate;
4466 $volhash->{$volid}->{shared} //= 0;
4467 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4469 $volhash->{$volid}->{referenced_in_config} //= 0;
4470 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4472 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4473 if defined($snapname);
4475 my $size = $drive->{size};
4476 $volhash->{$volid}->{size} //= $size if $size;
4478 $volhash->{$volid}->{is_vmstate} //= 0;
4479 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4481 $volhash->{$volid}->{is_unused} //= 0;
4482 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4484 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4487 my $include_opts = {
4488 extra_keys => ['vmstate
'],
4489 include_unused => 1,
4492 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4493 foreach my $snapname (keys %{$conf->{snapshots}}) {
4494 my $snap = $conf->{snapshots}->{$snapname};
4495 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4498 foreach my $volid (keys %$volhash) {
4499 &$func($volid, $volhash->{$volid}, @param);
4503 my $fast_plug_option = {
4511 'vmstatestorage
' => 1,
4516 # hotplug changes in [PENDING]
4517 # $selection hash can be used to only apply specified options, for
4518 # example: { cores => 1 } (only apply changed 'cores
')
4519 # $errors ref is used to return error messages
4520 sub vmconfig_hotplug_pending {
4521 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4523 my $defaults = load_defaults();
4524 my $arch = get_vm_arch($conf);
4525 my $machine_type = get_vm_machine($conf, undef, $arch);
4527 # commit values which do not have any impact on running VM first
4528 # Note: those option cannot raise errors, we we do not care about
4529 # $selection and always apply them.
4531 my $add_error = sub {
4532 my ($opt, $msg) = @_;
4533 $errors->{$opt} = "hotplug problem - $msg";
4537 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4538 if ($fast_plug_option->{$opt}) {
4539 $conf->{$opt} = $conf->{pending}->{$opt};
4540 delete $conf->{pending}->{$opt};
4546 PVE::QemuConfig->write_config($vmid, $conf);
4549 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4551 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4552 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4553 foreach my $opt (sort keys %$pending_delete_hash) {
4554 next if $selection && !$selection->{$opt};
4555 my $force = $pending_delete_hash->{$opt}->{force};
4557 if ($opt eq 'hotplug
') {
4558 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4559 } elsif ($opt eq 'tablet
') {
4560 die "skip\n" if !$hotplug_features->{usb};
4561 if ($defaults->{tablet}) {
4562 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4563 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4564 if $arch eq 'aarch64
';
4566 vm_deviceunplug($vmid, $conf, 'tablet
');
4567 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4569 } elsif ($opt =~ m/^usb\d+/) {
4571 # since we cannot reliably hot unplug usb devices we are disabling it
4572 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4573 #vm_deviceunplug($vmid, $conf, $opt);
4574 } elsif ($opt eq 'vcpus
') {
4575 die "skip\n" if !$hotplug_features->{cpu};
4576 qemu_cpu_hotplug($vmid, $conf, undef);
4577 } elsif ($opt eq 'balloon
') {
4578 # enable balloon device is not hotpluggable
4579 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4580 # here we reset the ballooning value to memory
4581 my $balloon = $conf->{memory} || $defaults->{memory};
4582 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4583 } elsif ($fast_plug_option->{$opt}) {
4585 } elsif ($opt =~ m/^net(\d+)$/) {
4586 die "skip\n" if !$hotplug_features->{network};
4587 vm_deviceunplug($vmid, $conf, $opt);
4588 } elsif (is_valid_drivename($opt)) {
4589 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4590 vm_deviceunplug($vmid, $conf, $opt);
4591 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4592 } elsif ($opt =~ m/^memory$/) {
4593 die "skip\n" if !$hotplug_features->{memory};
4594 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4595 } elsif ($opt eq 'cpuunits
') {
4596 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4597 } elsif ($opt eq 'cpulimit
') {
4598 $cgroup->change_cpu_quota(-1, 100000);
4604 &$add_error($opt, $err) if $err ne "skip\n";
4606 delete $conf->{$opt};
4607 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4611 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4612 $apply_pending_cloudinit = sub {
4613 return if $apply_pending_cloudinit_done; # once is enough
4614 $apply_pending_cloudinit_done = 1; # once is enough
4616 my ($key, $value) = @_;
4618 my @cloudinit_opts = keys %$confdesc_cloudinit;
4619 foreach my $opt (keys %{$conf->{pending}}) {
4620 next if !grep { $_ eq $opt } @cloudinit_opts;
4621 $conf->{$opt} = delete $conf->{pending}->{$opt};
4624 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4625 foreach my $opt (sort keys %$pending_delete_hash) {
4626 next if !grep { $_ eq $opt } @cloudinit_opts;
4627 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4628 delete $conf->{$opt};
4631 my $new_conf = { %$conf };
4632 $new_conf->{$key} = $value;
4633 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4636 foreach my $opt (keys %{$conf->{pending}}) {
4637 next if $selection && !$selection->{$opt};
4638 my $value = $conf->{pending}->{$opt};
4640 if ($opt eq 'hotplug
') {
4641 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4642 } elsif ($opt eq 'tablet
') {
4643 die "skip\n" if !$hotplug_features->{usb};
4645 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4646 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4647 if $arch eq 'aarch64
';
4648 } elsif ($value == 0) {
4649 vm_deviceunplug($vmid, $conf, 'tablet
');
4650 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4652 } elsif ($opt =~ m/^usb\d+$/) {
4654 # since we cannot reliably hot unplug usb devices we disable it for now
4655 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4656 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4657 #die "skip\n" if !$d;
4658 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4659 } elsif ($opt eq 'vcpus
') {
4660 die "skip\n" if !$hotplug_features->{cpu};
4661 qemu_cpu_hotplug($vmid, $conf, $value);
4662 } elsif ($opt eq 'balloon
') {
4663 # enable/disable balloning device is not hotpluggable
4664 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4665 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4666 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4668 # allow manual ballooning if shares is set to zero
4669 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4670 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4671 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4673 } elsif ($opt =~ m/^net(\d+)$/) {
4674 # some changes can be done without hotplug
4675 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4676 $vmid, $opt, $value, $arch, $machine_type);
4677 } elsif (is_valid_drivename($opt)) {
4678 die "skip\n" if $opt eq 'efidisk0
';
4679 # some changes can be done without hotplug
4680 my $drive = parse_drive($opt, $value);
4681 if (drive_is_cloudinit($drive)) {
4682 &$apply_pending_cloudinit($opt, $value);
4684 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4685 $vmid, $opt, $value, $arch, $machine_type);
4686 } elsif ($opt =~ m/^memory$/) { #dimms
4687 die "skip\n" if !$hotplug_features->{memory};
4688 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4689 } elsif ($opt eq 'cpuunits
') {
4690 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4691 } elsif ($opt eq 'cpulimit
') {
4692 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4693 $cgroup->change_cpu_quota($cpulimit, 100000);
4695 die "skip\n"; # skip non-hot-pluggable options
4699 &$add_error($opt, $err) if $err ne "skip\n";
4701 $conf->{$opt} = $value;
4702 delete $conf->{pending}->{$opt};
4706 PVE::QemuConfig->write_config($vmid, $conf);
4709 sub try_deallocate_drive {
4710 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4712 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4713 my $volid = $drive->{file};
4714 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4715 my $sid = PVE::Storage::parse_volume_id($volid);
4716 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4718 # check if the disk is really unused
4719 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4720 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4721 PVE::Storage::vdisk_free($storecfg, $volid);
4724 # If vm is not owner of this disk remove from config
4732 sub vmconfig_delete_or_detach_drive {
4733 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4735 my $drive = parse_drive($opt, $conf->{$opt});
4737 my $rpcenv = PVE::RPCEnvironment::get();
4738 my $authuser = $rpcenv->get_user();
4741 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4742 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4744 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4750 sub vmconfig_apply_pending {
4751 my ($vmid, $conf, $storecfg, $errors) = @_;
4753 my $add_apply_error = sub {
4754 my ($opt, $msg) = @_;
4755 my $err_msg = "unable to apply pending change $opt : $msg";
4756 $errors->{$opt} = $err_msg;
4762 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4763 foreach my $opt (sort keys %$pending_delete_hash) {
4764 my $force = $pending_delete_hash->{$opt}->{force};
4766 if ($opt =~ m/^unused/) {
4767 die "internal error";
4768 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4769 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4773 $add_apply_error->($opt, $err);
4775 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4776 delete $conf->{$opt};
4780 PVE::QemuConfig->cleanup_pending($conf);
4782 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4783 next if $opt eq 'delete'; # just to be sure
4785 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4786 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4790 $add_apply_error->($opt, $err);
4792 $conf->{$opt} = delete $conf->{pending}->{$opt};
4796 # write all changes at once to avoid unnecessary i/o
4797 PVE::QemuConfig->write_config($vmid, $conf);
4800 sub vmconfig_update_net {
4801 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4803 my $newnet = parse_net($value);
4805 if ($conf->{$opt}) {
4806 my $oldnet = parse_net($conf->{$opt});
4808 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4809 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4810 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4811 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4813 # for non online change, we try to hot-unplug
4814 die "skip\n" if !$hotplug;
4815 vm_deviceunplug($vmid, $conf, $opt);
4818 die "internal error" if $opt !~ m/net(\d+)/;
4819 my $iface = "tap${vmid}i$1";
4821 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4822 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4823 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4824 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4825 PVE::Network::tap_unplug($iface);
4828 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4830 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4832 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4833 # Rate can be applied on its own but any change above needs to
4834 # include the rate in tap_plug since OVS resets everything.
4835 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4838 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4839 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4847 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4853 sub vmconfig_update_disk {
4854 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4856 my $drive = parse_drive($opt, $value);
4858 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4859 my $media = $drive->{media} || 'disk
';
4860 my $oldmedia = $old_drive->{media} || 'disk
';
4861 die "unable to change media type\n" if $media ne $oldmedia;
4863 if (!drive_is_cdrom($old_drive)) {
4865 if ($drive->{file} ne $old_drive->{file}) {
4867 die "skip\n" if !$hotplug;
4869 # unplug and register as unused
4870 vm_deviceunplug($vmid, $conf, $opt);
4871 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4874 # update existing disk
4876 # skip non hotpluggable value
4877 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4878 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4879 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4880 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4881 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4886 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4887 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4888 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4889 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4890 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4891 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4892 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4893 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4894 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4895 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4896 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4897 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4898 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4899 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4900 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4901 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4902 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4903 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4905 qemu_block_set_io_throttle(
4907 ($drive->{mbps} || 0)*1024*1024,
4908 ($drive->{mbps_rd} || 0)*1024*1024,
4909 ($drive->{mbps_wr} || 0)*1024*1024,
4910 $drive->{iops} || 0,
4911 $drive->{iops_rd} || 0,
4912 $drive->{iops_wr} || 0,
4913 ($drive->{mbps_max} || 0)*1024*1024,
4914 ($drive->{mbps_rd_max} || 0)*1024*1024,
4915 ($drive->{mbps_wr_max} || 0)*1024*1024,
4916 $drive->{iops_max} || 0,
4917 $drive->{iops_rd_max} || 0,
4918 $drive->{iops_wr_max} || 0,
4919 $drive->{bps_max_length} || 1,
4920 $drive->{bps_rd_max_length} || 1,
4921 $drive->{bps_wr_max_length} || 1,
4922 $drive->{iops_max_length} || 1,
4923 $drive->{iops_rd_max_length} || 1,
4924 $drive->{iops_wr_max_length} || 1,
4934 if ($drive->{file} eq 'none
') {
4935 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4936 if (drive_is_cloudinit($old_drive)) {
4937 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4940 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4942 # force eject if locked
4943 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4946 mon_cmd($vmid, "blockdev-change-medium",
4947 id => "$opt", filename => "$path");
4955 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4957 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4958 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4961 # called in locked context by incoming migration
4962 sub vm_migrate_get_nbd_disks {
4963 my ($storecfg, $conf, $replicated_volumes) = @_;
4965 my $local_volumes = {};
4966 PVE::QemuConfig->foreach_volume($conf, sub {
4967 my ($ds, $drive) = @_;
4969 return if drive_is_cdrom($drive);
4971 my $volid = $drive->{file};
4975 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4977 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4978 return if $scfg->{shared};
4980 # replicated disks re-use existing state via bitmap
4981 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4982 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4984 return $local_volumes;
4987 # called in locked context by incoming migration
4988 sub vm_migrate_alloc_nbd_disks {
4989 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4994 foreach my $opt (sort keys %$source_volumes) {
4995 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4997 if ($use_existing) {
4998 $nbd->{$opt}->{drivestr} = print_drive($drive);
4999 $nbd->{$opt}->{volid} = $volid;
5000 $nbd->{$opt}->{replicated} = 1;
5004 # If a remote storage is specified and the format of the original
5005 # volume is not available there, fall back to the default format.
5006 # Otherwise use the same format as the original.
5007 if (!$storagemap->{identity}) {
5008 $storeid = map_storage($storagemap, $storeid);
5009 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5010 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5011 my $fileFormat = qemu_img_format($scfg, $volname);
5012 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5014 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5015 $format = qemu_img_format($scfg, $volname);
5018 my $size = $drive->{size} / 1024;
5019 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5020 my $newdrive = $drive;
5021 $newdrive->{format} = $format;
5022 $newdrive->{file} = $newvolid;
5023 my $drivestr = print_drive($newdrive);
5024 $nbd->{$opt}->{drivestr} = $drivestr;
5025 $nbd->{$opt}->{volid} = $newvolid;
5031 # see vm_start_nolock for parameters, additionally:
5033 # storagemap = parsed storage map for allocating NBD disks
5035 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5037 return PVE::QemuConfig->lock_config($vmid, sub {
5038 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5040 die "you can't start a vm
if it
's a template\n"
5041 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5043 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended
');
5044 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup
');
5046 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5048 if ($has_backup_lock && $running) {
5049 # a backup is currently running, attempt to start the guest in the
5050 # existing QEMU instance
5051 return vm_resume($vmid);
5054 PVE::QemuConfig->check_lock($conf)
5055 if !($params->{skiplock} || $has_suspended_lock);
5057 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5059 die "VM $vmid already running\n" if $running;
5061 if (my $storagemap = $migrate_opts->{storagemap}) {
5062 my $replicated = $migrate_opts->{replicated_volumes};
5063 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5064 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5066 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5067 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5071 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5077 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
5078 # skiplock => 0/1, skip checking for config lock
5079 # skiptemplate => 0/1, skip checking whether VM is template
5080 # forcemachine => to force Qemu machine (rollback/migration)
5081 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
5082 # timeout => in seconds
5083 # paused => start VM in paused state (backup)
5084 # resume => resume from hibernation
5095 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5096 # migratedfrom => source node
5097 # spice_ticket => used for spice migration, passed via tunnel/stdin
5098 # network => CIDR of migration network
5099 # type => secure/insecure - tunnel over encrypted connection or plain-text
5100 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5101 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5102 sub vm_start_nolock {
5103 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5105 my $statefile = $params->{statefile};
5106 my $resume = $params->{resume};
5108 my $migratedfrom = $migrate_opts->{migratedfrom};
5109 my $migration_type = $migrate_opts->{type};
5113 # clean up leftover reboot request files
5114 eval { clear_reboot_request($vmid); };
5117 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5118 vmconfig_apply_pending($vmid, $conf, $storecfg);
5119 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5122 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5124 my $defaults = load_defaults();
5126 # set environment variable useful inside network script
5127 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5129 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
5131 my $forcemachine = $params->{forcemachine};
5132 my $forcecpu = $params->{forcecpu};
5134 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5135 $forcemachine = $conf->{runningmachine};
5136 $forcecpu = $conf->{runningcpu};
5137 print "Resuming suspended VM\n";
5140 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5141 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing
'});
5144 my $get_migration_ip = sub {
5145 my ($nodename) = @_;
5147 return $migration_ip if defined($migration_ip);
5149 my $cidr = $migrate_opts->{network};
5151 if (!defined($cidr)) {
5152 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5153 $cidr = $dc_conf->{migration}->{network};
5156 if (defined($cidr)) {
5157 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5159 die "could not get IP: no address configured on local " .
5160 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5162 die "could not get IP: multiple addresses configured on local " .
5163 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5165 $migration_ip = @$ips[0];
5168 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5169 if !defined($migration_ip);
5171 return $migration_ip;
5176 if ($statefile eq 'tcp
') {
5177 my $localip = "localhost";
5178 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5179 my $nodename = nodename();
5181 if (!defined($migration_type)) {
5182 if (defined($datacenterconf->{migration}->{type})) {
5183 $migration_type = $datacenterconf->{migration}->{type};
5185 $migration_type = 'secure
';
5189 if ($migration_type eq 'insecure
') {
5190 $localip = $get_migration_ip->($nodename);
5191 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5194 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5195 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5196 $migrate_uri = "tcp:${localip}:${migrate_port}";
5197 push @$cmd, '-incoming
', $migrate_uri;
5200 } elsif ($statefile eq 'unix
') {
5201 # should be default for secure migrations as a ssh TCP forward
5202 # tunnel is not deterministic reliable ready and fails regurarly
5203 # to set up in time, so use UNIX socket forwards
5204 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5205 unlink $socket_addr;
5207 $migrate_uri = "unix:$socket_addr";
5209 push @$cmd, '-incoming
', $migrate_uri;
5212 } elsif (-e $statefile) {
5213 push @$cmd, '-loadstate
', $statefile;
5215 my $statepath = PVE::Storage::path($storecfg, $statefile);
5216 push @$vollist, $statefile;
5217 push @$cmd, '-loadstate
', $statepath;
5219 } elsif ($params->{paused}) {
5224 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
5225 my $d = parse_hostpci($conf->{"hostpci$i"});
5227 my $pcidevices = $d->{pciid};
5228 foreach my $pcidevice (@$pcidevices) {
5229 my $pciid = $pcidevice->{id};
5231 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5232 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5233 die "no pci device info for device '$pciid'\n" if !$info;
5236 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5237 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5239 die "can't unbind
/bind PCI group to VFIO
'$pciid'\n"
5240 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5241 die "can
't reset PCI device '$pciid'\n"
5242 if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5247 PVE::Storage::activate_volumes($storecfg, $vollist);
5250 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5251 outfunc => sub {}, errfunc => sub {});
5253 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5254 # timeout should be more than enough here...
5255 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5257 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5258 : $defaults->{cpuunits};
5260 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5262 timeout => $statefile ? undef : $start_timeout,
5267 # when migrating, prefix QEMU output so other side can pick up any
5268 # errors that might occur and show the user
5269 if ($migratedfrom) {
5270 $run_params{quiet} = 1;
5271 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5275 Slice => 'qemu
.slice
',
5279 if (PVE::CGroup::cgroup_mode() == 2) {
5280 $properties{CPUWeight} = $cpuunits;
5282 $properties{CPUShares} = $cpuunits;
5285 if (my $cpulimit = $conf->{cpulimit}) {
5286 $properties{CPUQuota} = int($cpulimit * 100);
5288 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5290 my $run_qemu = sub {
5291 PVE::Tools::run_fork sub {
5292 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5294 my $exitcode = run_command($cmd, %run_params);
5295 die "QEMU exited with code $exitcode\n" if $exitcode;
5299 if ($conf->{hugepages}) {
5302 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5303 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5305 PVE::QemuServer::Memory::hugepages_mount();
5306 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5308 eval { $run_qemu->() };
5310 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5311 if !$conf->{keephugepages};
5315 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5316 if !$conf->{keephugepages};
5318 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5321 eval { $run_qemu->() };
5325 # deactivate volumes if start fails
5326 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5327 die "start failed: $err";
5330 print "migration listens on $migrate_uri\n" if $migrate_uri;
5331 $res->{migrate_uri} = $migrate_uri;
5333 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5334 eval { mon_cmd($vmid, "cont"); };
5338 #start nbd server for storage migration
5339 if (my $nbd = $migrate_opts->{nbd}) {
5340 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5342 my $migrate_storage_uri;
5343 # nbd_protocol_version > 0 for unix socket support
5344 if ($nbd_protocol_version > 0 && $migration_type eq 'secure
') {
5345 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5346 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix
', data => { path => $socket_path } } );
5347 $migrate_storage_uri = "nbd:unix:$socket_path";
5349 my $nodename = nodename();
5350 my $localip = $get_migration_ip->($nodename);
5351 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5352 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5354 mon_cmd($vmid, "nbd-server-start", addr => {
5357 host => "${localip}",
5358 port => "${storage_migrate_port}",
5361 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5362 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5365 $res->{migrate_storage_uri} = $migrate_storage_uri;
5367 foreach my $opt (sort keys %$nbd) {
5368 my $drivestr = $nbd->{$opt}->{drivestr};
5369 my $volid = $nbd->{$opt}->{volid};
5370 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5371 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5372 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5373 print "re-using replicated volume: $opt - $volid\n"
5374 if $nbd->{$opt}->{replicated};
5376 $res->{drives}->{$opt} = $nbd->{$opt};
5377 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5381 if ($migratedfrom) {
5383 set_migration_caps($vmid);
5388 print "spice listens on port $spice_port\n";
5389 $res->{spice_port} = $spice_port;
5390 if ($migrate_opts->{spice_ticket}) {
5391 mon_cmd($vmid, "set_password", protocol => 'spice
', password =>
5392 $migrate_opts->{spice_ticket});
5393 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5398 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5399 if !$statefile && $conf->{balloon};
5401 foreach my $opt (keys %$conf) {
5402 next if $opt !~ m/^net\d+$/;
5403 my $nicconf = parse_net($conf->{$opt});
5404 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5408 mon_cmd($vmid, 'qom-set
',
5409 path => "machine/peripheral/balloon0",
5410 property => "guest-stats-polling-interval",
5411 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5414 print "Resumed VM, removing state\n";
5415 if (my $vmstate = $conf->{vmstate}) {
5416 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5417 PVE::Storage::vdisk_free($storecfg, $vmstate);
5419 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5420 PVE
::QemuConfig-
>write_config($vmid, $conf);
5423 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5428 sub vm_commandline
{
5429 my ($storecfg, $vmid, $snapname) = @_;
5431 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5436 my $snapshot = $conf->{snapshots
}->{$snapname};
5437 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5439 # check for machine or CPU overrides in snapshot
5440 $forcemachine = $snapshot->{runningmachine
};
5441 $forcecpu = $snapshot->{runningcpu
};
5443 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5448 my $defaults = load_defaults
();
5450 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults,
5451 $forcemachine, $forcecpu);
5453 return PVE
::Tools
::cmd2string
($cmd);
5457 my ($vmid, $skiplock) = @_;
5459 PVE
::QemuConfig-
>lock_config($vmid, sub {
5461 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5463 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5465 mon_cmd
($vmid, "system_reset");
5469 sub get_vm_volumes
{
5473 foreach_volid
($conf, sub {
5474 my ($volid, $attr) = @_;
5476 return if $volid =~ m
|^/|;
5478 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5481 push @$vollist, $volid;
5487 sub vm_stop_cleanup
{
5488 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5493 my $vollist = get_vm_volumes
($conf);
5494 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5497 foreach my $ext (qw(mon qmp pid vnc qga)) {
5498 unlink "/var/run/qemu-server/${vmid}.$ext";
5501 if ($conf->{ivshmem
}) {
5502 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5503 # just delete it for now, VMs which have this already open do not
5504 # are affected, but new VMs will get a separated one. If this
5505 # becomes an issue we either add some sort of ref-counting or just
5506 # add a "don't delete on stop" flag to the ivshmem format.
5507 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5510 foreach my $key (keys %$conf) {
5511 next if $key !~ m/^hostpci(\d+)$/;
5512 my $hostpciindex = $1;
5513 my $d = parse_hostpci
($conf->{$key});
5514 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5516 foreach my $pci (@{$d->{pciid
}}) {
5517 my $pciid = $pci->{id
};
5518 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5522 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5524 warn $@ if $@; # avoid errors - just warn
5527 # call only in locked context
5529 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5531 my $pid = check_running
($vmid, $nocheck);
5536 $conf = PVE
::QemuConfig-
>load_config($vmid);
5537 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5538 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5539 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5540 $timeout = $opts->{down
} if $opts->{down
};
5542 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5547 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
5548 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5550 mon_cmd
($vmid, "system_powerdown");
5553 mon_cmd
($vmid, "quit");
5559 $timeout = 60 if !defined($timeout);
5562 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5567 if ($count >= $timeout) {
5569 warn "VM still running - terminating now with SIGTERM\n";
5572 die "VM quit/powerdown failed - got timeout\n";
5575 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5579 if (!check_running
($vmid, $nocheck)) {
5580 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5584 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5587 die "VM quit/powerdown failed\n";
5595 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5600 if ($count >= $timeout) {
5601 warn "VM still running - terminating now with SIGKILL\n";
5606 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5609 # Note: use $nocheck to skip tests if VM configuration file exists.
5610 # We need that when migration VMs to other nodes (files already moved)
5611 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5613 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5615 $force = 1 if !defined($force) && !$shutdown;
5618 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5619 kill 15, $pid if $pid;
5620 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5621 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5625 PVE
::QemuConfig-
>lock_config($vmid, sub {
5626 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5631 my ($vmid, $timeout) = @_;
5633 PVE
::QemuConfig-
>lock_config($vmid, sub {
5636 # only reboot if running, as qmeventd starts it again on a stop event
5637 return if !check_running
($vmid);
5639 create_reboot_request
($vmid);
5641 my $storecfg = PVE
::Storage
::config
();
5642 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5646 # avoid that the next normal shutdown will be confused for a reboot
5647 clear_reboot_request
($vmid);
5653 # note: if using the statestorage parameter, the caller has to check privileges
5655 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5662 PVE
::QemuConfig-
>lock_config($vmid, sub {
5664 $conf = PVE
::QemuConfig-
>load_config($vmid);
5666 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5667 PVE
::QemuConfig-
>check_lock($conf)
5668 if !($skiplock || $is_backing_up);
5670 die "cannot suspend to disk during backup\n"
5671 if $is_backing_up && $includestate;
5673 if ($includestate) {
5674 $conf->{lock} = 'suspending';
5675 my $date = strftime
("%Y-%m-%d", localtime(time()));
5676 $storecfg = PVE
::Storage
::config
();
5677 if (!$statestorage) {
5678 $statestorage = find_vmstate_storage
($conf, $storecfg);
5679 # check permissions for the storage
5680 my $rpcenv = PVE
::RPCEnvironment
::get
();
5681 if ($rpcenv->{type
} ne 'cli') {
5682 my $authuser = $rpcenv->get_user();
5683 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5688 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
5689 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5690 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5691 PVE
::QemuConfig-
>write_config($vmid, $conf);
5693 mon_cmd
($vmid, "stop");
5697 if ($includestate) {
5699 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5702 set_migration_caps
($vmid, 1);
5703 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5705 my $state = mon_cmd
($vmid, "query-savevm");
5706 if (!$state->{status
}) {
5707 die "savevm not active\n";
5708 } elsif ($state->{status
} eq 'active') {
5711 } elsif ($state->{status
} eq 'completed') {
5712 print "State saved, quitting\n";
5714 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5715 die "query-savevm failed with error '$state->{error}'\n"
5717 die "query-savevm returned status '$state->{status}'\n";
5723 PVE
::QemuConfig-
>lock_config($vmid, sub {
5724 $conf = PVE
::QemuConfig-
>load_config($vmid);
5726 # cleanup, but leave suspending lock, to indicate something went wrong
5728 mon_cmd
($vmid, "savevm-end");
5729 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5730 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5731 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5732 PVE
::QemuConfig-
>write_config($vmid, $conf);
5738 die "lock changed unexpectedly\n"
5739 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5741 mon_cmd
($vmid, "quit");
5742 $conf->{lock} = 'suspended';
5743 PVE
::QemuConfig-
>write_config($vmid, $conf);
5749 my ($vmid, $skiplock, $nocheck) = @_;
5751 PVE
::QemuConfig-
>lock_config($vmid, sub {
5752 my $res = mon_cmd
($vmid, 'query-status');
5753 my $resume_cmd = 'cont';
5756 if ($res->{status
}) {
5757 return if $res->{status
} eq 'running'; # job done, go home
5758 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
5759 $reset = 1 if $res->{status
} eq 'shutdown';
5764 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5766 PVE
::QemuConfig-
>check_lock($conf)
5767 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5771 # required if a VM shuts down during a backup and we get a resume
5772 # request before the backup finishes for example
5773 mon_cmd
($vmid, "system_reset");
5775 mon_cmd
($vmid, $resume_cmd);
5780 my ($vmid, $skiplock, $key) = @_;
5782 PVE
::QemuConfig-
>lock_config($vmid, sub {
5784 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5786 # there is no qmp command, so we use the human monitor command
5787 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5788 die $res if $res ne '';
5792 # vzdump restore implementaion
5794 sub tar_archive_read_firstfile
{
5795 my $archive = shift;
5797 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5799 # try to detect archive type first
5800 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5801 die "unable to open file '$archive'\n";
5802 my $firstfile = <$fh>;
5806 die "ERROR: archive contaions no data\n" if !$firstfile;
5812 sub tar_restore_cleanup
{
5813 my ($storecfg, $statfile) = @_;
5815 print STDERR
"starting cleanup\n";
5817 if (my $fd = IO
::File-
>new($statfile, "r")) {
5818 while (defined(my $line = <$fd>)) {
5819 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5822 if ($volid =~ m
|^/|) {
5823 unlink $volid || die 'unlink failed\n';
5825 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5827 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5829 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5831 print STDERR
"unable to parse line in statfile - $line";
5838 sub restore_file_archive
{
5839 my ($archive, $vmid, $user, $opts) = @_;
5841 return restore_vma_archive
($archive, $vmid, $user, $opts)
5844 my $info = PVE
::Storage
::archive_info
($archive);
5845 my $format = $opts->{format
} // $info->{format
};
5846 my $comp = $info->{compression
};
5848 # try to detect archive format
5849 if ($format eq 'tar') {
5850 return restore_tar_archive
($archive, $vmid, $user, $opts);
5852 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5856 # hepler to remove disks that will not be used after restore
5857 my $restore_cleanup_oldconf = sub {
5858 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5860 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
5861 my ($ds, $drive) = @_;
5863 return if drive_is_cdrom
($drive, 1);
5865 my $volid = $drive->{file
};
5866 return if !$volid || $volid =~ m
|^/|;
5868 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
5869 return if !$path || !$owner || ($owner != $vmid);
5871 # Note: only delete disk we want to restore
5872 # other volumes will become unused
5873 if ($virtdev_hash->{$ds}) {
5874 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
5881 # delete vmstate files, after the restore we have no snapshots anymore
5882 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5883 my $snap = $oldconf->{snapshots
}->{$snapname};
5884 if ($snap->{vmstate
}) {
5885 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
5893 # Helper to parse vzdump backup device hints
5895 # $rpcenv: Environment, used to ckeck storage permissions
5896 # $user: User ID, to check storage permissions
5897 # $storecfg: Storage configuration
5898 # $fh: the file handle for reading the configuration
5899 # $devinfo: should contain device sizes for all backu-up'ed devices
5900 # $options: backup options (pool, default storage)
5902 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5903 my $parse_backup_hints = sub {
5904 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5906 my $virtdev_hash = {};
5908 while (defined(my $line = <$fh>)) {
5909 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5910 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5911 die "archive does not contain data for drive '$virtdev'\n"
5912 if !$devinfo->{$devname};
5914 if (defined($options->{storage
})) {
5915 $storeid = $options->{storage
} || 'local';
5916 } elsif (!$storeid) {
5919 $format = 'raw' if !$format;
5920 $devinfo->{$devname}->{devname
} = $devname;
5921 $devinfo->{$devname}->{virtdev
} = $virtdev;
5922 $devinfo->{$devname}->{format
} = $format;
5923 $devinfo->{$devname}->{storeid
} = $storeid;
5925 # check permission on storage
5926 my $pool = $options->{pool
}; # todo: do we need that?
5927 if ($user ne 'root@pam') {
5928 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5931 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5932 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5934 my $drive = parse_drive
($virtdev, $2);
5935 if (drive_is_cloudinit
($drive)) {
5936 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
5937 $storeid = $options->{storage
} if defined ($options->{storage
});
5938 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5939 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
5941 $virtdev_hash->{$virtdev} = {
5943 storeid
=> $storeid,
5944 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
5951 return $virtdev_hash;
5954 # Helper to allocate and activate all volumes required for a restore
5956 # $storecfg: Storage configuration
5957 # $virtdev_hash: as returned by parse_backup_hints()
5959 # Returns: { $virtdev => $volid }
5960 my $restore_allocate_devices = sub {
5961 my ($storecfg, $virtdev_hash, $vmid) = @_;
5964 foreach my $virtdev (sort keys %$virtdev_hash) {
5965 my $d = $virtdev_hash->{$virtdev};
5966 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5967 my $storeid = $d->{storeid
};
5968 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5970 # test if requested format is supported
5971 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5972 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
5973 $d->{format
} = $defFormat if !$supported;
5976 if ($d->{is_cloudinit
}) {
5977 $name = "vm-$vmid-cloudinit";
5978 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5979 if ($scfg->{path
}) {
5980 $name .= ".$d->{format}";
5984 my $volid = PVE
::Storage
::vdisk_alloc
(
5985 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
5987 print STDERR
"new volume ID is '$volid'\n";
5988 $d->{volid
} = $volid;
5990 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
5992 $map->{$virtdev} = $volid;
5998 sub restore_update_config_line
{
5999 my ($cookie, $map, $line, $unique) = @_;
6001 return '' if $line =~ m/^\#qmdump\#/;
6002 return '' if $line =~ m/^\#vzdump\#/;
6003 return '' if $line =~ m/^lock:/;
6004 return '' if $line =~ m/^unused\d+:/;
6005 return '' if $line =~ m/^parent:/;
6009 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6010 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6011 # try to convert old 1.X settings
6012 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6013 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6014 my ($model, $macaddr) = split(/\=/, $devconfig);
6015 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6018 bridge
=> "vmbr$ind",
6019 macaddr
=> $macaddr,
6021 my $netstr = print_net
($net);
6023 $res .= "net$cookie->{netcount}: $netstr\n";
6024 $cookie->{netcount
}++;
6026 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6027 my ($id, $netstr) = ($1, $2);
6028 my $net = parse_net
($netstr);
6029 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6030 $netstr = print_net
($net);
6031 $res .= "$id: $netstr\n";
6032 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6035 my $di = parse_drive
($virtdev, $value);
6036 if (defined($di->{backup
}) && !$di->{backup
}) {
6038 } elsif ($map->{$virtdev}) {
6039 delete $di->{format
}; # format can change on restore
6040 $di->{file
} = $map->{$virtdev};
6041 $value = print_drive
($di);
6042 $res .= "$virtdev: $value\n";
6046 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6048 if ($vmgenid ne '0') {
6049 # always generate a new vmgenid if there was a valid one setup
6050 $vmgenid = generate_uuid
();
6052 $res .= "vmgenid: $vmgenid\n";
6053 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6054 my ($uuid, $uuid_str);
6055 UUID
::generate
($uuid);
6056 UUID
::unparse
($uuid, $uuid_str);
6057 my $smbios1 = parse_smbios1
($2);
6058 $smbios1->{uuid
} = $uuid_str;
6059 $res .= $1.print_smbios1
($smbios1)."\n";
6067 my $restore_deactivate_volumes = sub {
6068 my ($storecfg, $devinfo) = @_;
6071 foreach my $devname (keys %$devinfo) {
6072 my $volid = $devinfo->{$devname}->{volid
};
6073 push @$vollist, $volid if $volid;
6076 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6079 my $restore_destroy_volumes = sub {
6080 my ($storecfg, $devinfo) = @_;
6082 foreach my $devname (keys %$devinfo) {
6083 my $volid = $devinfo->{$devname}->{volid
};
6086 if ($volid =~ m
|^/|) {
6087 unlink $volid || die 'unlink failed\n';
6089 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6091 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6093 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6098 my ($cfg, $vmid) = @_;
6100 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6102 my $volid_hash = {};
6103 foreach my $storeid (keys %$info) {
6104 foreach my $item (@{$info->{$storeid}}) {
6105 next if !($item->{volid
} && $item->{size
});
6106 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6107 $volid_hash->{$item->{volid
}} = $item;
6114 sub update_disk_config
{
6115 my ($vmid, $conf, $volid_hash) = @_;
6118 my $prefix = "VM $vmid";
6120 # used and unused disks
6121 my $referenced = {};
6123 # Note: it is allowed to define multiple storages with same path (alias), so
6124 # we need to check both 'volid' and real 'path' (two different volid can point
6125 # to the same path).
6127 my $referencedpath = {};
6130 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6131 my ($opt, $drive) = @_;
6133 my $volid = $drive->{file
};
6135 my $volume = $volid_hash->{$volid};
6137 # mark volid as "in-use" for next step
6138 $referenced->{$volid} = 1;
6139 if ($volume && (my $path = $volume->{path
})) {
6140 $referencedpath->{$path} = 1;
6143 return if drive_is_cdrom
($drive);
6146 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6147 if (defined($updated)) {
6149 $conf->{$opt} = print_drive
($updated);
6150 print "$prefix ($opt): $msg\n";
6154 # remove 'unusedX' entry if volume is used
6155 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6156 my ($opt, $drive) = @_;
6158 my $volid = $drive->{file
};
6162 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6163 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6164 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6166 delete $conf->{$opt};
6169 $referenced->{$volid} = 1;
6170 $referencedpath->{$path} = 1 if $path;
6173 foreach my $volid (sort keys %$volid_hash) {
6174 next if $volid =~ m/vm-$vmid-state-/;
6175 next if $referenced->{$volid};
6176 my $path = $volid_hash->{$volid}->{path
};
6177 next if !$path; # just to be sure
6178 next if $referencedpath->{$path};
6180 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6181 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6182 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6189 my ($vmid, $nolock, $dryrun) = @_;
6191 my $cfg = PVE
::Storage
::config
();
6193 print "rescan volumes...\n";
6194 my $volid_hash = scan_volids
($cfg, $vmid);
6196 my $updatefn = sub {
6199 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6201 PVE
::QemuConfig-
>check_lock($conf);
6204 foreach my $volid (keys %$volid_hash) {
6205 my $info = $volid_hash->{$volid};
6206 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6209 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6211 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6214 if (defined($vmid)) {
6218 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6221 my $vmlist = config_list
();
6222 foreach my $vmid (keys %$vmlist) {
6226 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6232 sub restore_proxmox_backup_archive
{
6233 my ($archive, $vmid, $user, $options) = @_;
6235 my $storecfg = PVE
::Storage
::config
();
6237 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6238 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6240 my $fingerprint = $scfg->{fingerprint
};
6241 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6243 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6245 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6246 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6247 local $ENV{PBS_PASSWORD
} = $password;
6248 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6250 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6251 PVE
::Storage
::parse_volname
($storecfg, $archive);
6253 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6255 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6257 my $tmpdir = "/var/tmp/vzdumptmp$$";
6261 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6262 # disable interrupts (always do cleanups)
6266 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6268 # Note: $oldconf is undef if VM does not exists
6269 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6270 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6271 my $new_conf_raw = '';
6273 my $rpcenv = PVE
::RPCEnvironment
::get
();
6282 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6284 my $cfgfn = "$tmpdir/qemu-server.conf";
6285 my $firewall_config_fn = "$tmpdir/fw.conf";
6286 my $index_fn = "$tmpdir/index.json";
6288 my $cmd = "restore";
6290 my $param = [$pbs_backup_name, "index.json", $index_fn];
6291 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6292 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6293 $index = decode_json
($index);
6295 # print Dumper($index);
6296 foreach my $info (@{$index->{files
}}) {
6297 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6299 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6300 $devinfo->{$devname}->{size
} = $1;
6302 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6307 my $is_qemu_server_backup = scalar(
6308 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6310 if (!$is_qemu_server_backup) {
6311 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6313 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6315 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6316 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6318 if ($has_firewall_config) {
6319 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6320 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6322 my $pve_firewall_dir = '/etc/pve/firewall';
6323 mkdir $pve_firewall_dir; # make sure the dir exists
6324 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6327 my $fh = IO
::File-
>new($cfgfn, "r") ||
6328 die "unable to read qemu-server.conf - $!\n";
6330 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6332 # fixme: rate limit?
6334 # create empty/temp config
6335 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6337 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6340 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6342 if (!$options->{live
}) {
6343 foreach my $virtdev (sort keys %$virtdev_hash) {
6344 my $d = $virtdev_hash->{$virtdev};
6345 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6347 my $volid = $d->{volid
};
6349 my $path = PVE
::Storage
::path
($storecfg, $volid);
6351 my $pbs_restore_cmd = [
6352 '/usr/bin/pbs-restore',
6353 '--repository', $repo,
6355 "$d->{devname}.img.fidx",
6360 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6361 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6363 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6364 push @$pbs_restore_cmd, '--skip-zero';
6367 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6368 print "restore proxmox backup image: $dbg_cmdstring\n";
6369 run_command
($pbs_restore_cmd);
6373 $fh->seek(0, 0) || die "seek failed - $!\n";
6375 my $cookie = { netcount
=> 0 };
6376 while (defined(my $line = <$fh>)) {
6377 $new_conf_raw .= restore_update_config_line
(
6389 if ($err || !$options->{live
}) {
6390 $restore_deactivate_volumes->($storecfg, $devinfo);
6396 $restore_destroy_volumes->($storecfg, $devinfo);
6400 if ($options->{live
}) {
6401 # keep lock during live-restore
6402 $new_conf_raw .= "\nlock: create";
6405 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6407 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6409 eval { rescan
($vmid, 1); };
6412 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
6414 if ($options->{live
}) {
6420 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
6422 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6423 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
6425 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6427 PVE
::QemuConfig-
>remove_lock($vmid, "create");
6431 sub pbs_live_restore
{
6432 my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6434 print "starting VM for live-restore\n";
6435 print "repository: '$repo', snapshot: '$snap'\n";
6437 my $pbs_backing = {};
6438 for my $ds (keys %$restored_disks) {
6439 $ds =~ m/^drive-(.*)$/;
6441 $pbs_backing->{$confname} = {
6442 repository
=> $repo,
6444 archive
=> "$ds.img.fidx",
6446 $pbs_backing->{$confname}->{keyfile
} = $keyfile if -e
$keyfile;
6448 my $drive = parse_drive
($confname, $conf->{$confname});
6449 print "restoring '$ds' to '$drive->{file}'\n";
6452 my $drives_streamed = 0;
6454 # make sure HA doesn't interrupt our restore by stopping the VM
6455 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
6456 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
6459 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6460 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6461 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'pbs-backing' => $pbs_backing}, {});
6463 my $qmeventd_fd = register_qmeventd_handle
($vmid);
6465 # begin streaming, i.e. data copy from PBS to target disk for every vol,
6466 # this will effectively collapse the backing image chain consisting of
6467 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6468 # removes itself once all backing images vanish with 'auto-remove=on')
6470 for my $ds (sort keys %$restored_disks) {
6471 my $job_id = "restore-$ds";
6472 mon_cmd
($vmid, 'block-stream',
6473 'job-id' => $job_id,
6476 $jobs->{$job_id} = {};
6479 mon_cmd
($vmid, 'cont');
6480 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
6482 print "restore-drive jobs finished successfully, removing all tracking block devices"
6483 ." to disconnect from Proxmox Backup Server\n";
6485 for my $ds (sort keys %$restored_disks) {
6486 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6489 close($qmeventd_fd);
6495 warn "An error occured during live-restore: $err\n";
6496 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
6497 die "live-restore failed\n";
6501 sub restore_vma_archive
{
6502 my ($archive, $vmid, $user, $opts, $comp) = @_;
6504 my $readfrom = $archive;
6506 my $cfg = PVE
::Storage
::config
();
6508 my $bwlimit = $opts->{bwlimit
};
6510 my $dbg_cmdstring = '';
6511 my $add_pipe = sub {
6513 push @$commands, $cmd;
6514 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6515 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6520 if ($archive eq '-') {
6523 # If we use a backup from a PVE defined storage we also consider that
6524 # storage's rate limit:
6525 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6526 if (defined($volid)) {
6527 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6528 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6530 print STDERR
"applying read rate limit: $readlimit\n";
6531 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6532 $add_pipe->($cstream);
6538 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6539 my $cmd = $info->{decompressor
};
6540 push @$cmd, $readfrom;
6544 my $tmpdir = "/var/tmp/vzdumptmp$$";
6547 # disable interrupts (always do cleanups)
6551 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6553 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6554 POSIX
::mkfifo
($mapfifo, 0600);
6556 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6558 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6565 my $rpcenv = PVE
::RPCEnvironment
::get
();
6567 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6569 # Note: $oldconf is undef if VM does not exist
6570 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6571 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6572 my $new_conf_raw = '';
6576 my $print_devmap = sub {
6577 my $cfgfn = "$tmpdir/qemu-server.conf";
6579 # we can read the config - that is already extracted
6580 my $fh = IO
::File-
>new($cfgfn, "r") ||
6581 die "unable to read qemu-server.conf - $!\n";
6583 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6585 my $pve_firewall_dir = '/etc/pve/firewall';
6586 mkdir $pve_firewall_dir; # make sure the dir exists
6587 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6590 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6592 foreach my $info (values %{$virtdev_hash}) {
6593 my $storeid = $info->{storeid
};
6594 next if defined($storage_limits{$storeid});
6596 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
6597 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6598 $storage_limits{$storeid} = $limit * 1024;
6601 foreach my $devname (keys %$devinfo) {
6602 die "found no device mapping information for device '$devname'\n"
6603 if !$devinfo->{$devname}->{virtdev
};
6606 # create empty/temp config
6608 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6609 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6613 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6615 # print restore information to $fifofh
6616 foreach my $virtdev (sort keys %$virtdev_hash) {
6617 my $d = $virtdev_hash->{$virtdev};
6618 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6620 my $storeid = $d->{storeid
};
6621 my $volid = $d->{volid
};
6624 if (my $limit = $storage_limits{$storeid}) {
6625 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6628 my $write_zeros = 1;
6629 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6633 my $path = PVE
::Storage
::path
($cfg, $volid);
6635 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6637 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6640 $fh->seek(0, 0) || die "seek failed - $!\n";
6642 my $cookie = { netcount
=> 0 };
6643 while (defined(my $line = <$fh>)) {
6644 $new_conf_raw .= restore_update_config_line
(
6661 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6662 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6664 $oldtimeout = alarm($timeout);
6671 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6672 my ($dev_id, $size, $devname) = ($1, $2, $3);
6673 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6674 } elsif ($line =~ m/^CTIME: /) {
6675 # we correctly received the vma config, so we can disable
6676 # the timeout now for disk allocation (set to 10 minutes, so
6677 # that we always timeout if something goes wrong)
6680 print $fifofh "done\n";
6681 my $tmp = $oldtimeout || 0;
6682 $oldtimeout = undef;
6689 print "restore vma archive: $dbg_cmdstring\n";
6690 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6694 alarm($oldtimeout) if $oldtimeout;
6696 $restore_deactivate_volumes->($cfg, $devinfo);
6698 close($fifofh) if $fifofh;
6703 $restore_destroy_volumes->($cfg, $devinfo);
6707 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6709 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6711 eval { rescan
($vmid, 1); };
6714 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
6717 sub restore_tar_archive
{
6718 my ($archive, $vmid, $user, $opts) = @_;
6720 if ($archive ne '-') {
6721 my $firstfile = tar_archive_read_firstfile
($archive);
6722 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6723 if $firstfile ne 'qemu-server.conf';
6726 my $storecfg = PVE
::Storage
::config
();
6728 # avoid zombie disks when restoring over an existing VM -> cleanup first
6729 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6730 # skiplock=1 because qmrestore has set the 'create' lock itself already
6731 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6732 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6734 my $tocmd = "/usr/lib/qemu-server/qmextract";
6736 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6737 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6738 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6739 $tocmd .= ' --info' if $opts->{info
};
6741 # tar option "xf" does not autodetect compression when read from STDIN,
6742 # so we pipe to zcat
6743 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6744 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6746 my $tmpdir = "/var/tmp/vzdumptmp$$";
6749 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6750 local $ENV{VZDUMP_VMID
} = $vmid;
6751 local $ENV{VZDUMP_USER
} = $user;
6753 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6754 my $new_conf_raw = '';
6756 # disable interrupts (always do cleanups)
6760 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6768 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6770 if ($archive eq '-') {
6771 print "extracting archive from STDIN\n";
6772 run_command
($cmd, input
=> "<&STDIN");
6774 print "extracting archive '$archive'\n";
6778 return if $opts->{info
};
6782 my $statfile = "$tmpdir/qmrestore.stat";
6783 if (my $fd = IO
::File-
>new($statfile, "r")) {
6784 while (defined (my $line = <$fd>)) {
6785 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6786 $map->{$1} = $2 if $1;
6788 print STDERR
"unable to parse line in statfile - $line\n";
6794 my $confsrc = "$tmpdir/qemu-server.conf";
6796 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6798 my $cookie = { netcount
=> 0 };
6799 while (defined (my $line = <$srcfd>)) {
6800 $new_conf_raw .= restore_update_config_line
(
6811 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6817 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6819 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6821 eval { rescan
($vmid, 1); };
6825 sub foreach_storage_used_by_vm
{
6826 my ($conf, $func) = @_;
6830 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6831 my ($ds, $drive) = @_;
6832 return if drive_is_cdrom
($drive);
6834 my $volid = $drive->{file
};
6836 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6837 $sidhash->{$sid} = $sid if $sid;
6840 foreach my $sid (sort keys %$sidhash) {
6845 my $qemu_snap_storage = {
6848 sub do_snapshots_with_qemu
{
6849 my ($storecfg, $volid) = @_;
6851 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6852 my $scfg = $storecfg->{ids
}->{$storage_name};
6853 die "could not find storage '$storage_name'\n" if !defined($scfg);
6855 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6859 if ($volid =~ m/\.(qcow2|qed)$/){
6866 sub qga_check_running
{
6867 my ($vmid, $nowarn) = @_;
6869 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6871 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6877 sub template_create
{
6878 my ($vmid, $conf, $disk) = @_;
6880 my $storecfg = PVE
::Storage
::config
();
6882 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6883 my ($ds, $drive) = @_;
6885 return if drive_is_cdrom
($drive);
6886 return if $disk && $ds ne $disk;
6888 my $volid = $drive->{file
};
6889 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6891 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6892 $drive->{file
} = $voliddst;
6893 $conf->{$ds} = print_drive
($drive);
6894 PVE
::QemuConfig-
>write_config($vmid, $conf);
6898 sub convert_iscsi_path
{
6901 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6906 my $initiator_name = get_initiator_name
();
6908 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6909 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6912 die "cannot convert iscsi path '$path', unkown format\n";
6915 sub qemu_img_convert
{
6916 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6918 my $storecfg = PVE
::Storage
::config
();
6919 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6920 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6922 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6926 my $src_is_iscsi = 0;
6930 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6931 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6932 $src_format = qemu_img_format
($src_scfg, $src_volname);
6933 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6934 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6935 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6936 } elsif (-f
$src_volid) {
6937 $src_path = $src_volid;
6938 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6943 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6945 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6946 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6947 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6948 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6951 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6952 push @$cmd, '-l', "snapshot.name=$snapname"
6953 if $snapname && $src_format && $src_format eq "qcow2";
6954 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6955 push @$cmd, '-T', $cachemode if defined($cachemode);
6957 if ($src_is_iscsi) {
6958 push @$cmd, '--image-opts';
6959 $src_path = convert_iscsi_path
($src_path);
6960 } elsif ($src_format) {
6961 push @$cmd, '-f', $src_format;
6964 if ($dst_is_iscsi) {
6965 push @$cmd, '--target-image-opts';
6966 $dst_path = convert_iscsi_path
($dst_path);
6968 push @$cmd, '-O', $dst_format;
6971 push @$cmd, $src_path;
6973 if (!$dst_is_iscsi && $is_zero_initialized) {
6974 push @$cmd, "zeroinit:$dst_path";
6976 push @$cmd, $dst_path;
6981 if($line =~ m/\((\S+)\/100\
%\)/){
6983 my $transferred = int($size * $percent / 100);
6984 my $total_h = render_bytes
($size, 1);
6985 my $transferred_h = render_bytes
($transferred, 1);
6987 print "transferred $transferred_h of $total_h ($percent%)\n";
6992 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6994 die "copy failed: $err" if $err;
6997 sub qemu_img_format
{
6998 my ($scfg, $volname) = @_;
7000 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7007 sub qemu_drive_mirror
{
7008 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7010 $jobs = {} if !$jobs;
7014 $jobs->{"drive-$drive"} = {};
7016 if ($dst_volid =~ /^nbd:/) {
7017 $qemu_target = $dst_volid;
7020 my $storecfg = PVE
::Storage
::config
();
7021 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7023 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7025 $format = qemu_img_format
($dst_scfg, $dst_volname);
7027 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7029 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7032 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7033 $opts->{format
} = $format if $format;
7035 if (defined($src_bitmap)) {
7036 $opts->{sync
} = 'incremental';
7037 $opts->{bitmap
} = $src_bitmap;
7038 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7041 if (defined($bwlimit)) {
7042 $opts->{speed
} = $bwlimit * 1024;
7043 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7045 print "drive mirror is starting for drive-$drive\n";
7048 # if a job already runs for this device we get an error, catch it for cleanup
7049 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7051 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7053 die "mirroring error: $err\n";
7056 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7059 # $completion can be either
7060 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7061 # 'cancel': wait until all jobs are ready, block-job-cancel them
7062 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7063 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7064 sub qemu_drive_mirror_monitor
{
7065 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7067 $completion //= 'complete';
7071 my $err_complete = 0;
7073 my $starttime = time ();
7075 die "block job ('$op') timed out\n" if $err_complete > 300;
7077 my $stats = mon_cmd
($vmid, "query-block-jobs");
7080 my $running_jobs = {};
7081 for my $stat (@$stats) {
7082 next if $stat->{type
} ne $op;
7083 $running_jobs->{$stat->{device
}} = $stat;
7086 my $readycounter = 0;
7088 for my $job_id (sort keys %$jobs) {
7089 my $job = $running_jobs->{$job_id};
7091 my $vanished = !defined($job);
7092 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7093 if($complete || ($vanished && $completion eq 'auto')) {
7094 print "$job_id: $op-job finished\n";
7095 delete $jobs->{$job_id};
7099 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7101 my $busy = $job->{busy
};
7102 my $ready = $job->{ready
};
7103 if (my $total = $job->{len
}) {
7104 my $transferred = $job->{offset
} || 0;
7105 my $remaining = $total - $transferred;
7106 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7108 my $duration = $ctime - $starttime;
7109 my $total_h = render_bytes
($total, 1);
7110 my $transferred_h = render_bytes
($transferred, 1);
7112 my $status = sprintf(
7113 "transferred $transferred_h of $total_h ($percent%%) in %s",
7114 render_duration
($duration),
7119 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7121 $status .= ", ready";
7124 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
7125 $jobs->{$job_id}->{ready
} = $ready;
7128 $readycounter++ if $job->{ready
};
7131 last if scalar(keys %$jobs) == 0;
7133 if ($readycounter == scalar(keys %$jobs)) {
7134 print "all '$op' jobs are ready\n";
7136 # do the complete later (or has already been done)
7137 last if $completion eq 'skip' || $completion eq 'auto';
7139 if ($vmiddst && $vmiddst != $vmid) {
7140 my $agent_running = $qga && qga_check_running
($vmid);
7141 if ($agent_running) {
7142 print "freeze filesystem\n";
7143 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
7145 print "suspend vm\n";
7146 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
7149 # if we clone a disk for a new target vm, we don't switch the disk
7150 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
7152 if ($agent_running) {
7153 print "unfreeze filesystem\n";
7154 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
7156 print "resume vm\n";
7157 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
7163 for my $job_id (sort keys %$jobs) {
7164 # try to switch the disk if source and destination are on the same guest
7165 print "$job_id: Completing block job_id...\n";
7168 if ($completion eq 'complete') {
7169 $op = 'block-job-complete';
7170 } elsif ($completion eq 'cancel') {
7171 $op = 'block-job-cancel';
7173 die "invalid completion value: $completion\n";
7175 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
7176 if ($@ =~ m/cannot be completed/) {
7177 print "$job_id: block job cannot be completed, trying again.\n";
7180 print "$job_id: Completed successfully.\n";
7181 $jobs->{$job_id}->{complete
} = 1;
7192 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7193 die "block job ($op) error: $err";
7197 sub qemu_blockjobs_cancel
{
7198 my ($vmid, $jobs) = @_;
7200 foreach my $job (keys %$jobs) {
7201 print "$job: Cancelling block job\n";
7202 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
7203 $jobs->{$job}->{cancel
} = 1;
7207 my $stats = mon_cmd
($vmid, "query-block-jobs");
7209 my $running_jobs = {};
7210 foreach my $stat (@$stats) {
7211 $running_jobs->{$stat->{device
}} = $stat;
7214 foreach my $job (keys %$jobs) {
7216 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
7217 print "$job: Done.\n";
7218 delete $jobs->{$job};
7222 last if scalar(keys %$jobs) == 0;
7229 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7230 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7235 print "create linked clone of drive $drivename ($drive->{file})\n";
7236 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
7237 push @$newvollist, $newvolid;
7240 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
7241 $storeid = $storage if $storage;
7243 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
7245 print "create full clone of drive $drivename ($drive->{file})\n";
7248 if (drive_is_cloudinit
($drive)) {
7249 $name = "vm-$newvmid-cloudinit";
7250 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7251 if ($scfg->{path
}) {
7252 $name .= ".$dst_format";
7255 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
7256 } elsif ($drivename eq 'efidisk0') {
7257 $size = get_efivars_size
($conf);
7259 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
7261 $newvolid = PVE
::Storage
::vdisk_alloc
(
7262 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7264 push @$newvollist, $newvolid;
7266 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
7268 if (drive_is_cloudinit
($drive)) {
7269 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7270 # if this is the case, we have to complete any block-jobs still there from
7271 # previous drive-mirrors
7272 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7273 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
7278 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
7279 if (!$running || $snapname) {
7280 # TODO: handle bwlimits
7281 if ($drivename eq 'efidisk0') {
7282 # the relevant data on the efidisk may be smaller than the source
7283 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7284 # that is given by the OVMF_VARS.fd
7285 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
7286 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
7288 # better for Ceph if block size is not too small, see bug #3324
7291 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7292 "if=$src_path", "of=$dst_path"]);
7294 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
7298 my $kvmver = get_running_qemu_version
($vmid);
7299 if (!min_version
($kvmver, 2, 7)) {
7300 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7301 if $drive->{iothread
};
7304 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7305 $completion, $qga, $bwlimit);
7310 my ($size) = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
7313 $disk->{format
} = undef;
7314 $disk->{file
} = $newvolid;
7315 $disk->{size
} = $size if defined($size);
7320 sub get_running_qemu_version
{
7322 my $res = mon_cmd
($vmid, "query-version");
7323 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7326 sub qemu_use_old_bios_files
{
7327 my ($machine_type) = @_;
7329 return if !$machine_type;
7331 my $use_old_bios_files = undef;
7333 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7335 $use_old_bios_files = 1;
7337 my $version = extract_version
($machine_type, kvm_user_version
());
7338 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7339 # load new efi bios files on migration. So this hack is required to allow
7340 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7341 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7342 $use_old_bios_files = !min_version
($version, 2, 4);
7345 return ($use_old_bios_files, $machine_type);
7348 sub get_efivars_size
{
7350 my $arch = get_vm_arch
($conf);
7351 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7352 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
7353 return -s
$ovmf_vars;
7356 sub update_efidisk_size
{
7359 return if !defined($conf->{efidisk0
});
7361 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
7362 $disk->{size
} = get_efivars_size
($conf);
7363 $conf->{efidisk0
} = print_drive
($disk);
7368 sub create_efidisk
($$$$$) {
7369 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7371 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7372 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7374 my $vars_size_b = -s
$ovmf_vars;
7375 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7376 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7377 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7379 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7380 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7382 return ($volid, $size/1024);
7385 sub vm_iothreads_list
{
7388 my $res = mon_cmd
($vmid, 'query-iothreads');
7391 foreach my $iothread (@$res) {
7392 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7399 my ($conf, $drive) = @_;
7403 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7405 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7411 my $controller = int($drive->{index} / $maxdev);
7412 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
7416 return ($maxdev, $controller, $controller_prefix);
7419 sub windows_version
{
7422 return 0 if !$ostype;
7426 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7428 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7430 } elsif ($ostype =~ m/^win(\d+)$/) {
7437 sub resolve_dst_disk_format
{
7438 my ($storecfg, $storeid, $src_volname, $format) = @_;
7439 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7442 # if no target format is specified, use the source disk format as hint
7444 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7445 $format = qemu_img_format
($scfg, $src_volname);
7451 # test if requested format is supported - else use default
7452 my $supported = grep { $_ eq $format } @$validFormats;
7453 $format = $defFormat if !$supported;
7457 # NOTE: if this logic changes, please update docs & possibly gui logic
7458 sub find_vmstate_storage
{
7459 my ($conf, $storecfg) = @_;
7461 # first, return storage from conf if set
7462 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7464 my ($target, $shared, $local);
7466 foreach_storage_used_by_vm
($conf, sub {
7468 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7469 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7470 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7473 # second, use shared storage where VM has at least one disk
7474 # third, use local storage where VM has at least one disk
7475 # fall back to local storage
7476 $target = $shared // $local // 'local';
7482 my ($uuid, $uuid_str);
7483 UUID
::generate
($uuid);
7484 UUID
::unparse
($uuid, $uuid_str);
7488 sub generate_smbios1_uuid
{
7489 return "uuid=".generate_uuid
();
7495 mon_cmd
($vmid, 'nbd-server-stop');
7498 sub create_reboot_request
{
7500 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7501 or die "failed to create reboot trigger file: $!\n";
7505 sub clear_reboot_request
{
7507 my $path = "/run/qemu-server/$vmid.reboot";
7510 $res = unlink($path);
7511 die "could not remove reboot request for $vmid: $!"
7512 if !$res && $! != POSIX
::ENOENT
;
7517 sub bootorder_from_legacy
{
7518 my ($conf, $bootcfg) = @_;
7520 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
7521 my $bootindex_hash = {};
7523 foreach my $o (split(//, $boot)) {
7524 $bootindex_hash->{$o} = $i*100;
7530 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7531 my ($ds, $drive) = @_;
7533 if (drive_is_cdrom
($drive, 1)) {
7534 if ($bootindex_hash->{d
}) {
7535 $bootorder->{$ds} = $bootindex_hash->{d
};
7536 $bootindex_hash->{d
} += 1;
7538 } elsif ($bootindex_hash->{c
}) {
7539 $bootorder->{$ds} = $bootindex_hash->{c
}
7540 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
7541 $bootindex_hash->{c
} += 1;
7545 if ($bootindex_hash->{n
}) {
7546 for (my $i = 0; $i < $MAX_NETS; $i++) {
7547 my $netname = "net$i";
7548 next if !$conf->{$netname};
7549 $bootorder->{$netname} = $bootindex_hash->{n
};
7550 $bootindex_hash->{n
} += 1;
7557 # Generate default device list for 'boot: order=' property. Matches legacy
7558 # default boot order, but with explicit device names. This is important, since
7559 # the fallback for when neither 'order' nor the old format is specified relies
7560 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7561 sub get_default_bootdevices
{
7567 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
7568 push @ret, $first if $first;
7571 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
7572 push @ret, $first if $first;
7575 for (my $i = 0; $i < $MAX_NETS; $i++) {
7576 my $netname = "net$i";
7577 next if !$conf->{$netname};
7578 push @ret, $netname;
7585 sub device_bootorder
{
7588 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
7590 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
7593 if (!defined($boot) || $boot->{legacy
}) {
7594 $bootorder = bootorder_from_legacy
($conf, $boot);
7595 } elsif ($boot->{order
}) {
7596 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7597 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
7598 $bootorder->{$dev} = $i++;
7605 sub register_qmeventd_handle
{
7609 my $peer = "/var/run/qmeventd.sock";
7614 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
7616 if ($! != EINTR
&& $! != EAGAIN
) {
7617 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7620 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7621 . "after $count retries\n";
7626 # send handshake to mark VM as backing up
7627 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
7629 # return handle to be closed later when inhibit is no longer required
7633 # bash completion helper
7635 sub complete_backup_archives
{
7636 my ($cmdname, $pname, $cvalue) = @_;
7638 my $cfg = PVE
::Storage
::config
();
7642 if ($cvalue =~ m/^([^:]+):/) {
7646 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7649 foreach my $id (keys %$data) {
7650 foreach my $item (@{$data->{$id}}) {
7651 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7652 push @$res, $item->{volid
} if defined($item->{volid
});
7659 my $complete_vmid_full = sub {
7662 my $idlist = vmstatus
();
7666 foreach my $id (keys %$idlist) {
7667 my $d = $idlist->{$id};
7668 if (defined($running)) {
7669 next if $d->{template
};
7670 next if $running && $d->{status
} ne 'running';
7671 next if !$running && $d->{status
} eq 'running';
7680 return &$complete_vmid_full();
7683 sub complete_vmid_stopped
{
7684 return &$complete_vmid_full(0);
7687 sub complete_vmid_running
{
7688 return &$complete_vmid_full(1);
7691 sub complete_storage
{
7693 my $cfg = PVE
::Storage
::config
();
7694 my $ids = $cfg->{ids
};
7697 foreach my $sid (keys %$ids) {
7698 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7699 next if !$ids->{$sid}->{content
}->{images
};
7706 sub complete_migration_storage
{
7707 my ($cmd, $param, $current_value, $all_args) = @_;
7709 my $targetnode = @$all_args[1];
7711 my $cfg = PVE
::Storage
::config
();
7712 my $ids = $cfg->{ids
};
7715 foreach my $sid (keys %$ids) {
7716 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7717 next if !$ids->{$sid}->{content
}->{images
};
7726 my $qmpstatus = eval {
7727 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
7728 mon_cmd
($vmid, "query-status");
7731 return $qmpstatus && $qmpstatus->{status
} eq "paused";