1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
31 use PVE
::DataCenterConfig
;
32 use PVE
::Exception
qw(raise raise_param_exc);
33 use PVE
::Format
qw(render_duration render_bytes);
34 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
36 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
39 use PVE
::RPCEnvironment
;
43 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
47 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
48 use PVE
::QemuServer
::Cloudinit
;
49 use PVE
::QemuServer
::CGroup
;
50 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
51 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
52 use PVE
::QemuServer
::Machine
;
53 use PVE
::QemuServer
::Memory
;
54 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
55 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
56 use PVE
::QemuServer
::USB
qw(parse_usb_device);
60 require PVE
::Network
::SDN
::Zones
;
64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
67 "$EDK2_FW_BASE/OVMF_CODE.fd",
68 "$EDK2_FW_BASE/OVMF_VARS.fd"
71 "$EDK2_FW_BASE/AAVMF_CODE.fd",
72 "$EDK2_FW_BASE/AAVMF_VARS.fd"
76 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
78 # Note about locking: we use flock on the config file protect
79 # against concurent actions.
80 # Aditionaly, we have a 'lock' setting in the config file. This
81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
82 # allowed when such lock is set. But you can ignore this kind of
83 # lock with the --skiplock flag.
85 cfs_register_file
('/qemu-server/',
89 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
90 description
=> "Some command save/restore state from this location.",
96 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
97 description
=> "Specifies the Qemu machine type.",
99 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
106 my ($map, $source) = @_;
108 return $source if !defined($map);
110 return $map->{entries
}->{$source}
111 if $map->{entries
} && defined($map->{entries
}->{$source});
113 return $map->{default} if $map->{default};
115 # identity (fallback)
119 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
120 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
122 format
=> 'storagepair-list',
126 #no warnings 'redefine';
130 $nodename_cache //= PVE
::INotify
::nodename
();
131 return $nodename_cache;
138 enum
=> [qw(i6300esb ib700)],
139 description
=> "Watchdog type to emulate.",
140 default => 'i6300esb',
145 enum
=> [qw(reset shutdown poweroff pause debug none)],
146 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
150 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
154 description
=> "Enable/disable Qemu GuestAgent.",
159 fstrim_cloned_disks
=> {
160 description
=> "Run fstrim after moving a disk or migrating the VM.",
166 description
=> "Select the agent type",
170 enum
=> [qw(virtio isa)],
176 description
=> "Select the VGA type.",
181 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
184 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
196 description
=> "The size of the file in MB.",
200 pattern
=> '[a-zA-Z0-9\-]+',
202 format_description
=> 'string',
203 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
210 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
211 description
=> "Configure an audio device."
215 enum
=> ['spice', 'none'],
218 description
=> "Driver backend for the audio device."
222 my $spice_enhancements_fmt = {
227 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
231 enum
=> ['off', 'all', 'filter'],
234 description
=> "Enable video streaming. Uses compression for detected video streams."
241 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
243 description
=> "The file on the host to gather entropy from. In most"
244 . " cases /dev/urandom should be preferred over /dev/random"
245 . " to avoid entropy-starvation issues on the host. Using"
246 . " urandom does *not* decrease security in any meaningful"
247 . " way, as it's still seeded from real entropy, and the"
248 . " bytes provided will most likely be mixed with real"
249 . " entropy on the guest as well. /dev/hwrng can be used"
250 . " to pass through a hardware RNG from the host.",
254 description
=> "Maximum bytes of entropy injected into the guest every"
255 . " 'period' milliseconds. Prefer a lower value when using"
256 . " /dev/random as source. Use 0 to disable limiting"
257 . " (potentially dangerous!).",
260 # default is 1 KiB/s, provides enough entropy to the guest to avoid
261 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
262 # of overwhelming the host, provided we're reading from /dev/urandom
267 description
=> "Every 'period' milliseconds the entropy-injection quota"
268 . " is reset, allowing the guest to retrieve another"
269 . " 'max_bytes' of entropy.",
279 description
=> "Specifies whether a VM will be started during system bootup.",
285 description
=> "Automatic restart after crash (currently ignored).",
290 type
=> 'string', format
=> 'pve-hotplug-features',
291 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
292 default => 'network,disk,usb',
297 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
303 description
=> "Lock/unlock the VM.",
304 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
309 description
=> "Limit of CPU usage.",
310 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
318 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
319 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
320 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
321 ." weights of all the other running VMs.",
324 default => 'cgroup v1: 1024, cgroup v2: 100',
329 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
336 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
342 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
350 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
351 "It should not be necessary to set it.",
352 enum
=> PVE
::Tools
::kvmkeymaplist
(),
357 type
=> 'string', format
=> 'dns-name',
358 description
=> "Set a name for the VM. Only used on the configuration web interface.",
363 description
=> "SCSI controller model",
364 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
370 description
=> "Description for the VM. Shown in the web-interface VM's summary."
371 ." This is saved as comment inside the configuration file.",
372 maxLength
=> 1024 * 8,
377 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
378 description
=> "Specify guest operating system.",
379 verbose_description
=> <<EODESC,
380 Specify guest operating system. This is used to enable special
381 optimization/features for specific operating systems:
384 other;; unspecified OS
385 wxp;; Microsoft Windows XP
386 w2k;; Microsoft Windows 2000
387 w2k3;; Microsoft Windows 2003
388 w2k8;; Microsoft Windows 2008
389 wvista;; Microsoft Windows Vista
390 win7;; Microsoft Windows 7
391 win8;; Microsoft Windows 8/2012/2012r2
392 win10;; Microsoft Windows 10/2016/2019
393 l24;; Linux 2.4 Kernel
394 l26;; Linux 2.6 - 5.X Kernel
395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
400 type
=> 'string', format
=> 'pve-qm-boot',
401 description
=> "Specify guest boot order. Use with 'order=', usage with"
402 . " no key or 'legacy=' is deprecated.",
406 type
=> 'string', format
=> 'pve-qm-bootdisk',
407 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
408 pattern
=> '(ide|sata|scsi|virtio)\d+',
413 description
=> "The number of CPUs. Please use option -sockets instead.",
420 description
=> "The number of CPU sockets.",
427 description
=> "The number of cores per socket.",
434 description
=> "Enable/disable NUMA.",
440 description
=> "Enable/disable hugepages memory.",
441 enum
=> [qw(any 2 1024)],
447 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
448 ." after VM shutdown and can be used for subsequent starts.",
453 description
=> "Number of hotplugged vcpus.",
460 description
=> "Enable/disable ACPI.",
465 description
=> "Enable/disable Qemu GuestAgent and its properties.",
467 format
=> $agent_fmt,
472 description
=> "Enable/disable KVM hardware virtualization.",
478 description
=> "Enable/disable time drift fix.",
484 description
=> "Set the real time clock to local time. This is enabled by default if ostype"
485 ." indicates a Microsoft OS.",
490 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
494 type
=> 'string', format
=> $vga_fmt,
495 description
=> "Configure the VGA hardware.",
496 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
497 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
498 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
499 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
500 ." display server. For win* OS you can select how many independent displays you want,"
501 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
502 ." using a serial device as terminal.",
506 type
=> 'string', format
=> 'pve-qm-watchdog',
507 description
=> "Create a virtual hardware watchdog device.",
508 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
509 ." action), the watchdog must be periodically polled by an agent inside the guest or"
510 ." else the watchdog will reset the guest (or execute the respective action specified)",
515 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
516 description
=> "Set the initial date of the real time clock. Valid format for date are:"
517 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
518 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
521 startup
=> get_standard_option
('pve-startup-order'),
525 description
=> "Enable/disable Template.",
531 description
=> "Arbitrary arguments passed to kvm.",
532 verbose_description
=> <<EODESCR,
533 Arbitrary arguments passed to kvm, for example:
535 args: -no-reboot -no-hpet
537 NOTE: this option is for experts only.
544 description
=> "Enable/disable the USB tablet device.",
545 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
546 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
547 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
548 ." may consider disabling this to save some context switches. This is turned off by"
549 ." default if you use spice (`qm set <vmid> --vga qxl`).",
554 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
558 migrate_downtime
=> {
561 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
567 type
=> 'string', format
=> 'pve-qm-ide',
568 typetext
=> '<volume>',
569 description
=> "This is an alias for option -ide2",
573 description
=> "Emulated CPU type.",
575 format
=> 'pve-vm-cpu-conf',
577 parent
=> get_standard_option
('pve-snapshot-name', {
579 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
583 description
=> "Timestamp for snapshots.",
589 type
=> 'string', format
=> 'pve-volume-id',
590 description
=> "Reference to a volume which stores the VM state. This is used internally"
593 vmstatestorage
=> get_standard_option
('pve-storage-id', {
594 description
=> "Default storage for VM state volumes/files.",
597 runningmachine
=> get_standard_option
('pve-qemu-machine', {
598 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
602 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
603 ." internally for snapshots.",
606 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
607 format_description
=> 'QEMU -cpu parameter'
609 machine
=> get_standard_option
('pve-qemu-machine'),
611 description
=> "Virtual processor architecture. Defaults to the host.",
614 enum
=> [qw(x86_64 aarch64)],
617 description
=> "Specify SMBIOS type 1 fields.",
618 type
=> 'string', format
=> 'pve-qm-smbios1',
625 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
626 ." remove disk operations.",
632 enum
=> [ qw(seabios ovmf) ],
633 description
=> "Select BIOS implementation.",
634 default => 'seabios',
638 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
639 format_description
=> 'UUID',
640 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
641 ." to disable explicitly.",
642 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
643 ." value identifier to the guest OS. This allows to notify the guest operating system"
644 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
645 ." execution or creation from a template). The guest operating system notices the"
646 ." change, and is then able to react as appropriate by marking its copies of"
647 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
648 ."Note that auto-creation only works when done through API/CLI create or update methods"
649 .", but not when manually editing the config file.",
650 default => "1 (autogenerated)",
655 format
=> 'pve-volume-id',
657 description
=> "Script that will be executed during various steps in the vms lifetime.",
661 format
=> $ivshmem_fmt,
662 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
668 format
=> $audio_fmt,
669 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
672 spice_enhancements
=> {
674 format
=> $spice_enhancements_fmt,
675 description
=> "Configure additional enhancements for SPICE.",
679 type
=> 'string', format
=> 'pve-tag-list',
680 description
=> 'Tags of the VM. This is only meta information.',
686 description
=> "Configure a VirtIO-based Random Number Generator.",
695 description
=> 'Specify a custom file containing all meta data passed to the VM via"
696 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
697 format
=> 'pve-volume-id',
698 format_description
=> 'volume',
703 description
=> 'Specify a custom file containing all network data passed to the VM via'
705 format
=> 'pve-volume-id',
706 format_description
=> 'volume',
711 description
=> 'Specify a custom file containing all user data passed to the VM via'
713 format
=> 'pve-volume-id',
714 format_description
=> 'volume',
717 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
719 my $confdesc_cloudinit = {
723 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
724 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
725 .' and `configdrive2` for windows.',
726 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
731 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
732 ." image's configured default user.",
737 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
738 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
739 .' support hashed passwords.',
744 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
746 format
=> 'pve-qm-cicustom',
751 description
=> "cloud-init: Sets DNS search domains for a container. Create will'
752 .' automatically use the setting from the host if neither searchdomain nor nameserver'
757 type
=> 'string', format
=> 'address-list',
758 description
=> "cloud-init: Sets DNS server IP address for a container. Create will'
759 .' automatically use the setting from the host if neither searchdomain nor nameserver'
765 format
=> 'urlencoded',
766 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
770 # what about other qemu settings ?
772 #machine => 'string',
785 ##soundhw => 'string',
787 while (my ($k, $v) = each %$confdesc) {
788 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
791 my $MAX_USB_DEVICES = 5;
793 my $MAX_SERIAL_PORTS = 4;
794 my $MAX_PARALLEL_PORTS = 3;
800 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
801 description
=> "CPUs accessing this NUMA node.",
802 format_description
=> "id[-id];...",
806 description
=> "Amount of memory this NUMA node provides.",
811 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
812 description
=> "Host NUMA nodes to use.",
813 format_description
=> "id[-id];...",
818 enum
=> [qw(preferred bind interleave)],
819 description
=> "NUMA allocation policy.",
823 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
826 type
=> 'string', format
=> $numa_fmt,
827 description
=> "NUMA topology.",
829 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
831 for (my $i = 0; $i < $MAX_NUMA; $i++) {
832 $confdesc->{"numa$i"} = $numadesc;
835 my $nic_model_list = [
851 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
853 my $net_fmt_bridge_descr = <<__EOD__;
854 Bridge to attach the network device to. The Proxmox VE standard bridge
857 If you do not specify a bridge, we create a kvm user (NATed) network
858 device, which provides DHCP and DNS services. The following addresses
865 The DHCP server assign addresses to the guest starting from 10.0.2.15.
869 macaddr
=> get_standard_option
('mac-addr', {
870 description
=> "MAC address. That address must be unique withing your network. This is"
871 ." automatically generated if not specified.",
875 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
876 ." very low CPU overhead. If your guest does not support this driver, it is usually"
877 ." best to use 'e1000'.",
878 enum
=> $nic_model_list,
881 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
884 description
=> $net_fmt_bridge_descr,
885 format_description
=> 'bridge',
886 pattern
=> '[-_.\w\d]+',
891 minimum
=> 0, maximum
=> 16,
892 description
=> 'Number of packet queues to be used on the device.',
898 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
903 minimum
=> 1, maximum
=> 4094,
904 description
=> 'VLAN tag to apply to packets on this interface.',
909 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
910 description
=> 'VLAN trunks to pass through this interface.',
911 format_description
=> 'vlanid[;vlanid...]',
916 description
=> 'Whether this interface should be protected by the firewall.',
921 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
926 minimum
=> 1, maximum
=> 65520,
927 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
934 type
=> 'string', format
=> $net_fmt,
935 description
=> "Specify network devices.",
938 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
943 format
=> 'pve-ipv4-config',
944 format_description
=> 'IPv4Format/CIDR',
945 description
=> 'IPv4 address in CIDR format.',
952 format_description
=> 'GatewayIPv4',
953 description
=> 'Default gateway for IPv4 traffic.',
959 format
=> 'pve-ipv6-config',
960 format_description
=> 'IPv6Format/CIDR',
961 description
=> 'IPv6 address in CIDR format.',
968 format_description
=> 'GatewayIPv6',
969 description
=> 'Default gateway for IPv6 traffic.',
974 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
977 type
=> 'string', format
=> 'pve-qm-ipconfig',
978 description
=> <<'EODESCR',
979 cloud-init: Specify IP addresses and gateways for the corresponding interface.
981 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
983 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
984 gateway should be provided.
985 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
986 cloud-init 19.4 or newer.
988 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
992 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
994 for (my $i = 0; $i < $MAX_NETS; $i++) {
995 $confdesc->{"net$i"} = $netdesc;
996 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
999 foreach my $key (keys %$confdesc_cloudinit) {
1000 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1003 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1004 sub verify_volume_id_or_qm_path
{
1005 my ($volid, $noerr) = @_;
1007 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
1011 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1012 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1023 type
=> 'string', format
=> 'pve-qm-usb-device',
1024 format_description
=> 'HOSTUSBDEVICE|spice',
1025 description
=> <<EODESCR,
1026 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1028 'bus-port(.port)*' (decimal numbers) or
1029 'vendor_id:product_id' (hexadeciaml numbers) or
1032 You can use the 'lsusb -t' command to list existing usb devices.
1034 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1035 machines - use with special care.
1037 The value 'spice' can be used to add a usb redirection devices for spice.
1043 description
=> "Specifies whether if given host option is a USB3 device or port.",
1050 type
=> 'string', format
=> $usb_fmt,
1051 description
=> "Configure an USB device (n is 0 to 4).",
1053 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1058 pattern
=> '(/dev/.+|socket)',
1059 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1060 verbose_description
=> <<EODESCR,
1061 Create a serial device inside the VM (n is 0 to 3), and pass through a
1062 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1063 host side (use 'qm terminal' to open a terminal connection).
1065 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1066 use with special care.
1068 CAUTION: Experimental! User reported problems with this option.
1075 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1076 description
=> "Map host parallel devices (n is 0 to 2).",
1077 verbose_description
=> <<EODESCR,
1078 Map host parallel devices (n is 0 to 2).
1080 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1081 machines - use with special care.
1083 CAUTION: Experimental! User reported problems with this option.
1087 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1088 $confdesc->{"parallel$i"} = $paralleldesc;
1091 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1092 $confdesc->{"serial$i"} = $serialdesc;
1095 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1096 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1099 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1100 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1103 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1104 $confdesc->{"usb$i"} = $usbdesc;
1112 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1113 . " Deprecated, use 'order=' instead.",
1114 pattern
=> '[acdn]{1,4}',
1115 format_description
=> "[acdn]{1,4}",
1117 # note: this is also the fallback if boot: is not given at all
1123 format
=> 'pve-qm-bootdev-list',
1124 format_description
=> "device[;device...]",
1125 description
=> <<EODESC,
1126 The guest will attempt to boot from devices in the order they appear here.
1128 Disks, optical drives and passed-through storage USB devices will be directly
1129 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1130 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1132 Note that only devices in this list will be marked as bootable and thus loaded
1133 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1134 (e.g. software-raid), you need to specify all of them here.
1136 Overrides the deprecated 'legacy=[acdn]*' value when given.
1140 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1142 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1143 sub verify_bootdev
{
1144 my ($dev, $noerr) = @_;
1146 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && $dev !~ m/^efidisk/;
1150 return 0 if $dev !~ m/^$base\d+$/;
1151 return 0 if !$confdesc->{$dev};
1155 return $dev if $check->("net");
1156 return $dev if $check->("usb");
1157 return $dev if $check->("hostpci");
1160 die "invalid boot device '$dev'\n";
1163 sub print_bootorder
{
1165 return "" if !@$devs;
1166 my $data = { order
=> join(';', @$devs) };
1167 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1170 my $kvm_api_version = 0;
1173 return $kvm_api_version if $kvm_api_version;
1175 open my $fh, '<', '/dev/kvm' or return;
1177 # 0xae00 => KVM_GET_API_VERSION
1178 $kvm_api_version = ioctl($fh, 0xae00, 0);
1181 return $kvm_api_version;
1184 my $kvm_user_version = {};
1187 sub kvm_user_version
{
1190 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1191 my $st = stat($binary);
1193 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1194 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1195 $cachedmtime == $st->mtime;
1197 $kvm_user_version->{$binary} = 'unknown';
1198 $kvm_mtime->{$binary} = $st->mtime;
1202 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1203 $kvm_user_version->{$binary} = $2;
1207 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1210 return $kvm_user_version->{$binary};
1213 my sub extract_version
{
1214 my ($machine_type, $version) = @_;
1215 $version = kvm_user_version
() if !defined($version);
1216 PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1219 sub kernel_has_vhost_net
{
1220 return -c
'/dev/vhost-net';
1225 return defined($confdesc->{$key});
1229 sub get_cdrom_path
{
1231 return $cdrom_path if $cdrom_path;
1233 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1234 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1235 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1239 my ($storecfg, $vmid, $cdrom) = @_;
1241 if ($cdrom eq 'cdrom') {
1242 return get_cdrom_path
();
1243 } elsif ($cdrom eq 'none') {
1245 } elsif ($cdrom =~ m
|^/|) {
1248 return PVE
::Storage
::path
($storecfg, $cdrom);
1252 # try to convert old style file names to volume IDs
1253 sub filename_to_volume_id
{
1254 my ($vmid, $file, $media) = @_;
1256 if (!($file eq 'none' || $file eq 'cdrom' ||
1257 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1259 return if $file =~ m
|/|;
1261 if ($media && $media eq 'cdrom') {
1262 $file = "local:iso/$file";
1264 $file = "local:$vmid/$file";
1271 sub verify_media_type
{
1272 my ($opt, $vtype, $media) = @_;
1277 if ($media eq 'disk') {
1279 } elsif ($media eq 'cdrom') {
1282 die "internal error";
1285 return if ($vtype eq $etype);
1287 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1290 sub cleanup_drive_path
{
1291 my ($opt, $storecfg, $drive) = @_;
1293 # try to convert filesystem paths to volume IDs
1295 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1296 ($drive->{file
} !~ m
|^/dev/.+|) &&
1297 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1298 ($drive->{file
} !~ m/^\d+$/)) {
1299 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1300 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1302 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1303 verify_media_type
($opt, $vtype, $drive->{media
});
1304 $drive->{file
} = $volid;
1307 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1310 sub parse_hotplug_features
{
1315 return $res if $data eq '0';
1317 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1319 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1320 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1323 die "invalid hotplug feature '$feature'\n";
1329 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1330 sub pve_verify_hotplug_features
{
1331 my ($value, $noerr) = @_;
1333 return $value if parse_hotplug_features
($value);
1337 die "unable to parse hotplug option\n";
1341 my($fh, $noerr) = @_;
1344 my $SG_GET_VERSION_NUM = 0x2282;
1346 my $versionbuf = "\x00" x
8;
1347 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1349 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1352 my $version = unpack("I", $versionbuf);
1353 if ($version < 30000) {
1354 die "scsi generic interface too old\n" if !$noerr;
1358 my $buf = "\x00" x
36;
1359 my $sensebuf = "\x00" x
8;
1360 my $cmd = pack("C x3 C x1", 0x12, 36);
1362 # see /usr/include/scsi/sg.h
1363 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1365 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1366 length($sensebuf), 0, length($buf), $buf,
1367 $cmd, $sensebuf, 6000);
1369 $ret = ioctl($fh, $SG_IO, $packet);
1371 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1375 my @res = unpack($sg_io_hdr_t, $packet);
1376 if ($res[17] || $res[18]) {
1377 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1382 (my $byte0, my $byte1, $res->{vendor
},
1383 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1385 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1386 $res->{type
} = $byte0 & 31;
1394 my $fh = IO
::File-
>new("+<$path") || return;
1395 my $res = scsi_inquiry
($fh, 1);
1401 sub print_tabletdevice_full
{
1402 my ($conf, $arch) = @_;
1404 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1406 # we use uhci for old VMs because tablet driver was buggy in older qemu
1408 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1414 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1417 sub print_keyboarddevice_full
{
1418 my ($conf, $arch, $machine) = @_;
1420 return if $arch ne 'aarch64';
1422 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1425 my sub get_drive_id
{
1427 return "$drive->{interface}$drive->{index}";
1430 sub print_drivedevice_full
{
1431 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1436 my $drive_id = get_drive_id
($drive);
1437 if ($drive->{interface
} eq 'virtio') {
1438 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1439 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1440 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1441 } elsif ($drive->{interface
} eq 'scsi') {
1443 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1444 my $unit = $drive->{index} % $maxdev;
1445 my $devicetype = 'hd';
1447 if (drive_is_cdrom
($drive)) {
1450 if ($drive->{file
} =~ m
|^/|) {
1451 $path = $drive->{file
};
1452 if (my $info = path_is_scsi
($path)) {
1453 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1454 $devicetype = 'block';
1455 } elsif ($info->{type
} == 1) { # tape
1456 $devicetype = 'generic';
1460 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1463 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1464 my $version = extract_version
($machine_type, kvm_user_version
());
1465 if ($path =~ m/^iscsi\:\/\
// &&
1466 !min_version
($version, 4, 1)) {
1467 $devicetype = 'generic';
1471 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1472 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1474 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1475 .",lun=$drive->{index}";
1477 $device .= ",drive=drive-$drive_id,id=$drive_id";
1479 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1480 $device .= ",rotation_rate=1";
1482 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1484 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1485 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1486 my $controller = int($drive->{index} / $maxdev);
1487 my $unit = $drive->{index} % $maxdev;
1488 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1490 $device = "ide-$devicetype";
1491 if ($drive->{interface
} eq 'ide') {
1492 $device .= ",bus=ide.$controller,unit=$unit";
1494 $device .= ",bus=ahci$controller.$unit";
1496 $device .= ",drive=drive-$drive_id,id=$drive_id";
1498 if ($devicetype eq 'hd') {
1499 if (my $model = $drive->{model
}) {
1500 $model = URI
::Escape
::uri_unescape
($model);
1501 $device .= ",model=$model";
1503 if ($drive->{ssd
}) {
1504 $device .= ",rotation_rate=1";
1507 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1508 } elsif ($drive->{interface
} eq 'usb') {
1510 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1512 die "unsupported interface type";
1515 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1517 if (my $serial = $drive->{serial
}) {
1518 $serial = URI
::Escape
::uri_unescape
($serial);
1519 $device .= ",serial=$serial";
1526 sub get_initiator_name
{
1529 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1530 while (defined(my $line = <$fh>)) {
1531 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1540 sub print_drive_commandline_full
{
1541 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1544 my $volid = $drive->{file
};
1545 my $format = $drive->{format
};
1546 my $drive_id = get_drive_id
($drive);
1548 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1549 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1551 if (drive_is_cdrom
($drive)) {
1552 $path = get_iso_path
($storecfg, $vmid, $volid);
1553 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1556 $path = PVE
::Storage
::path
($storecfg, $volid);
1557 $format //= qemu_img_format
($scfg, $volname);
1564 my $is_rbd = $path =~ m/^rbd:/;
1567 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1568 foreach my $o (@qemu_drive_options) {
1569 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1572 # snapshot only accepts on|off
1573 if (defined($drive->{snapshot
})) {
1574 my $v = $drive->{snapshot
} ?
'on' : 'off';
1575 $opts .= ",snapshot=$v";
1578 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1579 my ($dir, $qmpname) = @$type;
1580 if (my $v = $drive->{"mbps$dir"}) {
1581 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1583 if (my $v = $drive->{"mbps${dir}_max"}) {
1584 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1586 if (my $v = $drive->{"bps${dir}_max_length"}) {
1587 $opts .= ",throttling.bps$qmpname-max-length=$v";
1589 if (my $v = $drive->{"iops${dir}"}) {
1590 $opts .= ",throttling.iops$qmpname=$v";
1592 if (my $v = $drive->{"iops${dir}_max"}) {
1593 $opts .= ",throttling.iops$qmpname-max=$v";
1595 if (my $v = $drive->{"iops${dir}_max_length"}) {
1596 $opts .= ",throttling.iops$qmpname-max-length=$v";
1601 $format = "rbd" if $is_rbd;
1602 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1604 $opts .= ",format=alloc-track,file.driver=$format";
1606 $opts .= ",format=$format";
1609 my $cache_direct = 0;
1611 if (my $cache = $drive->{cache
}) {
1612 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1613 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1614 $opts .= ",cache=none";
1618 # io_uring with cache mode writeback or writethrough on krbd will hang...
1619 my $rbd_no_io_uring = $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1621 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1622 # sometimes, just plain disable...
1623 my $lvm_no_io_uring = $scfg && $scfg->{type
} eq 'lvm';
1625 if (!$drive->{aio
}) {
1626 if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1627 # io_uring supports all cache modes
1628 $opts .= ",aio=io_uring";
1630 # aio native works only with O_DIRECT
1632 $opts .= ",aio=native";
1634 $opts .= ",aio=threads";
1639 if (!drive_is_cdrom
($drive)) {
1641 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1642 $detectzeroes = 'off';
1643 } elsif ($drive->{discard
}) {
1644 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1646 # This used to be our default with discard not being specified:
1647 $detectzeroes = 'on';
1650 # note: 'detect-zeroes' works per blockdev and we want it to persist
1651 # after the alloc-track is removed, so put it on 'file' directly
1652 my $dz_param = $pbs_name ?
"file.detect-zeroes" : "detect-zeroes";
1653 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1657 $opts .= ",backing=$pbs_name";
1658 $opts .= ",auto-remove=on";
1661 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1662 my $file_param = "file";
1664 # non-rbd drivers require the underlying file to be a seperate block
1665 # node, so add a second .file indirection
1666 $file_param .= ".file" if !$is_rbd;
1667 $file_param .= ".filename";
1669 my $pathinfo = $path ?
"$file_param=$path," : '';
1671 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1674 sub print_pbs_blockdev
{
1675 my ($pbs_conf, $pbs_name) = @_;
1676 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1677 $blockdev .= ",repository=$pbs_conf->{repository}";
1678 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1679 $blockdev .= ",archive=$pbs_conf->{archive}";
1680 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1684 sub print_netdevice_full
{
1685 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1687 my $device = $net->{model
};
1688 if ($net->{model
} eq 'virtio') {
1689 $device = 'virtio-net-pci';
1692 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1693 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1694 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1695 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1696 # and out of each queue plus one config interrupt and control vector queue
1697 my $vectors = $net->{queues
} * 2 + 2;
1698 $tmpstr .= ",vectors=$vectors,mq=on";
1700 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1702 if (my $mtu = $net->{mtu
}) {
1703 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1704 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1707 } elsif ($mtu < 576) {
1708 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1709 } elsif ($mtu > $bridge_mtu) {
1710 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1712 $tmpstr .= ",host_mtu=$mtu";
1714 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1718 if ($use_old_bios_files) {
1720 if ($device eq 'virtio-net-pci') {
1721 $romfile = 'pxe-virtio.rom';
1722 } elsif ($device eq 'e1000') {
1723 $romfile = 'pxe-e1000.rom';
1724 } elsif ($device eq 'e1000e') {
1725 $romfile = 'pxe-e1000e.rom';
1726 } elsif ($device eq 'ne2k') {
1727 $romfile = 'pxe-ne2k_pci.rom';
1728 } elsif ($device eq 'pcnet') {
1729 $romfile = 'pxe-pcnet.rom';
1730 } elsif ($device eq 'rtl8139') {
1731 $romfile = 'pxe-rtl8139.rom';
1733 $tmpstr .= ",romfile=$romfile" if $romfile;
1739 sub print_netdev_full
{
1740 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1743 if ($netid =~ m/^net(\d+)$/) {
1747 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1749 my $ifname = "tap${vmid}i$i";
1751 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1752 die "interface name '$ifname' is too long (max 15 character)\n"
1753 if length($ifname) >= 16;
1755 my $vhostparam = '';
1756 if (is_native
($arch)) {
1757 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1760 my $vmname = $conf->{name
} || "vm$vmid";
1763 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1765 if ($net->{bridge
}) {
1766 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1767 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1769 $netdev = "type=user,id=$netid,hostname=$vmname";
1772 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1778 'cirrus' => 'cirrus-vga',
1780 'vmware' => 'vmware-svga',
1781 'virtio' => 'virtio-vga',
1784 sub print_vga_device
{
1785 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1787 my $type = $vga_map->{$vga->{type
}};
1788 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1789 $type = 'virtio-gpu';
1791 my $vgamem_mb = $vga->{memory
};
1793 my $max_outputs = '';
1795 $type = $id ?
'qxl' : 'qxl-vga';
1797 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1798 # set max outputs so linux can have up to 4 qxl displays with one device
1799 if (min_version
($machine_version, 4, 1)) {
1800 $max_outputs = ",max_outputs=4";
1805 die "no devicetype for $vga->{type}\n" if !$type;
1809 if ($vga->{type
} eq 'virtio') {
1810 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1811 $memory = ",max_hostmem=$bytes";
1813 # from https://www.spice-space.org/multiple-monitors.html
1814 $memory = ",vgamem_mb=$vga->{memory}";
1815 my $ram = $vgamem_mb * 4;
1816 my $vram = $vgamem_mb * 2;
1817 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1819 $memory = ",vgamem_mb=$vga->{memory}";
1821 } elsif ($qxlnum && $id) {
1822 $memory = ",ram_size=67108864,vram_size=33554432";
1826 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1827 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1830 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1831 my $vgaid = "vga" . ($id // '');
1834 if ($q35 && $vgaid eq 'vga') {
1835 # the first display uses pcie.0 bus on q35 machines
1836 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1838 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1841 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1844 sub parse_number_sets
{
1847 foreach my $part (split(/;/, $set)) {
1848 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1849 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1850 push @$res, [ $1, $2 ];
1852 die "invalid range: $part\n";
1861 my $res = parse_property_string
($numa_fmt, $data);
1862 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1863 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1867 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1871 my $res = eval { parse_property_string
($net_fmt, $data) };
1876 if (!defined($res->{macaddr
})) {
1877 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1878 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1883 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1884 sub parse_ipconfig
{
1887 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1893 if ($res->{gw
} && !$res->{ip
}) {
1894 warn 'gateway specified without specifying an IP address';
1897 if ($res->{gw6
} && !$res->{ip6
}) {
1898 warn 'IPv6 gateway specified without specifying an IPv6 address';
1901 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1902 warn 'gateway specified together with DHCP';
1905 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1907 warn "IPv6 gateway specified together with $res->{ip6} address";
1911 if (!$res->{ip
} && !$res->{ip6
}) {
1912 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1921 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1924 sub add_random_macs
{
1925 my ($settings) = @_;
1927 foreach my $opt (keys %$settings) {
1928 next if $opt !~ m/^net(\d+)$/;
1929 my $net = parse_net
($settings->{$opt});
1931 $settings->{$opt} = print_net
($net);
1935 sub vm_is_volid_owner
{
1936 my ($storecfg, $vmid, $volid) = @_;
1938 if ($volid !~ m
|^/|) {
1940 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1941 if ($owner && ($owner == $vmid)) {
1949 sub vmconfig_register_unused_drive
{
1950 my ($storecfg, $vmid, $conf, $drive) = @_;
1952 if (drive_is_cloudinit
($drive)) {
1953 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1955 } elsif (!drive_is_cdrom
($drive)) {
1956 my $volid = $drive->{file
};
1957 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1958 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1963 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1967 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1968 format_description
=> 'UUID',
1969 description
=> "Set SMBIOS1 UUID.",
1974 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1975 format_description
=> 'Base64 encoded string',
1976 description
=> "Set SMBIOS1 version.",
1981 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1982 format_description
=> 'Base64 encoded string',
1983 description
=> "Set SMBIOS1 serial number.",
1988 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1989 format_description
=> 'Base64 encoded string',
1990 description
=> "Set SMBIOS1 manufacturer.",
1995 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1996 format_description
=> 'Base64 encoded string',
1997 description
=> "Set SMBIOS1 product ID.",
2002 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2003 format_description
=> 'Base64 encoded string',
2004 description
=> "Set SMBIOS1 SKU string.",
2009 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2010 format_description
=> 'Base64 encoded string',
2011 description
=> "Set SMBIOS1 family string.",
2016 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2024 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2031 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2034 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2036 sub parse_watchdog
{
2041 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2046 sub parse_guest_agent
{
2049 return {} if !defined($conf->{agent
});
2051 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2054 # if the agent is disabled ignore the other potentially set properties
2055 return {} if !$res->{enabled
};
2060 my ($conf, $key) = @_;
2061 return undef if !defined($conf->{agent
});
2063 my $agent = parse_guest_agent
($conf);
2064 return $agent->{$key};
2070 return {} if !$value;
2071 my $res = eval { parse_property_string
($vga_fmt, $value) };
2081 my $res = eval { parse_property_string
($rng_fmt, $value) };
2086 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2087 sub verify_usb_device
{
2088 my ($value, $noerr) = @_;
2090 return $value if parse_usb_device
($value);
2094 die "unable to parse usb device\n";
2097 # add JSON properties for create and set function
2098 sub json_config_properties
{
2101 foreach my $opt (keys %$confdesc) {
2102 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2103 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2104 $prop->{$opt} = $confdesc->{$opt};
2110 # return copy of $confdesc_cloudinit to generate documentation
2111 sub cloudinit_config_properties
{
2113 return dclone
($confdesc_cloudinit);
2117 my ($key, $value) = @_;
2119 die "unknown setting '$key'\n" if !$confdesc->{$key};
2121 my $type = $confdesc->{$key}->{type
};
2123 if (!defined($value)) {
2124 die "got undefined value\n";
2127 if ($value =~ m/[\n\r]/) {
2128 die "property contains a line feed\n";
2131 if ($type eq 'boolean') {
2132 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2133 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2134 die "type check ('boolean') failed - got '$value'\n";
2135 } elsif ($type eq 'integer') {
2136 return int($1) if $value =~ m/^(\d+)$/;
2137 die "type check ('integer') failed - got '$value'\n";
2138 } elsif ($type eq 'number') {
2139 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2140 die "type check ('number') failed - got '$value'\n";
2141 } elsif ($type eq 'string') {
2142 if (my $fmt = $confdesc->{$key}->{format
}) {
2143 PVE
::JSONSchema
::check_format
($fmt, $value);
2146 $value =~ s/^\"(.*)\"$/$1/;
2149 die "internal error"
2154 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2156 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2158 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2160 if ($conf->{template
}) {
2161 # check if any base image is still used by a linked clone
2162 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2163 my ($ds, $drive) = @_;
2164 return if drive_is_cdrom
($drive);
2166 my $volid = $drive->{file
};
2167 return if !$volid || $volid =~ m
|^/|;
2169 die "base volume '$volid' is still in use by linked cloned\n"
2170 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2175 my $remove_owned_drive = sub {
2176 my ($ds, $drive) = @_;
2177 return if drive_is_cdrom
($drive, 1);
2179 my $volid = $drive->{file
};
2180 return if !$volid || $volid =~ m
|^/|;
2182 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2183 return if !$path || !$owner || ($owner != $vmid);
2185 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2186 warn "Could not remove disk '$volid', check manually: $@" if $@;
2189 # only remove disks owned by this VM (referenced in the config)
2190 my $include_opts = {
2191 include_unused
=> 1,
2192 extra_keys
=> ['vmstate'],
2194 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2196 for my $snap (values %{$conf->{snapshots
}}) {
2197 next if !defined($snap->{vmstate
});
2198 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2199 next if !defined($drive);
2200 $remove_owned_drive->('vmstate', $drive);
2203 if ($purge_unreferenced) { # also remove unreferenced disk
2204 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2205 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2206 my ($volid, $sid, $volname, $d) = @_;
2207 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2212 if (defined $replacement_conf) {
2213 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2215 PVE
::QemuConfig-
>destroy_config($vmid);
2219 sub parse_vm_config
{
2220 my ($filename, $raw) = @_;
2222 return if !defined($raw);
2225 digest
=> Digest
::SHA
::sha1_hex
($raw),
2230 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2231 || die "got strange filename '$filename'";
2239 my @lines = split(/\n/, $raw);
2240 foreach my $line (@lines) {
2241 next if $line =~ m/^\s*$/;
2243 if ($line =~ m/^\[PENDING\]\s*$/i) {
2244 $section = 'pending';
2245 if (defined($descr)) {
2247 $conf->{description
} = $descr;
2250 $conf = $res->{$section} = {};
2253 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2255 if (defined($descr)) {
2257 $conf->{description
} = $descr;
2260 $conf = $res->{snapshots
}->{$section} = {};
2264 if ($line =~ m/^\#(.*)\s*$/) {
2265 $descr = '' if !defined($descr);
2266 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2270 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2271 $descr = '' if !defined($descr);
2272 $descr .= PVE
::Tools
::decode_text
($2);
2273 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2274 $conf->{snapstate
} = $1;
2275 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2278 $conf->{$key} = $value;
2279 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2281 if ($section eq 'pending') {
2282 $conf->{delete} = $value; # we parse this later
2284 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2286 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2289 eval { $value = check_type
($key, $value); };
2291 warn "vm $vmid - unable to parse value of '$key' - $@";
2293 $key = 'ide2' if $key eq 'cdrom';
2294 my $fmt = $confdesc->{$key}->{format
};
2295 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2296 my $v = parse_drive
($key, $value);
2297 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2298 $v->{file
} = $volid;
2299 $value = print_drive
($v);
2301 warn "vm $vmid - unable to parse value of '$key'\n";
2306 $conf->{$key} = $value;
2309 warn "vm $vmid - unable to parse config: $line\n";
2313 if (defined($descr)) {
2315 $conf->{description
} = $descr;
2317 delete $res->{snapstate
}; # just to be sure
2322 sub write_vm_config
{
2323 my ($filename, $conf) = @_;
2325 delete $conf->{snapstate
}; # just to be sure
2327 if ($conf->{cdrom
}) {
2328 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2329 $conf->{ide2
} = $conf->{cdrom
};
2330 delete $conf->{cdrom
};
2333 # we do not use 'smp' any longer
2334 if ($conf->{sockets
}) {
2335 delete $conf->{smp
};
2336 } elsif ($conf->{smp
}) {
2337 $conf->{sockets
} = $conf->{smp
};
2338 delete $conf->{cores
};
2339 delete $conf->{smp
};
2342 my $used_volids = {};
2344 my $cleanup_config = sub {
2345 my ($cref, $pending, $snapname) = @_;
2347 foreach my $key (keys %$cref) {
2348 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2349 $key eq 'snapstate' || $key eq 'pending';
2350 my $value = $cref->{$key};
2351 if ($key eq 'delete') {
2352 die "propertry 'delete' is only allowed in [PENDING]\n"
2354 # fixme: check syntax?
2357 eval { $value = check_type
($key, $value); };
2358 die "unable to parse value of '$key' - $@" if $@;
2360 $cref->{$key} = $value;
2362 if (!$snapname && is_valid_drivename
($key)) {
2363 my $drive = parse_drive
($key, $value);
2364 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2369 &$cleanup_config($conf);
2371 &$cleanup_config($conf->{pending
}, 1);
2373 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2374 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2375 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2378 # remove 'unusedX' settings if we re-add a volume
2379 foreach my $key (keys %$conf) {
2380 my $value = $conf->{$key};
2381 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2382 delete $conf->{$key};
2386 my $generate_raw_config = sub {
2387 my ($conf, $pending) = @_;
2391 # add description as comment to top of file
2392 if (defined(my $descr = $conf->{description
})) {
2394 foreach my $cl (split(/\n/, $descr)) {
2395 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2398 $raw .= "#\n" if $pending;
2402 foreach my $key (sort keys %$conf) {
2403 next if $key =~ /^(digest|description|pending|snapshots)$/;
2404 $raw .= "$key: $conf->{$key}\n";
2409 my $raw = &$generate_raw_config($conf);
2411 if (scalar(keys %{$conf->{pending
}})){
2412 $raw .= "\n[PENDING]\n";
2413 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2416 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2417 $raw .= "\n[$snapname]\n";
2418 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2428 # we use static defaults from our JSON schema configuration
2429 foreach my $key (keys %$confdesc) {
2430 if (defined(my $default = $confdesc->{$key}->{default})) {
2431 $res->{$key} = $default;
2439 my $vmlist = PVE
::Cluster
::get_vmlist
();
2441 return $res if !$vmlist || !$vmlist->{ids
};
2442 my $ids = $vmlist->{ids
};
2443 my $nodename = nodename
();
2445 foreach my $vmid (keys %$ids) {
2446 my $d = $ids->{$vmid};
2447 next if !$d->{node
} || $d->{node
} ne $nodename;
2448 next if !$d->{type
} || $d->{type
} ne 'qemu';
2449 $res->{$vmid}->{exists} = 1;
2454 # test if VM uses local resources (to prevent migration)
2455 sub check_local_resources
{
2456 my ($conf, $noerr) = @_;
2460 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2461 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2463 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2465 foreach my $k (keys %$conf) {
2466 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2467 # sockets are safe: they will recreated be on the target side post-migrate
2468 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2469 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2472 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2477 # check if used storages are available on all nodes (use by migrate)
2478 sub check_storage_availability
{
2479 my ($storecfg, $conf, $node) = @_;
2481 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2482 my ($ds, $drive) = @_;
2484 my $volid = $drive->{file
};
2487 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2490 # check if storage is available on both nodes
2491 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2492 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2494 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2496 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2497 if !$scfg->{content
}->{$vtype};
2501 # list nodes where all VM images are available (used by has_feature API)
2503 my ($conf, $storecfg) = @_;
2505 my $nodelist = PVE
::Cluster
::get_nodelist
();
2506 my $nodehash = { map { $_ => 1 } @$nodelist };
2507 my $nodename = nodename
();
2509 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2510 my ($ds, $drive) = @_;
2512 my $volid = $drive->{file
};
2515 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2517 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2518 if ($scfg->{disable
}) {
2520 } elsif (my $avail = $scfg->{nodes
}) {
2521 foreach my $node (keys %$nodehash) {
2522 delete $nodehash->{$node} if !$avail->{$node};
2524 } elsif (!$scfg->{shared
}) {
2525 foreach my $node (keys %$nodehash) {
2526 delete $nodehash->{$node} if $node ne $nodename
2535 sub check_local_storage_availability
{
2536 my ($conf, $storecfg) = @_;
2538 my $nodelist = PVE
::Cluster
::get_nodelist
();
2539 my $nodehash = { map { $_ => {} } @$nodelist };
2541 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2542 my ($ds, $drive) = @_;
2544 my $volid = $drive->{file
};
2547 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2549 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2551 if ($scfg->{disable
}) {
2552 foreach my $node (keys %$nodehash) {
2553 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2555 } elsif (my $avail = $scfg->{nodes
}) {
2556 foreach my $node (keys %$nodehash) {
2557 if (!$avail->{$node}) {
2558 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2565 foreach my $node (values %$nodehash) {
2566 if (my $unavail = $node->{unavailable_storages
}) {
2567 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2574 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2576 my ($vmid, $nocheck, $node) = @_;
2578 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2579 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2584 my $vzlist = config_list
();
2586 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2588 while (defined(my $de = $fd->read)) {
2589 next if $de !~ m/^(\d+)\.pid$/;
2591 next if !defined($vzlist->{$vmid});
2592 if (my $pid = check_running
($vmid)) {
2593 $vzlist->{$vmid}->{pid
} = $pid;
2600 our $vmstatus_return_properties = {
2601 vmid
=> get_standard_option
('pve-vmid'),
2603 description
=> "Qemu process status.",
2605 enum
=> ['stopped', 'running'],
2608 description
=> "Maximum memory in bytes.",
2611 renderer
=> 'bytes',
2614 description
=> "Root disk size in bytes.",
2617 renderer
=> 'bytes',
2620 description
=> "VM name.",
2625 description
=> "Qemu QMP agent status.",
2630 description
=> "PID of running qemu process.",
2635 description
=> "Uptime.",
2638 renderer
=> 'duration',
2641 description
=> "Maximum usable CPUs.",
2646 description
=> "The current config lock, if any.",
2651 description
=> "The current configured tags, if any",
2655 'running-machine' => {
2656 description
=> "The currently running machine type (if running).",
2661 description
=> "The currently running QEMU version (if running).",
2667 my $last_proc_pid_stat;
2669 # get VM status information
2670 # This must be fast and should not block ($full == false)
2671 # We only query KVM using QMP if $full == true (this can be slow)
2673 my ($opt_vmid, $full) = @_;
2677 my $storecfg = PVE
::Storage
::config
();
2679 my $list = vzlist
();
2680 my $defaults = load_defaults
();
2682 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2684 my $cpucount = $cpuinfo->{cpus
} || 1;
2686 foreach my $vmid (keys %$list) {
2687 next if $opt_vmid && ($vmid ne $opt_vmid);
2689 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2691 my $d = { vmid
=> int($vmid) };
2692 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2694 # fixme: better status?
2695 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2697 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2698 if (defined($size)) {
2699 $d->{disk
} = 0; # no info available
2700 $d->{maxdisk
} = $size;
2706 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2707 * ($conf->{cores
} || $defaults->{cores
});
2708 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2709 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2711 $d->{name
} = $conf->{name
} || "VM $vmid";
2712 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2713 : $defaults->{memory
}*(1024*1024);
2715 if ($conf->{balloon
}) {
2716 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2717 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2718 : $defaults->{shares
};
2729 $d->{diskwrite
} = 0;
2731 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2733 $d->{serial
} = 1 if conf_has_serial
($conf);
2734 $d->{lock} = $conf->{lock} if $conf->{lock};
2735 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2740 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2741 foreach my $dev (keys %$netdev) {
2742 next if $dev !~ m/^tap([1-9]\d*)i/;
2744 my $d = $res->{$vmid};
2747 $d->{netout
} += $netdev->{$dev}->{receive
};
2748 $d->{netin
} += $netdev->{$dev}->{transmit
};
2751 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2752 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2757 my $ctime = gettimeofday
;
2759 foreach my $vmid (keys %$list) {
2761 my $d = $res->{$vmid};
2762 my $pid = $d->{pid
};
2765 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2766 next if !$pstat; # not running
2768 my $used = $pstat->{utime} + $pstat->{stime
};
2770 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2772 if ($pstat->{vsize
}) {
2773 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2776 my $old = $last_proc_pid_stat->{$pid};
2778 $last_proc_pid_stat->{$pid} = {
2786 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2788 if ($dtime > 1000) {
2789 my $dutime = $used - $old->{used
};
2791 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2792 $last_proc_pid_stat->{$pid} = {
2798 $d->{cpu
} = $old->{cpu
};
2802 return $res if !$full;
2804 my $qmpclient = PVE
::QMPClient-
>new();
2806 my $ballooncb = sub {
2807 my ($vmid, $resp) = @_;
2809 my $info = $resp->{'return'};
2810 return if !$info->{max_mem
};
2812 my $d = $res->{$vmid};
2814 # use memory assigned to VM
2815 $d->{maxmem
} = $info->{max_mem
};
2816 $d->{balloon
} = $info->{actual
};
2818 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2819 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2820 $d->{freemem
} = $info->{free_mem
};
2823 $d->{ballooninfo
} = $info;
2826 my $blockstatscb = sub {
2827 my ($vmid, $resp) = @_;
2828 my $data = $resp->{'return'} || [];
2829 my $totalrdbytes = 0;
2830 my $totalwrbytes = 0;
2832 for my $blockstat (@$data) {
2833 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2834 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2836 $blockstat->{device
} =~ s/drive-//;
2837 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2839 $res->{$vmid}->{diskread
} = $totalrdbytes;
2840 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2843 my $machinecb = sub {
2844 my ($vmid, $resp) = @_;
2845 my $data = $resp->{'return'} || [];
2847 $res->{$vmid}->{'running-machine'} =
2848 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
2851 my $versioncb = sub {
2852 my ($vmid, $resp) = @_;
2853 my $data = $resp->{'return'} // {};
2854 my $version = 'unknown';
2856 if (my $v = $data->{qemu
}) {
2857 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
2860 $res->{$vmid}->{'running-qemu'} = $version;
2863 my $statuscb = sub {
2864 my ($vmid, $resp) = @_;
2866 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2867 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2868 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2869 # this fails if ballon driver is not loaded, so this must be
2870 # the last commnand (following command are aborted if this fails).
2871 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2873 my $status = 'unknown';
2874 if (!defined($status = $resp->{'return'}->{status
})) {
2875 warn "unable to get VM status\n";
2879 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2882 foreach my $vmid (keys %$list) {
2883 next if $opt_vmid && ($vmid ne $opt_vmid);
2884 next if !$res->{$vmid}->{pid
}; # not running
2885 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2888 $qmpclient->queue_execute(undef, 2);
2890 foreach my $vmid (keys %$list) {
2891 next if $opt_vmid && ($vmid ne $opt_vmid);
2892 next if !$res->{$vmid}->{pid
}; #not running
2894 # we can't use the $qmpclient since it might have already aborted on
2895 # 'query-balloon', but this might also fail for older versions...
2896 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
2897 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2900 foreach my $vmid (keys %$list) {
2901 next if $opt_vmid && ($vmid ne $opt_vmid);
2902 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2908 sub conf_has_serial
{
2911 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2912 if ($conf->{"serial$i"}) {
2920 sub conf_has_audio
{
2921 my ($conf, $id) = @_;
2924 my $audio = $conf->{"audio$id"};
2925 return if !defined($audio);
2927 my $audioproperties = parse_property_string
($audio_fmt, $audio);
2928 my $audiodriver = $audioproperties->{driver
} // 'spice';
2931 dev
=> $audioproperties->{device
},
2932 dev_id
=> "audiodev$id",
2933 backend
=> $audiodriver,
2934 backend_id
=> "$audiodriver-backend${id}",
2939 my ($audio, $audiopciaddr, $machine_version) = @_;
2943 my $id = $audio->{dev_id
};
2945 if (min_version
($machine_version, 4, 2)) {
2946 $audiodev = ",audiodev=$audio->{backend_id}";
2949 if ($audio->{dev
} eq 'AC97') {
2950 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2951 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
2952 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2953 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2954 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2956 die "unkown audio device '$audio->{dev}', implement me!";
2959 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2964 sub vga_conf_has_spice
{
2967 my $vgaconf = parse_vga
($vga);
2968 my $vgatype = $vgaconf->{type
};
2969 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2976 return get_host_arch
() eq $arch;
2981 return $conf->{arch
} // get_host_arch
();
2984 my $default_machines = {
2989 sub get_installed_machine_version
{
2990 my ($kvmversion) = @_;
2991 $kvmversion = kvm_user_version
() if !defined($kvmversion);
2992 $kvmversion =~ m/^(\d+\.\d+)/;
2996 sub windows_get_pinned_machine_version
{
2997 my ($machine, $base_version, $kvmversion) = @_;
2999 my $pin_version = $base_version;
3000 if (!defined($base_version) ||
3001 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3003 $pin_version = get_installed_machine_version
($kvmversion);
3005 if (!$machine || $machine eq 'pc') {
3006 $machine = "pc-i440fx-$pin_version";
3007 } elsif ($machine eq 'q35') {
3008 $machine = "pc-q35-$pin_version";
3009 } elsif ($machine eq 'virt') {
3010 $machine = "virt-$pin_version";
3012 warn "unknown machine type '$machine', not touching that!\n";
3018 sub get_vm_machine
{
3019 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3021 my $machine = $forcemachine || $conf->{machine
};
3023 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3024 $kvmversion //= kvm_user_version
();
3025 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3026 # layout which confuses windows quite a bit and may result in various regressions..
3027 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3028 if (windows_version
($conf->{ostype
})) {
3029 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3032 $machine ||= $default_machines->{$arch};
3033 if ($add_pve_version) {
3034 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3035 $machine .= "+pve$pvever";
3039 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3040 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3041 $machine = $1 if $is_pxe;
3043 # for version-pinned machines that do not include a pve-version (e.g.
3044 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3045 $machine .= '+pve0';
3047 $machine .= '.pxe' if $is_pxe;
3053 sub get_ovmf_files
($) {
3056 my $ovmf = $OVMF->{$arch}
3057 or die "no OVMF images known for architecture '$arch'\n";
3063 aarch64
=> '/usr/bin/qemu-system-aarch64',
3064 x86_64
=> '/usr/bin/qemu-system-x86_64',
3066 sub get_command_for_arch
($) {
3068 return '/usr/bin/kvm' if is_native
($arch);
3070 my $cmd = $Arch2Qemu->{$arch}
3071 or die "don't know how to emulate architecture '$arch'\n";
3075 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3076 # to use in a QEMU command line (-cpu element), first array_intersect the result
3077 # of query_supported_ with query_understood_. This is necessary because:
3079 # a) query_understood_ returns flags the host cannot use and
3080 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3081 # flags, but CPU settings - with most of them being flags. Those settings
3082 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3084 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3085 # expensive. If you need the value returned from this, you can get it much
3086 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3087 # $accel being 'kvm' or 'tcg'.
3089 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3090 # changes, automatically populating pmxcfs.
3092 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3093 # since kvm and tcg machines support different flags
3095 sub query_supported_cpu_flags
{
3098 $arch //= get_host_arch
();
3099 my $default_machine = $default_machines->{$arch};
3103 # FIXME: Once this is merged, the code below should work for ARM as well:
3104 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3105 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3108 my $kvm_supported = defined(kvm_version
());
3109 my $qemu_cmd = get_command_for_arch
($arch);
3111 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3113 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3114 my $query_supported_run_qemu = sub {
3120 '-machine', $default_machine,
3122 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3123 '-mon', 'chardev=qmp,mode=control',
3124 '-pidfile', $pidfile,
3129 push @$cmd, '-accel', 'tcg';
3132 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3133 die "QEMU flag querying VM exited with code " . $rc if $rc;
3136 my $cmd_result = mon_cmd
(
3138 'query-cpu-model-expansion',
3140 model
=> { name
=> 'host' }
3143 my $props = $cmd_result->{model
}->{props
};
3144 foreach my $prop (keys %$props) {
3145 next if $props->{$prop} ne '1';
3146 # QEMU returns some flags multiple times, with '_', '.' or '-'
3147 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3148 # We only keep those with underscores, to match /proc/cpuinfo
3149 $prop =~ s/\.|-/_/g;
3150 $flags->{$prop} = 1;
3155 # force stop with 10 sec timeout and 'nocheck'
3156 # always stop, even if QMP failed
3157 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3161 return [ sort keys %$flags ];
3164 # We need to query QEMU twice, since KVM and TCG have different supported flags
3165 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3166 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3167 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3169 if ($kvm_supported) {
3170 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3171 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3178 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3179 my $understood_cpu_flag_dir = "/usr/share/kvm";
3180 sub query_understood_cpu_flags
{
3181 my $arch = get_host_arch
();
3182 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3184 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3187 my $raw = file_get_contents
($filepath);
3188 $raw =~ s/^\s+|\s+$//g;
3189 my @flags = split(/\s+/, $raw);
3194 my sub get_cpuunits
{
3196 return $conf->{cpuunits
} // (PVE
::CGroup
::cgroup_mode
() == 2 ?
100 : 1024);
3198 sub config_to_command
{
3199 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3203 my $globalFlags = [];
3204 my $machineFlags = [];
3209 my $ostype = $conf->{ostype
};
3210 my $winversion = windows_version
($ostype);
3211 my $kvm = $conf->{kvm
};
3212 my $nodename = nodename
();
3214 my $arch = get_vm_arch
($conf);
3215 my $kvm_binary = get_command_for_arch
($arch);
3216 my $kvmver = kvm_user_version
($kvm_binary);
3218 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3219 $kvmver //= "undefined";
3220 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3223 my $add_pve_version = min_version
($kvmver, 4, 1);
3225 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3226 my $machine_version = extract_version
($machine_type, $kvmver);
3227 $kvm //= 1 if is_native
($arch);
3229 $machine_version =~ m/(\d+)\.(\d+)/;
3230 my ($machine_major, $machine_minor) = ($1, $2);
3232 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3233 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3234 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3235 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3236 ." please upgrade node '$nodename'\n"
3237 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3238 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3239 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3240 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3241 ." node '$nodename'\n";
3244 # if a specific +pve version is required for a feature, use $version_guard
3245 # instead of min_version to allow machines to be run with the minimum
3247 my $required_pve_version = 0;
3248 my $version_guard = sub {
3249 my ($major, $minor, $pve) = @_;
3250 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3251 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3252 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3253 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3257 if ($kvm && !defined kvm_version
()) {
3258 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3259 ." or enable in BIOS.\n";
3262 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3263 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3264 my $use_old_bios_files = undef;
3265 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3267 my $cpuunits = get_cpuunits
($conf);
3269 push @$cmd, $kvm_binary;
3271 push @$cmd, '-id', $vmid;
3273 my $vmname = $conf->{name
} || "vm$vmid";
3275 push @$cmd, '-name', $vmname;
3277 push @$cmd, '-no-shutdown';
3281 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3282 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3283 push @$cmd, '-mon', "chardev=qmp,mode=control";
3285 if (min_version
($machine_version, 2, 12)) {
3286 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3287 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3290 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3292 push @$cmd, '-daemonize';
3294 if ($conf->{smbios1
}) {
3295 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3296 if ($smbios_conf->{base64
}) {
3297 # Do not pass base64 flag to qemu
3298 delete $smbios_conf->{base64
};
3299 my $smbios_string = "";
3300 foreach my $key (keys %$smbios_conf) {
3302 if ($key eq "uuid") {
3303 $value = $smbios_conf->{uuid
}
3305 $value = decode_base64
($smbios_conf->{$key});
3307 # qemu accepts any binary data, only commas need escaping by double comma
3309 $smbios_string .= "," . $key . "=" . $value if $value;
3311 push @$cmd, '-smbios', "type=1" . $smbios_string;
3313 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3317 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3318 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3319 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3321 my ($path, $format);
3322 my $read_only_str = '';
3323 if (my $efidisk = $conf->{efidisk0
}) {
3324 my $d = parse_drive
('efidisk0', $efidisk);
3325 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3326 $format = $d->{format
};
3328 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3329 if (!defined($format)) {
3330 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3331 $format = qemu_img_format
($scfg, $volname);
3335 die "efidisk format must be specified\n"
3336 if !defined($format);
3339 $read_only_str = ',readonly=on' if drive_is_read_only
($conf, $d);
3341 warn "no efidisk configured! Using temporary efivars disk.\n";
3342 $path = "/tmp/$vmid-ovmf.fd";
3343 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3349 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3350 $size_str = ",size=" . (-s
$ovmf_vars);
3353 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3355 if ($path =~ m/^rbd:/) {
3356 $cache = ',cache=writeback';
3357 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3360 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3361 push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3366 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3367 if (min_version
($machine_version, 4, 0)) {
3368 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3370 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3374 if ($conf->{vmgenid
}) {
3375 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3378 # add usb controllers
3379 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3380 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3381 push @$devices, @usbcontrollers if @usbcontrollers;
3382 my $vga = parse_vga
($conf->{vga
});
3384 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3385 $vga->{type
} = 'qxl' if $qxlnum;
3387 if (!$vga->{type
}) {
3388 if ($arch eq 'aarch64') {
3389 $vga->{type
} = 'virtio';
3390 } elsif (min_version
($machine_version, 2, 9)) {
3391 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3393 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3397 # enable absolute mouse coordinates (needed by vnc)
3399 if (defined($conf->{tablet
})) {
3400 $tablet = $conf->{tablet
};
3402 $tablet = $defaults->{tablet
};
3403 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3404 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3408 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3409 my $kbd = print_keyboarddevice_full
($conf, $arch);
3410 push @$devices, '-device', $kbd if defined($kbd);
3413 my $bootorder = device_bootorder
($conf);
3415 # host pci device passthrough
3416 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3417 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3420 my $usb_dev_features = {};
3421 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3423 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3424 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3425 push @$devices, @usbdevices if @usbdevices;
3428 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3429 if (my $path = $conf->{"serial$i"}) {
3430 if ($path eq 'socket') {
3431 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3432 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3433 # On aarch64, serial0 is the UART device. Qemu only allows
3434 # connecting UART devices via the '-serial' command line, as
3435 # the device has a fixed slot on the hardware...
3436 if ($arch eq 'aarch64' && $i == 0) {
3437 push @$devices, '-serial', "chardev:serial$i";
3439 push @$devices, '-device', "isa-serial,chardev=serial$i";
3442 die "no such serial device\n" if ! -c
$path;
3443 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3444 push @$devices, '-device', "isa-serial,chardev=serial$i";
3450 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3451 if (my $path = $conf->{"parallel$i"}) {
3452 die "no such parallel device\n" if ! -c
$path;
3453 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3454 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3455 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3459 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3460 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3461 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3462 push @$devices, @$audio_devs;
3466 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3467 $sockets = $conf->{sockets
} if $conf->{sockets
};
3469 my $cores = $conf->{cores
} || 1;
3471 my $maxcpus = $sockets * $cores;
3473 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3475 my $allowed_vcpus = $cpuinfo->{cpus
};
3477 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3478 if ($allowed_vcpus < $maxcpus);
3480 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3482 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3483 for (my $i = 2; $i <= $vcpus; $i++) {
3484 my $cpustr = print_cpu_device
($conf,$i);
3485 push @$cmd, '-device', $cpustr;
3490 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3492 push @$cmd, '-nodefaults';
3494 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3496 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3498 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3500 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3501 push @$devices, '-device', print_vga_device
(
3502 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3503 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3504 push @$cmd, '-vnc', "unix:$socket,password=on";
3506 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3507 push @$cmd, '-nographic';
3511 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3512 my $useLocaltime = $conf->{localtime};
3514 if ($winversion >= 5) { # windows
3515 $useLocaltime = 1 if !defined($conf->{localtime});
3517 # use time drift fix when acpi is enabled
3518 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3519 $tdf = 1 if !defined($conf->{tdf
});
3523 if ($winversion >= 6) {
3524 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3525 push @$cmd, '-no-hpet';
3528 push @$rtcFlags, 'driftfix=slew' if $tdf;
3530 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3531 push @$rtcFlags, "base=$conf->{startdate}";
3532 } elsif ($useLocaltime) {
3533 push @$rtcFlags, 'base=localtime';
3537 push @$cmd, '-cpu', $forcecpu;
3539 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3542 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3544 push @$cmd, '-S' if $conf->{freeze
};
3546 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3548 my $guest_agent = parse_guest_agent
($conf);
3550 if ($guest_agent->{enabled
}) {
3551 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3552 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3554 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3555 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3556 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3557 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3558 } elsif ($guest_agent->{type
} eq 'isa') {
3559 push @$devices, '-device', "isa-serial,chardev=qga0";
3563 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3564 if ($rng && $version_guard->(4, 1, 2)) {
3565 check_rng_source
($rng->{source
});
3567 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3568 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3569 my $limiter_str = "";
3571 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3574 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3575 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3576 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3584 for (my $i = 1; $i < $qxlnum; $i++){
3585 push @$devices, '-device', print_vga_device
(
3586 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3589 # assume other OS works like Linux
3590 my ($ram, $vram) = ("134217728", "67108864");
3591 if ($vga->{memory
}) {
3592 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3593 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3595 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3596 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3600 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3602 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3603 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3604 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3606 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3607 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3608 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3610 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3611 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3613 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3614 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3615 if ($spice_enhancement->{foldersharing
}) {
3616 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3617 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3620 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3621 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3622 if $spice_enhancement->{videostreaming
};
3624 push @$devices, '-spice', "$spice_opts";
3627 # enable balloon by default, unless explicitly disabled
3628 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3629 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3630 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3633 if ($conf->{watchdog
}) {
3634 my $wdopts = parse_watchdog
($conf->{watchdog
});
3635 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3636 my $watchdog = $wdopts->{model
} || 'i6300esb';
3637 push @$devices, '-device', "$watchdog$pciaddr";
3638 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3642 my $scsicontroller = {};
3643 my $ahcicontroller = {};
3644 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3646 # Add iscsi initiator name if available
3647 if (my $initiator = get_initiator_name
()) {
3648 push @$devices, '-iscsi', "initiator-name=$initiator";
3651 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3652 my ($ds, $drive) = @_;
3654 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3655 check_volume_storage_type
($storecfg, $drive->{file
});
3656 push @$vollist, $drive->{file
};
3659 # ignore efidisk here, already added in bios/fw handling code above
3660 return if $drive->{interface
} eq 'efidisk';
3662 $use_virtio = 1 if $ds =~ m/^virtio/;
3664 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3666 if ($drive->{interface
} eq 'virtio'){
3667 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3670 if ($drive->{interface
} eq 'scsi') {
3672 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3674 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3675 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3677 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3678 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3681 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3682 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3683 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3684 } elsif ($drive->{iothread
}) {
3685 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3689 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3690 $queues = ",num_queues=$drive->{queues}";
3693 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3694 if !$scsicontroller->{$controller};
3695 $scsicontroller->{$controller}=1;
3698 if ($drive->{interface
} eq 'sata') {
3699 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3700 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3701 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3702 if !$ahcicontroller->{$controller};
3703 $ahcicontroller->{$controller}=1;
3706 my $pbs_conf = $pbs_backing->{$ds};
3707 my $pbs_name = undef;
3709 $pbs_name = "drive-$ds-pbs";
3710 push @$devices, '-blockdev', print_pbs_blockdev
($pbs_conf, $pbs_name);
3713 my $drive_cmd = print_drive_commandline_full
(
3714 $storecfg, $vmid, $drive, $pbs_name, min_version
($kvmver, 6, 0));
3716 # extra protection for templates, but SATA and IDE don't support it..
3717 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
3719 push @$devices, '-drive',$drive_cmd;
3720 push @$devices, '-device', print_drivedevice_full
(
3721 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3724 for (my $i = 0; $i < $MAX_NETS; $i++) {
3725 my $netname = "net$i";
3727 next if !$conf->{$netname};
3728 my $d = parse_net
($conf->{$netname});
3731 $use_virtio = 1 if $d->{model
} eq 'virtio';
3733 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
3735 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
3736 push @$devices, '-netdev', $netdevfull;
3738 my $netdevicefull = print_netdevice_full
(
3739 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3741 push @$devices, '-device', $netdevicefull;
3744 if ($conf->{ivshmem
}) {
3745 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3749 $bus = print_pcie_addr
("ivshmem");
3751 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3754 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3755 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3757 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3758 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3759 .",size=$ivshmem->{size}M";
3762 # pci.4 is nested in pci.1
3763 $bridges->{1} = 1 if $bridges->{4};
3767 if (min_version
($machine_version, 2, 3)) {
3772 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3776 for my $k (sort {$b cmp $a} keys %$bridges) {
3777 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3780 if ($k == 2 && $legacy_igd) {
3783 $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
3785 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3787 # add after -readconfig pve-q35.cfg
3788 splice @$devices, 2, 0, '-device', $devstr;
3790 unshift @$devices, '-device', $devstr if $k > 0;
3795 push @$machineFlags, 'accel=tcg';
3798 my $machine_type_min = $machine_type;
3799 if ($add_pve_version) {
3800 $machine_type_min =~ s/\+pve\d+$//;
3801 $machine_type_min .= "+pve$required_pve_version";
3803 push @$machineFlags, "type=${machine_type_min}";
3805 push @$cmd, @$devices;
3806 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3807 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3808 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3810 if (my $vmstate = $conf->{vmstate
}) {
3811 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3812 push @$vollist, $vmstate;
3813 push @$cmd, '-loadstate', $statepath;
3814 print "activating and using '$vmstate' as vmstate\n";
3817 if (PVE
::QemuConfig-
>is_template($conf)) {
3818 # needed to workaround base volumes being read-only
3819 push @$cmd, '-snapshot';
3823 if ($conf->{args
}) {
3824 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3828 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3831 sub check_rng_source
{
3834 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3835 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3838 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3839 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
3840 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3841 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3842 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3843 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3851 my $res = mon_cmd
($vmid, 'query-spice');
3853 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3856 sub vm_devices_list
{
3859 my $res = mon_cmd
($vmid, 'query-pci');
3860 my $devices_to_check = [];
3862 foreach my $pcibus (@$res) {
3863 push @$devices_to_check, @{$pcibus->{devices
}},
3866 while (@$devices_to_check) {
3868 for my $d (@$devices_to_check) {
3869 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3870 next if !$d->{'pci_bridge'};
3872 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3873 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3875 $devices_to_check = $to_check;
3878 my $resblock = mon_cmd
($vmid, 'query-block');
3879 foreach my $block (@$resblock) {
3880 if($block->{device
} =~ m/^drive-(\S+)/){
3885 my $resmice = mon_cmd
($vmid, 'query-mice');
3886 foreach my $mice (@$resmice) {
3887 if ($mice->{name
} eq 'QEMU HID Tablet') {
3888 $devices->{tablet
} = 1;
3893 # for usb devices there is no query-usb
3894 # but we can iterate over the entries in
3895 # qom-list path=/machine/peripheral
3896 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3897 foreach my $per (@$resperipheral) {
3898 if ($per->{name
} =~ m/^usb\d+$/) {
3899 $devices->{$per->{name
}} = 1;
3907 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3909 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3911 my $devices_list = vm_devices_list
($vmid);
3912 return 1 if defined($devices_list->{$deviceid});
3914 # add PCI bridge if we need it for the device
3915 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3917 if ($deviceid eq 'tablet') {
3919 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
3921 } elsif ($deviceid eq 'keyboard') {
3923 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
3925 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3927 die "usb hotplug currently not reliable\n";
3928 # since we can't reliably hot unplug all added usb devices and usb
3929 # passthrough breaks live migration we disable usb hotplugging for now
3930 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3932 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3934 qemu_iothread_add
($vmid, $deviceid, $device);
3936 qemu_driveadd
($storecfg, $vmid, $device);
3937 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3939 qemu_deviceadd
($vmid, $devicefull);
3940 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3942 eval { qemu_drivedel
($vmid, $deviceid); };
3947 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3950 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3951 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3952 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3954 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3956 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3957 qemu_iothread_add
($vmid, $deviceid, $device);
3958 $devicefull .= ",iothread=iothread-$deviceid";
3961 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3962 $devicefull .= ",num_queues=$device->{queues}";
3965 qemu_deviceadd
($vmid, $devicefull);
3966 qemu_deviceaddverify
($vmid, $deviceid);
3968 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3970 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3971 qemu_driveadd
($storecfg, $vmid, $device);
3973 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3974 eval { qemu_deviceadd
($vmid, $devicefull); };
3976 eval { qemu_drivedel
($vmid, $deviceid); };
3981 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3983 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
3985 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
3986 my $use_old_bios_files = undef;
3987 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3989 my $netdevicefull = print_netdevice_full
(
3990 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3991 qemu_deviceadd
($vmid, $netdevicefull);
3993 qemu_deviceaddverify
($vmid, $deviceid);
3994 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
3997 eval { qemu_netdevdel
($vmid, $deviceid); };
4002 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4005 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4006 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4008 qemu_deviceadd
($vmid, $devicefull);
4009 qemu_deviceaddverify
($vmid, $deviceid);
4012 die "can't hotplug device '$deviceid'\n";
4018 # fixme: this should raise exceptions on error!
4019 sub vm_deviceunplug
{
4020 my ($vmid, $conf, $deviceid) = @_;
4022 my $devices_list = vm_devices_list
($vmid);
4023 return 1 if !defined($devices_list->{$deviceid});
4025 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4026 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4028 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4030 qemu_devicedel
($vmid, $deviceid);
4032 } elsif ($deviceid =~ m/^usb\d+$/) {
4034 die "usb hotplug currently not reliable\n";
4035 # when unplugging usb devices this way, there may be remaining usb
4036 # controllers/hubs so we disable it for now
4037 #qemu_devicedel($vmid, $deviceid);
4038 #qemu_devicedelverify($vmid, $deviceid);
4040 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4041 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4043 qemu_devicedel
($vmid, $deviceid);
4044 qemu_devicedelverify
($vmid, $deviceid);
4045 qemu_drivedel
($vmid, $deviceid);
4046 qemu_iothread_del
($vmid, $deviceid, $device);
4048 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4050 qemu_devicedel
($vmid, $deviceid);
4051 qemu_devicedelverify
($vmid, $deviceid);
4053 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4054 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4056 qemu_devicedel
($vmid, $deviceid);
4057 qemu_drivedel
($vmid, $deviceid);
4058 qemu_deletescsihw
($conf, $vmid, $deviceid);
4060 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4061 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4063 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4065 qemu_devicedel
($vmid, $deviceid);
4066 qemu_devicedelverify
($vmid, $deviceid);
4067 qemu_netdevdel
($vmid, $deviceid);
4070 die "can't unplug device '$deviceid'\n";
4076 sub qemu_deviceadd
{
4077 my ($vmid, $devicefull) = @_;
4079 $devicefull = "driver=".$devicefull;
4080 my %options = split(/[=,]/, $devicefull);
4082 mon_cmd
($vmid, "device_add" , %options);
4085 sub qemu_devicedel
{
4086 my ($vmid, $deviceid) = @_;
4088 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4091 sub qemu_iothread_add
{
4092 my($vmid, $deviceid, $device) = @_;
4094 if ($device->{iothread
}) {
4095 my $iothreads = vm_iothreads_list
($vmid);
4096 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4100 sub qemu_iothread_del
{
4101 my($vmid, $deviceid, $device) = @_;
4103 if ($device->{iothread
}) {
4104 my $iothreads = vm_iothreads_list
($vmid);
4105 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4109 sub qemu_objectadd
{
4110 my($vmid, $objectid, $qomtype) = @_;
4112 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4117 sub qemu_objectdel
{
4118 my($vmid, $objectid) = @_;
4120 mon_cmd
($vmid, "object-del", id
=> $objectid);
4126 my ($storecfg, $vmid, $device) = @_;
4128 my $kvmver = get_running_qemu_version
($vmid);
4129 my $io_uring = min_version
($kvmver, 6, 0);
4130 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4131 $drive =~ s/\\/\\\\/g;
4132 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4134 # If the command succeeds qemu prints: "OK
"
4135 return 1 if $ret =~ m/OK/s;
4137 die "adding drive failed
: $ret\n";
4141 my($vmid, $deviceid) = @_;
4143 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4146 return 1 if $ret eq "";
4148 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4149 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4151 die "deleting drive
$deviceid failed
: $ret\n";
4154 sub qemu_deviceaddverify {
4155 my ($vmid, $deviceid) = @_;
4157 for (my $i = 0; $i <= 5; $i++) {
4158 my $devices_list = vm_devices_list($vmid);
4159 return 1 if defined($devices_list->{$deviceid});
4163 die "error on hotplug device
'$deviceid'\n";
4167 sub qemu_devicedelverify {
4168 my ($vmid, $deviceid) = @_;
4170 # need to verify that the device is correctly removed as device_del
4171 # is async and empty return is not reliable
4173 for (my $i = 0; $i <= 5; $i++) {
4174 my $devices_list = vm_devices_list($vmid);
4175 return 1 if !defined($devices_list->{$deviceid});
4179 die "error on hot-unplugging device
'$deviceid'\n";
4182 sub qemu_findorcreatescsihw {
4183 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4185 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4187 my $scsihwid="$controller_prefix$controller";
4188 my $devices_list = vm_devices_list($vmid);
4190 if(!defined($devices_list->{$scsihwid})) {
4191 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4197 sub qemu_deletescsihw {
4198 my ($conf, $vmid, $opt) = @_;
4200 my $device = parse_drive($opt, $conf->{$opt});
4202 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4203 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4207 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4209 my $devices_list = vm_devices_list($vmid);
4210 foreach my $opt (keys %{$devices_list}) {
4211 if (is_valid_drivename($opt)) {
4212 my $drive = parse_drive($opt, $conf->{$opt});
4213 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4219 my $scsihwid="scsihw
$controller";
4221 vm_deviceunplug($vmid, $conf, $scsihwid);
4226 sub qemu_add_pci_bridge {
4227 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4233 print_pci_addr($device, $bridges, $arch, $machine_type);
4235 while (my ($k, $v) = each %$bridges) {
4238 return 1 if !defined($bridgeid) || $bridgeid < 1;
4240 my $bridge = "pci
.$bridgeid";
4241 my $devices_list = vm_devices_list($vmid);
4243 if (!defined($devices_list->{$bridge})) {
4244 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4250 sub qemu_set_link_status {
4251 my ($vmid, $device, $up) = @_;
4253 mon_cmd($vmid, "set_link
", name => $device,
4254 up => $up ? JSON::true : JSON::false);
4257 sub qemu_netdevadd {
4258 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4260 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4261 my %options = split(/[=,]/, $netdev);
4263 if (defined(my $vhost = $options{vhost})) {
4264 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4267 if (defined(my $queues = $options{queues})) {
4268 $options{queues} = $queues + 0;
4271 mon_cmd($vmid, "netdev_add
", %options);
4275 sub qemu_netdevdel {
4276 my ($vmid, $deviceid) = @_;
4278 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4281 sub qemu_usb_hotplug {
4282 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4286 # remove the old one first
4287 vm_deviceunplug($vmid, $conf, $deviceid);
4289 # check if xhci controller is necessary and available
4290 if ($device->{usb3}) {
4292 my $devicelist = vm_devices_list($vmid);
4294 if (!$devicelist->{xhci}) {
4295 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4296 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4299 my $d = parse_usb_device($device->{host});
4300 $d->{usb3} = $device->{usb3};
4303 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4306 sub qemu_cpu_hotplug {
4307 my ($vmid, $conf, $vcpus) = @_;
4309 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4312 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4313 $sockets = $conf->{sockets} if $conf->{sockets};
4314 my $cores = $conf->{cores} || 1;
4315 my $maxcpus = $sockets * $cores;
4317 $vcpus = $maxcpus if !$vcpus;
4319 die "you can
't add more vcpus than maxcpus\n"
4320 if $vcpus > $maxcpus;
4322 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4324 if ($vcpus < $currentvcpus) {
4326 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4328 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4329 qemu_devicedel($vmid, "cpu$i");
4331 my $currentrunningvcpus = undef;
4333 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4334 last if scalar(@{$currentrunningvcpus}) == $i-1;
4335 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4339 #update conf after each succesfull cpu unplug
4340 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4341 PVE::QemuConfig->write_config($vmid, $conf);
4344 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4350 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4351 die "vcpus in running vm does not match its configuration\n"
4352 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4354 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4356 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4357 my $cpustr = print_cpu_device($conf, $i);
4358 qemu_deviceadd($vmid, $cpustr);
4361 my $currentrunningvcpus = undef;
4363 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4364 last if scalar(@{$currentrunningvcpus}) == $i;
4365 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4369 #update conf after each succesfull cpu hotplug
4370 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4371 PVE::QemuConfig->write_config($vmid, $conf);
4375 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4376 mon_cmd($vmid, "cpu-add", id => int($i));
4381 sub qemu_block_set_io_throttle {
4382 my ($vmid, $deviceid,
4383 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4384 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4385 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4386 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4388 return if !check_running($vmid) ;
4390 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4392 bps_rd => int($bps_rd),
4393 bps_wr => int($bps_wr),
4395 iops_rd => int($iops_rd),
4396 iops_wr => int($iops_wr),
4397 bps_max => int($bps_max),
4398 bps_rd_max => int($bps_rd_max),
4399 bps_wr_max => int($bps_wr_max),
4400 iops_max => int($iops_max),
4401 iops_rd_max => int($iops_rd_max),
4402 iops_wr_max => int($iops_wr_max),
4403 bps_max_length => int($bps_max_length),
4404 bps_rd_max_length => int($bps_rd_max_length),
4405 bps_wr_max_length => int($bps_wr_max_length),
4406 iops_max_length => int($iops_max_length),
4407 iops_rd_max_length => int($iops_rd_max_length),
4408 iops_wr_max_length => int($iops_wr_max_length),
4413 sub qemu_block_resize {
4414 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4416 my $running = check_running($vmid);
4418 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4420 return if !$running;
4422 my $padding = (1024 - $size % 1024) % 1024;
4423 $size = $size + $padding;
4428 device => $deviceid,
4434 sub qemu_volume_snapshot {
4435 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4437 my $running = check_running($vmid);
4439 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4440 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4442 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4446 sub qemu_volume_snapshot_delete {
4447 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4449 my $running = check_running($vmid);
4454 my $conf = PVE::QemuConfig->load_config($vmid);
4455 PVE::QemuConfig->foreach_volume($conf, sub {
4456 my ($ds, $drive) = @_;
4457 $running = 1 if $drive->{file} eq $volid;
4461 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4462 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4464 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4468 sub set_migration_caps {
4469 my ($vmid, $savevm) = @_;
4471 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4473 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4474 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4479 "auto-converge" => 1,
4481 "x-rdma-pin-all" => 0,
4484 "dirty-bitmaps" => $dirty_bitmaps,
4487 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4489 for my $supported_capability (@$supported_capabilities) {
4491 capability => $supported_capability->{capability},
4492 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4496 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4500 my ($conf, $func, @param) = @_;
4504 my $test_volid = sub {
4505 my ($key, $drive, $snapname) = @_;
4507 my $volid = $drive->{file};
4510 $volhash->{$volid}->{cdrom} //= 1;
4511 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4513 my $replicate = $drive->{replicate} // 1;
4514 $volhash->{$volid}->{replicate} //= 0;
4515 $volhash->{$volid}->{replicate} = 1 if $replicate;
4517 $volhash->{$volid}->{shared} //= 0;
4518 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4520 $volhash->{$volid}->{referenced_in_config} //= 0;
4521 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4523 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4524 if defined($snapname);
4526 my $size = $drive->{size};
4527 $volhash->{$volid}->{size} //= $size if $size;
4529 $volhash->{$volid}->{is_vmstate} //= 0;
4530 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4532 $volhash->{$volid}->{is_unused} //= 0;
4533 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4535 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4538 my $include_opts = {
4539 extra_keys => ['vmstate
'],
4540 include_unused => 1,
4543 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4544 foreach my $snapname (keys %{$conf->{snapshots}}) {
4545 my $snap = $conf->{snapshots}->{$snapname};
4546 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4549 foreach my $volid (keys %$volhash) {
4550 &$func($volid, $volhash->{$volid}, @param);
4554 my $fast_plug_option = {
4562 'vmstatestorage
' => 1,
4567 # hotplug changes in [PENDING]
4568 # $selection hash can be used to only apply specified options, for
4569 # example: { cores => 1 } (only apply changed 'cores
')
4570 # $errors ref is used to return error messages
4571 sub vmconfig_hotplug_pending {
4572 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4574 my $defaults = load_defaults();
4575 my $arch = get_vm_arch($conf);
4576 my $machine_type = get_vm_machine($conf, undef, $arch);
4578 # commit values which do not have any impact on running VM first
4579 # Note: those option cannot raise errors, we we do not care about
4580 # $selection and always apply them.
4582 my $add_error = sub {
4583 my ($opt, $msg) = @_;
4584 $errors->{$opt} = "hotplug problem - $msg";
4588 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4589 if ($fast_plug_option->{$opt}) {
4590 $conf->{$opt} = $conf->{pending}->{$opt};
4591 delete $conf->{pending}->{$opt};
4597 PVE::QemuConfig->write_config($vmid, $conf);
4600 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4602 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4603 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4604 foreach my $opt (sort keys %$pending_delete_hash) {
4605 next if $selection && !$selection->{$opt};
4606 my $force = $pending_delete_hash->{$opt}->{force};
4608 if ($opt eq 'hotplug
') {
4609 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4610 } elsif ($opt eq 'tablet
') {
4611 die "skip\n" if !$hotplug_features->{usb};
4612 if ($defaults->{tablet}) {
4613 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4614 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4615 if $arch eq 'aarch64
';
4617 vm_deviceunplug($vmid, $conf, 'tablet
');
4618 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4620 } elsif ($opt =~ m/^usb\d+/) {
4622 # since we cannot reliably hot unplug usb devices we are disabling it
4623 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4624 #vm_deviceunplug($vmid, $conf, $opt);
4625 } elsif ($opt eq 'vcpus
') {
4626 die "skip\n" if !$hotplug_features->{cpu};
4627 qemu_cpu_hotplug($vmid, $conf, undef);
4628 } elsif ($opt eq 'balloon
') {
4629 # enable balloon device is not hotpluggable
4630 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4631 # here we reset the ballooning value to memory
4632 my $balloon = $conf->{memory} || $defaults->{memory};
4633 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4634 } elsif ($fast_plug_option->{$opt}) {
4636 } elsif ($opt =~ m/^net(\d+)$/) {
4637 die "skip\n" if !$hotplug_features->{network};
4638 vm_deviceunplug($vmid, $conf, $opt);
4639 } elsif (is_valid_drivename($opt)) {
4640 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4641 vm_deviceunplug($vmid, $conf, $opt);
4642 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4643 } elsif ($opt =~ m/^memory$/) {
4644 die "skip\n" if !$hotplug_features->{memory};
4645 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4646 } elsif ($opt eq 'cpuunits
') {
4647 $cgroup->change_cpu_shares(undef, 1024);
4648 } elsif ($opt eq 'cpulimit
') {
4649 $cgroup->change_cpu_quota(-1, 100000);
4655 &$add_error($opt, $err) if $err ne "skip\n";
4657 delete $conf->{$opt};
4658 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4662 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4663 $apply_pending_cloudinit = sub {
4664 return if $apply_pending_cloudinit_done; # once is enough
4665 $apply_pending_cloudinit_done = 1; # once is enough
4667 my ($key, $value) = @_;
4669 my @cloudinit_opts = keys %$confdesc_cloudinit;
4670 foreach my $opt (keys %{$conf->{pending}}) {
4671 next if !grep { $_ eq $opt } @cloudinit_opts;
4672 $conf->{$opt} = delete $conf->{pending}->{$opt};
4675 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4676 foreach my $opt (sort keys %$pending_delete_hash) {
4677 next if !grep { $_ eq $opt } @cloudinit_opts;
4678 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4679 delete $conf->{$opt};
4682 my $new_conf = { %$conf };
4683 $new_conf->{$key} = $value;
4684 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4687 foreach my $opt (keys %{$conf->{pending}}) {
4688 next if $selection && !$selection->{$opt};
4689 my $value = $conf->{pending}->{$opt};
4691 if ($opt eq 'hotplug
') {
4692 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4693 } elsif ($opt eq 'tablet
') {
4694 die "skip\n" if !$hotplug_features->{usb};
4696 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4697 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4698 if $arch eq 'aarch64
';
4699 } elsif ($value == 0) {
4700 vm_deviceunplug($vmid, $conf, 'tablet
');
4701 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4703 } elsif ($opt =~ m/^usb\d+$/) {
4705 # since we cannot reliably hot unplug usb devices we disable it for now
4706 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4707 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4708 #die "skip\n" if !$d;
4709 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4710 } elsif ($opt eq 'vcpus
') {
4711 die "skip\n" if !$hotplug_features->{cpu};
4712 qemu_cpu_hotplug($vmid, $conf, $value);
4713 } elsif ($opt eq 'balloon
') {
4714 # enable/disable balloning device is not hotpluggable
4715 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4716 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4717 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4719 # allow manual ballooning if shares is set to zero
4720 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4721 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4722 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4724 } elsif ($opt =~ m/^net(\d+)$/) {
4725 # some changes can be done without hotplug
4726 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4727 $vmid, $opt, $value, $arch, $machine_type);
4728 } elsif (is_valid_drivename($opt)) {
4729 die "skip\n" if $opt eq 'efidisk0
';
4730 # some changes can be done without hotplug
4731 my $drive = parse_drive($opt, $value);
4732 if (drive_is_cloudinit($drive)) {
4733 &$apply_pending_cloudinit($opt, $value);
4735 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4736 $vmid, $opt, $value, $arch, $machine_type);
4737 } elsif ($opt =~ m/^memory$/) { #dimms
4738 die "skip\n" if !$hotplug_features->{memory};
4739 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4740 } elsif ($opt eq 'cpuunits
') {
4741 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4742 } elsif ($opt eq 'cpulimit
') {
4743 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4744 $cgroup->change_cpu_quota($cpulimit, 100000);
4746 die "skip\n"; # skip non-hot-pluggable options
4750 &$add_error($opt, $err) if $err ne "skip\n";
4752 $conf->{$opt} = $value;
4753 delete $conf->{pending}->{$opt};
4757 PVE::QemuConfig->write_config($vmid, $conf);
4760 sub try_deallocate_drive {
4761 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4763 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4764 my $volid = $drive->{file};
4765 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4766 my $sid = PVE::Storage::parse_volume_id($volid);
4767 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4769 # check if the disk is really unused
4770 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4771 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4772 PVE::Storage::vdisk_free($storecfg, $volid);
4775 # If vm is not owner of this disk remove from config
4783 sub vmconfig_delete_or_detach_drive {
4784 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4786 my $drive = parse_drive($opt, $conf->{$opt});
4788 my $rpcenv = PVE::RPCEnvironment::get();
4789 my $authuser = $rpcenv->get_user();
4792 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4793 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4795 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4801 sub vmconfig_apply_pending {
4802 my ($vmid, $conf, $storecfg, $errors) = @_;
4804 my $add_apply_error = sub {
4805 my ($opt, $msg) = @_;
4806 my $err_msg = "unable to apply pending change $opt : $msg";
4807 $errors->{$opt} = $err_msg;
4813 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4814 foreach my $opt (sort keys %$pending_delete_hash) {
4815 my $force = $pending_delete_hash->{$opt}->{force};
4817 if ($opt =~ m/^unused/) {
4818 die "internal error";
4819 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4820 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4824 $add_apply_error->($opt, $err);
4826 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4827 delete $conf->{$opt};
4831 PVE::QemuConfig->cleanup_pending($conf);
4833 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4834 next if $opt eq 'delete'; # just to be sure
4836 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4837 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4841 $add_apply_error->($opt, $err);
4843 $conf->{$opt} = delete $conf->{pending}->{$opt};
4847 # write all changes at once to avoid unnecessary i/o
4848 PVE::QemuConfig->write_config($vmid, $conf);
4851 sub vmconfig_update_net {
4852 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4854 my $newnet = parse_net($value);
4856 if ($conf->{$opt}) {
4857 my $oldnet = parse_net($conf->{$opt});
4859 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4860 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4861 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4862 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4864 # for non online change, we try to hot-unplug
4865 die "skip\n" if !$hotplug;
4866 vm_deviceunplug($vmid, $conf, $opt);
4869 die "internal error" if $opt !~ m/net(\d+)/;
4870 my $iface = "tap${vmid}i$1";
4872 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4873 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4874 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4875 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4876 PVE::Network::tap_unplug($iface);
4879 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4881 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4883 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4884 # Rate can be applied on its own but any change above needs to
4885 # include the rate in tap_plug since OVS resets everything.
4886 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4889 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4890 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4898 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4904 sub vmconfig_update_disk {
4905 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4907 my $drive = parse_drive($opt, $value);
4909 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4910 my $media = $drive->{media} || 'disk
';
4911 my $oldmedia = $old_drive->{media} || 'disk
';
4912 die "unable to change media type\n" if $media ne $oldmedia;
4914 if (!drive_is_cdrom($old_drive)) {
4916 if ($drive->{file} ne $old_drive->{file}) {
4918 die "skip\n" if !$hotplug;
4920 # unplug and register as unused
4921 vm_deviceunplug($vmid, $conf, $opt);
4922 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4925 # update existing disk
4927 # skip non hotpluggable value
4928 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4929 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4930 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4931 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4932 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4937 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4938 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4939 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4940 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4941 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4942 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4943 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4944 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4945 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4946 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4947 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4948 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4949 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4950 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4951 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4952 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4953 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4954 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4956 qemu_block_set_io_throttle(
4958 ($drive->{mbps} || 0)*1024*1024,
4959 ($drive->{mbps_rd} || 0)*1024*1024,
4960 ($drive->{mbps_wr} || 0)*1024*1024,
4961 $drive->{iops} || 0,
4962 $drive->{iops_rd} || 0,
4963 $drive->{iops_wr} || 0,
4964 ($drive->{mbps_max} || 0)*1024*1024,
4965 ($drive->{mbps_rd_max} || 0)*1024*1024,
4966 ($drive->{mbps_wr_max} || 0)*1024*1024,
4967 $drive->{iops_max} || 0,
4968 $drive->{iops_rd_max} || 0,
4969 $drive->{iops_wr_max} || 0,
4970 $drive->{bps_max_length} || 1,
4971 $drive->{bps_rd_max_length} || 1,
4972 $drive->{bps_wr_max_length} || 1,
4973 $drive->{iops_max_length} || 1,
4974 $drive->{iops_rd_max_length} || 1,
4975 $drive->{iops_wr_max_length} || 1,
4985 if ($drive->{file} eq 'none
') {
4986 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4987 if (drive_is_cloudinit($old_drive)) {
4988 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4991 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4993 # force eject if locked
4994 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4997 mon_cmd($vmid, "blockdev-change-medium",
4998 id => "$opt", filename => "$path");
5006 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5008 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5009 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5012 # called in locked context by incoming migration
5013 sub vm_migrate_get_nbd_disks {
5014 my ($storecfg, $conf, $replicated_volumes) = @_;
5016 my $local_volumes = {};
5017 PVE::QemuConfig->foreach_volume($conf, sub {
5018 my ($ds, $drive) = @_;
5020 return if drive_is_cdrom($drive);
5022 my $volid = $drive->{file};
5026 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5028 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5029 return if $scfg->{shared};
5031 # replicated disks re-use existing state via bitmap
5032 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5033 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5035 return $local_volumes;
5038 # called in locked context by incoming migration
5039 sub vm_migrate_alloc_nbd_disks {
5040 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5045 foreach my $opt (sort keys %$source_volumes) {
5046 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5048 if ($use_existing) {
5049 $nbd->{$opt}->{drivestr} = print_drive($drive);
5050 $nbd->{$opt}->{volid} = $volid;
5051 $nbd->{$opt}->{replicated} = 1;
5055 # If a remote storage is specified and the format of the original
5056 # volume is not available there, fall back to the default format.
5057 # Otherwise use the same format as the original.
5058 if (!$storagemap->{identity}) {
5059 $storeid = map_storage($storagemap, $storeid);
5060 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5061 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5062 my $fileFormat = qemu_img_format($scfg, $volname);
5063 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5065 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5066 $format = qemu_img_format($scfg, $volname);
5069 my $size = $drive->{size} / 1024;
5070 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5071 my $newdrive = $drive;
5072 $newdrive->{format} = $format;
5073 $newdrive->{file} = $newvolid;
5074 my $drivestr = print_drive($newdrive);
5075 $nbd->{$opt}->{drivestr} = $drivestr;
5076 $nbd->{$opt}->{volid} = $newvolid;
5082 # see vm_start_nolock for parameters, additionally:
5084 # storagemap = parsed storage map for allocating NBD disks
5086 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5088 return PVE::QemuConfig->lock_config($vmid, sub {
5089 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5091 die "you can't start a vm
if it
's a template\n"
5092 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5094 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended
');
5095 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup
');
5097 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5099 if ($has_backup_lock && $running) {
5100 # a backup is currently running, attempt to start the guest in the
5101 # existing QEMU instance
5102 return vm_resume($vmid);
5105 PVE::QemuConfig->check_lock($conf)
5106 if !($params->{skiplock} || $has_suspended_lock);
5108 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5110 die "VM $vmid already running\n" if $running;
5112 if (my $storagemap = $migrate_opts->{storagemap}) {
5113 my $replicated = $migrate_opts->{replicated_volumes};
5114 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5115 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5117 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5118 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5122 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5128 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
5129 # skiplock => 0/1, skip checking for config lock
5130 # skiptemplate => 0/1, skip checking whether VM is template
5131 # forcemachine => to force Qemu machine (rollback/migration)
5132 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
5133 # timeout => in seconds
5134 # paused => start VM in paused state (backup)
5135 # resume => resume from hibernation
5146 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5147 # migratedfrom => source node
5148 # spice_ticket => used for spice migration, passed via tunnel/stdin
5149 # network => CIDR of migration network
5150 # type => secure/insecure - tunnel over encrypted connection or plain-text
5151 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5152 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5153 sub vm_start_nolock {
5154 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5156 my $statefile = $params->{statefile};
5157 my $resume = $params->{resume};
5159 my $migratedfrom = $migrate_opts->{migratedfrom};
5160 my $migration_type = $migrate_opts->{type};
5164 # clean up leftover reboot request files
5165 eval { clear_reboot_request($vmid); };
5168 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5169 vmconfig_apply_pending($vmid, $conf, $storecfg);
5170 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5173 # don't regenerate the ISO
if the VM
is started as part of a live migration
5174 # this way we can reuse the old ISO with the correct config
5175 PVE
::QemuServer
::Cloudinit
::generate_cloudinitconfig
($conf, $vmid) if !$migratedfrom;
5177 my $defaults = load_defaults
();
5179 # set environment variable useful inside network script
5180 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom if $migratedfrom;
5182 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5184 my $forcemachine = $params->{forcemachine
};
5185 my $forcecpu = $params->{forcecpu
};
5187 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5188 $forcemachine = $conf->{runningmachine
};
5189 $forcecpu = $conf->{runningcpu
};
5190 print "Resuming suspended VM\n";
5193 my ($cmd, $vollist, $spice_port) = config_to_command
($storecfg, $vmid,
5194 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5197 my $get_migration_ip = sub {
5198 my ($nodename) = @_;
5200 return $migration_ip if defined($migration_ip);
5202 my $cidr = $migrate_opts->{network
};
5204 if (!defined($cidr)) {
5205 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5206 $cidr = $dc_conf->{migration
}->{network
};
5209 if (defined($cidr)) {
5210 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5212 die "could not get IP: no address configured on local " .
5213 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5215 die "could not get IP: multiple addresses configured on local " .
5216 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5218 $migration_ip = @$ips[0];
5221 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5222 if !defined($migration_ip);
5224 return $migration_ip;
5229 if ($statefile eq 'tcp') {
5230 my $localip = "localhost";
5231 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5232 my $nodename = nodename
();
5234 if (!defined($migration_type)) {
5235 if (defined($datacenterconf->{migration
}->{type
})) {
5236 $migration_type = $datacenterconf->{migration
}->{type
};
5238 $migration_type = 'secure';
5242 if ($migration_type eq 'insecure') {
5243 $localip = $get_migration_ip->($nodename);
5244 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5247 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5248 my $migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5249 $migrate_uri = "tcp:${localip}:${migrate_port}";
5250 push @$cmd, '-incoming', $migrate_uri;
5253 } elsif ($statefile eq 'unix') {
5254 # should be default for secure migrations as a ssh TCP forward
5255 # tunnel is not deterministic reliable ready and fails regurarly
5256 # to set up in time, so use UNIX socket forwards
5257 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5258 unlink $socket_addr;
5260 $migrate_uri = "unix:$socket_addr";
5262 push @$cmd, '-incoming', $migrate_uri;
5265 } elsif (-e
$statefile) {
5266 push @$cmd, '-loadstate', $statefile;
5268 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5269 push @$vollist, $statefile;
5270 push @$cmd, '-loadstate', $statepath;
5272 } elsif ($params->{paused
}) {
5277 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
5278 my $d = parse_hostpci
($conf->{"hostpci$i"});
5280 my $pcidevices = $d->{pciid
};
5281 foreach my $pcidevice (@$pcidevices) {
5282 my $pciid = $pcidevice->{id
};
5284 my $info = PVE
::SysFSTools
::pci_device_info
("$pciid");
5285 die "IOMMU not present\n" if !PVE
::SysFSTools
::check_iommu_support
();
5286 die "no pci device info for device '$pciid'\n" if !$info;
5289 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $i);
5290 PVE
::SysFSTools
::pci_create_mdev_device
($pciid, $uuid, $d->{mdev
});
5292 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5293 if !PVE
::SysFSTools
::pci_dev_group_bind_to_vfio
($pciid);
5294 die "can't reset PCI device '$pciid'\n"
5295 if $info->{has_fl_reset
} && !PVE
::SysFSTools
::pci_dev_reset
($info);
5300 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5303 run_command
(['/bin/systemctl', 'stop', "$vmid.scope"],
5304 outfunc
=> sub {}, errfunc
=> sub {});
5306 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5307 # timeout should be more than enough here...
5308 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 5);
5310 my $cpuunits = get_cpuunits
($conf);
5312 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $resume);
5314 timeout
=> $statefile ?
undef : $start_timeout,
5319 # when migrating, prefix QEMU output so other side can pick up any
5320 # errors that might occur and show the user
5321 if ($migratedfrom) {
5322 $run_params{quiet
} = 1;
5323 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5327 Slice
=> 'qemu.slice',
5328 KillMode
=> 'process',
5330 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5333 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5334 $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5335 $properties{CPUWeight
} = $cpuunits;
5337 $properties{CPUShares
} = $cpuunits;
5340 if (my $cpulimit = $conf->{cpulimit
}) {
5341 $properties{CPUQuota
} = int($cpulimit * 100);
5343 $properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5345 my $run_qemu = sub {
5346 PVE
::Tools
::run_fork
sub {
5347 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %properties);
5349 my $exitcode = run_command
($cmd, %run_params);
5350 die "QEMU exited with code $exitcode\n" if $exitcode;
5354 if ($conf->{hugepages
}) {
5357 my $hugepages_topology = PVE
::QemuServer
::Memory
::hugepages_topology
($conf);
5358 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5360 PVE
::QemuServer
::Memory
::hugepages_mount
();
5361 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5363 eval { $run_qemu->() };
5365 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5366 if !$conf->{keephugepages
};
5370 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5371 if !$conf->{keephugepages
};
5373 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5376 eval { $run_qemu->() };
5380 # deactivate volumes if start fails
5381 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5382 die "start failed: $err";
5385 print "migration listens on $migrate_uri\n" if $migrate_uri;
5386 $res->{migrate_uri
} = $migrate_uri;
5388 if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
5389 eval { mon_cmd
($vmid, "cont"); };
5393 #start nbd server for storage migration
5394 if (my $nbd = $migrate_opts->{nbd
}) {
5395 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5397 my $migrate_storage_uri;
5398 # nbd_protocol_version > 0 for unix socket support
5399 if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5400 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5401 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
5402 $migrate_storage_uri = "nbd:unix:$socket_path";
5404 my $nodename = nodename
();
5405 my $localip = $get_migration_ip->($nodename);
5406 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5407 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5409 mon_cmd
($vmid, "nbd-server-start", addr
=> {
5412 host
=> "${localip}",
5413 port
=> "${storage_migrate_port}",
5416 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5417 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5420 $res->{migrate_storage_uri
} = $migrate_storage_uri;
5422 foreach my $opt (sort keys %$nbd) {
5423 my $drivestr = $nbd->{$opt}->{drivestr
};
5424 my $volid = $nbd->{$opt}->{volid
};
5425 mon_cmd
($vmid, "nbd-server-add", device
=> "drive-$opt", writable
=> JSON
::true
);
5426 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5427 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5428 print "re-using replicated volume: $opt - $volid\n"
5429 if $nbd->{$opt}->{replicated
};
5431 $res->{drives
}->{$opt} = $nbd->{$opt};
5432 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
5436 if ($migratedfrom) {
5438 set_migration_caps
($vmid);
5443 print "spice listens on port $spice_port\n";
5444 $res->{spice_port
} = $spice_port;
5445 if ($migrate_opts->{spice_ticket
}) {
5446 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
5447 $migrate_opts->{spice_ticket
});
5448 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
5453 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
5454 if !$statefile && $conf->{balloon
};
5456 foreach my $opt (keys %$conf) {
5457 next if $opt !~ m/^net\d+$/;
5458 my $nicconf = parse_net
($conf->{$opt});
5459 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
5463 mon_cmd
($vmid, 'qom-set',
5464 path
=> "machine/peripheral/balloon0",
5465 property
=> "guest-stats-polling-interval",
5466 value
=> 2) if (!defined($conf->{balloon
}) || $conf->{balloon
});
5469 print "Resumed VM, removing state\n";
5470 if (my $vmstate = $conf->{vmstate
}) {
5471 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5472 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5474 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5475 PVE
::QemuConfig-
>write_config($vmid, $conf);
5478 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5483 sub vm_commandline
{
5484 my ($storecfg, $vmid, $snapname) = @_;
5486 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5491 my $snapshot = $conf->{snapshots
}->{$snapname};
5492 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5494 # check for machine or CPU overrides in snapshot
5495 $forcemachine = $snapshot->{runningmachine
};
5496 $forcecpu = $snapshot->{runningcpu
};
5498 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5503 my $defaults = load_defaults
();
5505 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults,
5506 $forcemachine, $forcecpu);
5508 return PVE
::Tools
::cmd2string
($cmd);
5512 my ($vmid, $skiplock) = @_;
5514 PVE
::QemuConfig-
>lock_config($vmid, sub {
5516 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5518 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5520 mon_cmd
($vmid, "system_reset");
5524 sub get_vm_volumes
{
5528 foreach_volid
($conf, sub {
5529 my ($volid, $attr) = @_;
5531 return if $volid =~ m
|^/|;
5533 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5536 push @$vollist, $volid;
5542 sub vm_stop_cleanup
{
5543 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5548 my $vollist = get_vm_volumes
($conf);
5549 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5552 foreach my $ext (qw(mon qmp pid vnc qga)) {
5553 unlink "/var/run/qemu-server/${vmid}.$ext";
5556 if ($conf->{ivshmem
}) {
5557 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5558 # just delete it for now, VMs which have this already open do not
5559 # are affected, but new VMs will get a separated one. If this
5560 # becomes an issue we either add some sort of ref-counting or just
5561 # add a "don't delete on stop" flag to the ivshmem format.
5562 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5565 foreach my $key (keys %$conf) {
5566 next if $key !~ m/^hostpci(\d+)$/;
5567 my $hostpciindex = $1;
5568 my $d = parse_hostpci
($conf->{$key});
5569 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5571 foreach my $pci (@{$d->{pciid
}}) {
5572 my $pciid = $pci->{id
};
5573 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5577 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5579 warn $@ if $@; # avoid errors - just warn
5582 # call only in locked context
5584 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5586 my $pid = check_running
($vmid, $nocheck);
5591 $conf = PVE
::QemuConfig-
>load_config($vmid);
5592 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5593 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5594 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5595 $timeout = $opts->{down
} if $opts->{down
};
5597 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5602 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
5603 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5605 mon_cmd
($vmid, "system_powerdown");
5608 mon_cmd
($vmid, "quit");
5614 $timeout = 60 if !defined($timeout);
5617 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5622 if ($count >= $timeout) {
5624 warn "VM still running - terminating now with SIGTERM\n";
5627 die "VM quit/powerdown failed - got timeout\n";
5630 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5634 if (!check_running
($vmid, $nocheck)) {
5635 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5639 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5642 die "VM quit/powerdown failed\n";
5650 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5655 if ($count >= $timeout) {
5656 warn "VM still running - terminating now with SIGKILL\n";
5661 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5664 # Note: use $nocheck to skip tests if VM configuration file exists.
5665 # We need that when migration VMs to other nodes (files already moved)
5666 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5668 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5670 $force = 1 if !defined($force) && !$shutdown;
5673 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5674 kill 15, $pid if $pid;
5675 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5676 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5680 PVE
::QemuConfig-
>lock_config($vmid, sub {
5681 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5686 my ($vmid, $timeout) = @_;
5688 PVE
::QemuConfig-
>lock_config($vmid, sub {
5691 # only reboot if running, as qmeventd starts it again on a stop event
5692 return if !check_running
($vmid);
5694 create_reboot_request
($vmid);
5696 my $storecfg = PVE
::Storage
::config
();
5697 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5701 # avoid that the next normal shutdown will be confused for a reboot
5702 clear_reboot_request
($vmid);
5708 # note: if using the statestorage parameter, the caller has to check privileges
5710 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5717 PVE
::QemuConfig-
>lock_config($vmid, sub {
5719 $conf = PVE
::QemuConfig-
>load_config($vmid);
5721 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5722 PVE
::QemuConfig-
>check_lock($conf)
5723 if !($skiplock || $is_backing_up);
5725 die "cannot suspend to disk during backup\n"
5726 if $is_backing_up && $includestate;
5728 if ($includestate) {
5729 $conf->{lock} = 'suspending';
5730 my $date = strftime
("%Y-%m-%d", localtime(time()));
5731 $storecfg = PVE
::Storage
::config
();
5732 if (!$statestorage) {
5733 $statestorage = find_vmstate_storage
($conf, $storecfg);
5734 # check permissions for the storage
5735 my $rpcenv = PVE
::RPCEnvironment
::get
();
5736 if ($rpcenv->{type
} ne 'cli') {
5737 my $authuser = $rpcenv->get_user();
5738 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5743 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
5744 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5745 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5746 PVE
::QemuConfig-
>write_config($vmid, $conf);
5748 mon_cmd
($vmid, "stop");
5752 if ($includestate) {
5754 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5757 set_migration_caps
($vmid, 1);
5758 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5760 my $state = mon_cmd
($vmid, "query-savevm");
5761 if (!$state->{status
}) {
5762 die "savevm not active\n";
5763 } elsif ($state->{status
} eq 'active') {
5766 } elsif ($state->{status
} eq 'completed') {
5767 print "State saved, quitting\n";
5769 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5770 die "query-savevm failed with error '$state->{error}'\n"
5772 die "query-savevm returned status '$state->{status}'\n";
5778 PVE
::QemuConfig-
>lock_config($vmid, sub {
5779 $conf = PVE
::QemuConfig-
>load_config($vmid);
5781 # cleanup, but leave suspending lock, to indicate something went wrong
5783 mon_cmd
($vmid, "savevm-end");
5784 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5785 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5786 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5787 PVE
::QemuConfig-
>write_config($vmid, $conf);
5793 die "lock changed unexpectedly\n"
5794 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5796 mon_cmd
($vmid, "quit");
5797 $conf->{lock} = 'suspended';
5798 PVE
::QemuConfig-
>write_config($vmid, $conf);
5804 my ($vmid, $skiplock, $nocheck) = @_;
5806 PVE
::QemuConfig-
>lock_config($vmid, sub {
5807 my $res = mon_cmd
($vmid, 'query-status');
5808 my $resume_cmd = 'cont';
5811 if ($res->{status
}) {
5812 return if $res->{status
} eq 'running'; # job done, go home
5813 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
5814 $reset = 1 if $res->{status
} eq 'shutdown';
5819 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5821 PVE
::QemuConfig-
>check_lock($conf)
5822 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5826 # required if a VM shuts down during a backup and we get a resume
5827 # request before the backup finishes for example
5828 mon_cmd
($vmid, "system_reset");
5830 mon_cmd
($vmid, $resume_cmd);
5835 my ($vmid, $skiplock, $key) = @_;
5837 PVE
::QemuConfig-
>lock_config($vmid, sub {
5839 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5841 # there is no qmp command, so we use the human monitor command
5842 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5843 die $res if $res ne '';
5847 # vzdump restore implementaion
5849 sub tar_archive_read_firstfile
{
5850 my $archive = shift;
5852 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5854 # try to detect archive type first
5855 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5856 die "unable to open file '$archive'\n";
5857 my $firstfile = <$fh>;
5861 die "ERROR: archive contaions no data\n" if !$firstfile;
5867 sub tar_restore_cleanup
{
5868 my ($storecfg, $statfile) = @_;
5870 print STDERR
"starting cleanup\n";
5872 if (my $fd = IO
::File-
>new($statfile, "r")) {
5873 while (defined(my $line = <$fd>)) {
5874 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5877 if ($volid =~ m
|^/|) {
5878 unlink $volid || die 'unlink failed\n';
5880 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5882 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5884 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5886 print STDERR
"unable to parse line in statfile - $line";
5893 sub restore_file_archive
{
5894 my ($archive, $vmid, $user, $opts) = @_;
5896 return restore_vma_archive
($archive, $vmid, $user, $opts)
5899 my $info = PVE
::Storage
::archive_info
($archive);
5900 my $format = $opts->{format
} // $info->{format
};
5901 my $comp = $info->{compression
};
5903 # try to detect archive format
5904 if ($format eq 'tar') {
5905 return restore_tar_archive
($archive, $vmid, $user, $opts);
5907 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5911 # hepler to remove disks that will not be used after restore
5912 my $restore_cleanup_oldconf = sub {
5913 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5915 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
5916 my ($ds, $drive) = @_;
5918 return if drive_is_cdrom
($drive, 1);
5920 my $volid = $drive->{file
};
5921 return if !$volid || $volid =~ m
|^/|;
5923 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
5924 return if !$path || !$owner || ($owner != $vmid);
5926 # Note: only delete disk we want to restore
5927 # other volumes will become unused
5928 if ($virtdev_hash->{$ds}) {
5929 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
5936 # delete vmstate files, after the restore we have no snapshots anymore
5937 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5938 my $snap = $oldconf->{snapshots
}->{$snapname};
5939 if ($snap->{vmstate
}) {
5940 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
5948 # Helper to parse vzdump backup device hints
5950 # $rpcenv: Environment, used to ckeck storage permissions
5951 # $user: User ID, to check storage permissions
5952 # $storecfg: Storage configuration
5953 # $fh: the file handle for reading the configuration
5954 # $devinfo: should contain device sizes for all backu-up'ed devices
5955 # $options: backup options (pool, default storage)
5957 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5958 my $parse_backup_hints = sub {
5959 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5961 my $virtdev_hash = {};
5963 while (defined(my $line = <$fh>)) {
5964 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5965 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5966 die "archive does not contain data for drive '$virtdev'\n"
5967 if !$devinfo->{$devname};
5969 if (defined($options->{storage
})) {
5970 $storeid = $options->{storage
} || 'local';
5971 } elsif (!$storeid) {
5974 $format = 'raw' if !$format;
5975 $devinfo->{$devname}->{devname
} = $devname;
5976 $devinfo->{$devname}->{virtdev
} = $virtdev;
5977 $devinfo->{$devname}->{format
} = $format;
5978 $devinfo->{$devname}->{storeid
} = $storeid;
5980 # check permission on storage
5981 my $pool = $options->{pool
}; # todo: do we need that?
5982 if ($user ne 'root@pam') {
5983 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5986 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5987 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5989 my $drive = parse_drive
($virtdev, $2);
5990 if (drive_is_cloudinit
($drive)) {
5991 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
5992 $storeid = $options->{storage
} if defined ($options->{storage
});
5993 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5994 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
5996 $virtdev_hash->{$virtdev} = {
5998 storeid
=> $storeid,
5999 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6006 return $virtdev_hash;
6009 # Helper to allocate and activate all volumes required for a restore
6011 # $storecfg: Storage configuration
6012 # $virtdev_hash: as returned by parse_backup_hints()
6014 # Returns: { $virtdev => $volid }
6015 my $restore_allocate_devices = sub {
6016 my ($storecfg, $virtdev_hash, $vmid) = @_;
6019 foreach my $virtdev (sort keys %$virtdev_hash) {
6020 my $d = $virtdev_hash->{$virtdev};
6021 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6022 my $storeid = $d->{storeid
};
6023 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6025 # test if requested format is supported
6026 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6027 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6028 $d->{format
} = $defFormat if !$supported;
6031 if ($d->{is_cloudinit
}) {
6032 $name = "vm-$vmid-cloudinit";
6033 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6034 if ($scfg->{path
}) {
6035 $name .= ".$d->{format}";
6039 my $volid = PVE
::Storage
::vdisk_alloc
(
6040 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6042 print STDERR
"new volume ID is '$volid'\n";
6043 $d->{volid
} = $volid;
6045 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6047 $map->{$virtdev} = $volid;
6053 sub restore_update_config_line
{
6054 my ($cookie, $map, $line, $unique) = @_;
6056 return '' if $line =~ m/^\#qmdump\#/;
6057 return '' if $line =~ m/^\#vzdump\#/;
6058 return '' if $line =~ m/^lock:/;
6059 return '' if $line =~ m/^unused\d+:/;
6060 return '' if $line =~ m/^parent:/;
6064 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6065 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6066 # try to convert old 1.X settings
6067 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6068 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6069 my ($model, $macaddr) = split(/\=/, $devconfig);
6070 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6073 bridge
=> "vmbr$ind",
6074 macaddr
=> $macaddr,
6076 my $netstr = print_net
($net);
6078 $res .= "net$cookie->{netcount}: $netstr\n";
6079 $cookie->{netcount
}++;
6081 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6082 my ($id, $netstr) = ($1, $2);
6083 my $net = parse_net
($netstr);
6084 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6085 $netstr = print_net
($net);
6086 $res .= "$id: $netstr\n";
6087 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6090 my $di = parse_drive
($virtdev, $value);
6091 if (defined($di->{backup
}) && !$di->{backup
}) {
6093 } elsif ($map->{$virtdev}) {
6094 delete $di->{format
}; # format can change on restore
6095 $di->{file
} = $map->{$virtdev};
6096 $value = print_drive
($di);
6097 $res .= "$virtdev: $value\n";
6101 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6103 if ($vmgenid ne '0') {
6104 # always generate a new vmgenid if there was a valid one setup
6105 $vmgenid = generate_uuid
();
6107 $res .= "vmgenid: $vmgenid\n";
6108 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6109 my ($uuid, $uuid_str);
6110 UUID
::generate
($uuid);
6111 UUID
::unparse
($uuid, $uuid_str);
6112 my $smbios1 = parse_smbios1
($2);
6113 $smbios1->{uuid
} = $uuid_str;
6114 $res .= $1.print_smbios1
($smbios1)."\n";
6122 my $restore_deactivate_volumes = sub {
6123 my ($storecfg, $devinfo) = @_;
6126 foreach my $devname (keys %$devinfo) {
6127 my $volid = $devinfo->{$devname}->{volid
};
6128 push @$vollist, $volid if $volid;
6131 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6134 my $restore_destroy_volumes = sub {
6135 my ($storecfg, $devinfo) = @_;
6137 foreach my $devname (keys %$devinfo) {
6138 my $volid = $devinfo->{$devname}->{volid
};
6141 if ($volid =~ m
|^/|) {
6142 unlink $volid || die 'unlink failed\n';
6144 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6146 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6148 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6153 my ($cfg, $vmid) = @_;
6155 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6157 my $volid_hash = {};
6158 foreach my $storeid (keys %$info) {
6159 foreach my $item (@{$info->{$storeid}}) {
6160 next if !($item->{volid
} && $item->{size
});
6161 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6162 $volid_hash->{$item->{volid
}} = $item;
6169 sub update_disk_config
{
6170 my ($vmid, $conf, $volid_hash) = @_;
6173 my $prefix = "VM $vmid";
6175 # used and unused disks
6176 my $referenced = {};
6178 # Note: it is allowed to define multiple storages with same path (alias), so
6179 # we need to check both 'volid' and real 'path' (two different volid can point
6180 # to the same path).
6182 my $referencedpath = {};
6185 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6186 my ($opt, $drive) = @_;
6188 my $volid = $drive->{file
};
6190 my $volume = $volid_hash->{$volid};
6192 # mark volid as "in-use" for next step
6193 $referenced->{$volid} = 1;
6194 if ($volume && (my $path = $volume->{path
})) {
6195 $referencedpath->{$path} = 1;
6198 return if drive_is_cdrom
($drive);
6201 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6202 if (defined($updated)) {
6204 $conf->{$opt} = print_drive
($updated);
6205 print "$prefix ($opt): $msg\n";
6209 # remove 'unusedX' entry if volume is used
6210 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6211 my ($opt, $drive) = @_;
6213 my $volid = $drive->{file
};
6217 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6218 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6219 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6221 delete $conf->{$opt};
6224 $referenced->{$volid} = 1;
6225 $referencedpath->{$path} = 1 if $path;
6228 foreach my $volid (sort keys %$volid_hash) {
6229 next if $volid =~ m/vm-$vmid-state-/;
6230 next if $referenced->{$volid};
6231 my $path = $volid_hash->{$volid}->{path
};
6232 next if !$path; # just to be sure
6233 next if $referencedpath->{$path};
6235 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6236 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6237 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6244 my ($vmid, $nolock, $dryrun) = @_;
6246 my $cfg = PVE
::Storage
::config
();
6248 print "rescan volumes...\n";
6249 my $volid_hash = scan_volids
($cfg, $vmid);
6251 my $updatefn = sub {
6254 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6256 PVE
::QemuConfig-
>check_lock($conf);
6259 foreach my $volid (keys %$volid_hash) {
6260 my $info = $volid_hash->{$volid};
6261 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6264 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6266 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6269 if (defined($vmid)) {
6273 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6276 my $vmlist = config_list
();
6277 foreach my $vmid (keys %$vmlist) {
6281 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6287 sub restore_proxmox_backup_archive
{
6288 my ($archive, $vmid, $user, $options) = @_;
6290 my $storecfg = PVE
::Storage
::config
();
6292 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6293 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6295 my $fingerprint = $scfg->{fingerprint
};
6296 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6298 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6300 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6301 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6302 local $ENV{PBS_PASSWORD
} = $password;
6303 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6305 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6306 PVE
::Storage
::parse_volname
($storecfg, $archive);
6308 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6310 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6312 my $tmpdir = "/var/tmp/vzdumptmp$$";
6316 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6317 # disable interrupts (always do cleanups)
6321 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6323 # Note: $oldconf is undef if VM does not exists
6324 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6325 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6326 my $new_conf_raw = '';
6328 my $rpcenv = PVE
::RPCEnvironment
::get
();
6337 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6339 my $cfgfn = "$tmpdir/qemu-server.conf";
6340 my $firewall_config_fn = "$tmpdir/fw.conf";
6341 my $index_fn = "$tmpdir/index.json";
6343 my $cmd = "restore";
6345 my $param = [$pbs_backup_name, "index.json", $index_fn];
6346 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6347 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6348 $index = decode_json
($index);
6350 # print Dumper($index);
6351 foreach my $info (@{$index->{files
}}) {
6352 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6354 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6355 $devinfo->{$devname}->{size
} = $1;
6357 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6362 my $is_qemu_server_backup = scalar(
6363 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6365 if (!$is_qemu_server_backup) {
6366 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6368 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6370 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6371 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6373 if ($has_firewall_config) {
6374 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6375 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6377 my $pve_firewall_dir = '/etc/pve/firewall';
6378 mkdir $pve_firewall_dir; # make sure the dir exists
6379 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6382 my $fh = IO
::File-
>new($cfgfn, "r") ||
6383 die "unable to read qemu-server.conf - $!\n";
6385 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6387 # fixme: rate limit?
6389 # create empty/temp config
6390 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6392 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6395 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6397 foreach my $virtdev (sort keys %$virtdev_hash) {
6398 my $d = $virtdev_hash->{$virtdev};
6399 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6401 # this fails if storage is unavailable
6402 my $volid = $d->{volid
};
6403 my $path = PVE
::Storage
::path
($storecfg, $volid);
6405 # for live-restore we only want to preload the efidisk
6406 next if $options->{live
} && $virtdev ne 'efidisk0';
6408 my $pbs_restore_cmd = [
6409 '/usr/bin/pbs-restore',
6410 '--repository', $repo,
6412 "$d->{devname}.img.fidx",
6417 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6418 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6420 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6421 push @$pbs_restore_cmd, '--skip-zero';
6424 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6425 print "restore proxmox backup image: $dbg_cmdstring\n";
6426 run_command
($pbs_restore_cmd);
6429 $fh->seek(0, 0) || die "seek failed - $!\n";
6431 my $cookie = { netcount
=> 0 };
6432 while (defined(my $line = <$fh>)) {
6433 $new_conf_raw .= restore_update_config_line
(
6445 if ($err || !$options->{live
}) {
6446 $restore_deactivate_volumes->($storecfg, $devinfo);
6452 $restore_destroy_volumes->($storecfg, $devinfo);
6456 if ($options->{live
}) {
6457 # keep lock during live-restore
6458 $new_conf_raw .= "\nlock: create";
6461 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6463 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6465 eval { rescan
($vmid, 1); };
6468 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
6470 if ($options->{live
}) {
6476 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
6478 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6479 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
6481 delete $devinfo->{'drive-efidisk0'}; # this special drive is already restored before start
6482 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6484 PVE
::QemuConfig-
>remove_lock($vmid, "create");
6488 sub pbs_live_restore
{
6489 my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6491 print "starting VM for live-restore\n";
6492 print "repository: '$repo', snapshot: '$snap'\n";
6494 my $pbs_backing = {};
6495 for my $ds (keys %$restored_disks) {
6496 $ds =~ m/^drive-(.*)$/;
6498 $pbs_backing->{$confname} = {
6499 repository
=> $repo,
6501 archive
=> "$ds.img.fidx",
6503 $pbs_backing->{$confname}->{keyfile
} = $keyfile if -e
$keyfile;
6505 my $drive = parse_drive
($confname, $conf->{$confname});
6506 print "restoring '$ds' to '$drive->{file}'\n";
6509 my $drives_streamed = 0;
6511 # make sure HA doesn't interrupt our restore by stopping the VM
6512 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
6513 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
6516 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6517 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6518 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'pbs-backing' => $pbs_backing}, {});
6520 my $qmeventd_fd = register_qmeventd_handle
($vmid);
6522 # begin streaming, i.e. data copy from PBS to target disk for every vol,
6523 # this will effectively collapse the backing image chain consisting of
6524 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6525 # removes itself once all backing images vanish with 'auto-remove=on')
6527 for my $ds (sort keys %$restored_disks) {
6528 my $job_id = "restore-$ds";
6529 mon_cmd
($vmid, 'block-stream',
6530 'job-id' => $job_id,
6533 $jobs->{$job_id} = {};
6536 mon_cmd
($vmid, 'cont');
6537 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
6539 print "restore-drive jobs finished successfully, removing all tracking block devices"
6540 ." to disconnect from Proxmox Backup Server\n";
6542 for my $ds (sort keys %$restored_disks) {
6543 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6546 close($qmeventd_fd);
6552 warn "An error occured during live-restore: $err\n";
6553 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
6554 die "live-restore failed\n";
6558 sub restore_vma_archive
{
6559 my ($archive, $vmid, $user, $opts, $comp) = @_;
6561 my $readfrom = $archive;
6563 my $cfg = PVE
::Storage
::config
();
6565 my $bwlimit = $opts->{bwlimit
};
6567 my $dbg_cmdstring = '';
6568 my $add_pipe = sub {
6570 push @$commands, $cmd;
6571 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6572 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6577 if ($archive eq '-') {
6580 # If we use a backup from a PVE defined storage we also consider that
6581 # storage's rate limit:
6582 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6583 if (defined($volid)) {
6584 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6585 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6587 print STDERR
"applying read rate limit: $readlimit\n";
6588 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6589 $add_pipe->($cstream);
6595 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6596 my $cmd = $info->{decompressor
};
6597 push @$cmd, $readfrom;
6601 my $tmpdir = "/var/tmp/vzdumptmp$$";
6604 # disable interrupts (always do cleanups)
6608 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6610 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6611 POSIX
::mkfifo
($mapfifo, 0600);
6613 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6615 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6622 my $rpcenv = PVE
::RPCEnvironment
::get
();
6624 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6626 # Note: $oldconf is undef if VM does not exist
6627 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6628 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6629 my $new_conf_raw = '';
6633 my $print_devmap = sub {
6634 my $cfgfn = "$tmpdir/qemu-server.conf";
6636 # we can read the config - that is already extracted
6637 my $fh = IO
::File-
>new($cfgfn, "r") ||
6638 die "unable to read qemu-server.conf - $!\n";
6640 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6642 my $pve_firewall_dir = '/etc/pve/firewall';
6643 mkdir $pve_firewall_dir; # make sure the dir exists
6644 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6647 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6649 foreach my $info (values %{$virtdev_hash}) {
6650 my $storeid = $info->{storeid
};
6651 next if defined($storage_limits{$storeid});
6653 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
6654 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6655 $storage_limits{$storeid} = $limit * 1024;
6658 foreach my $devname (keys %$devinfo) {
6659 die "found no device mapping information for device '$devname'\n"
6660 if !$devinfo->{$devname}->{virtdev
};
6663 # create empty/temp config
6665 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6666 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6670 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6672 # print restore information to $fifofh
6673 foreach my $virtdev (sort keys %$virtdev_hash) {
6674 my $d = $virtdev_hash->{$virtdev};
6675 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6677 my $storeid = $d->{storeid
};
6678 my $volid = $d->{volid
};
6681 if (my $limit = $storage_limits{$storeid}) {
6682 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6685 my $write_zeros = 1;
6686 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6690 my $path = PVE
::Storage
::path
($cfg, $volid);
6692 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6694 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6697 $fh->seek(0, 0) || die "seek failed - $!\n";
6699 my $cookie = { netcount
=> 0 };
6700 while (defined(my $line = <$fh>)) {
6701 $new_conf_raw .= restore_update_config_line
(
6718 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6719 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6721 $oldtimeout = alarm($timeout);
6728 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6729 my ($dev_id, $size, $devname) = ($1, $2, $3);
6730 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6731 } elsif ($line =~ m/^CTIME: /) {
6732 # we correctly received the vma config, so we can disable
6733 # the timeout now for disk allocation (set to 10 minutes, so
6734 # that we always timeout if something goes wrong)
6737 print $fifofh "done\n";
6738 my $tmp = $oldtimeout || 0;
6739 $oldtimeout = undef;
6746 print "restore vma archive: $dbg_cmdstring\n";
6747 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6751 alarm($oldtimeout) if $oldtimeout;
6753 $restore_deactivate_volumes->($cfg, $devinfo);
6755 close($fifofh) if $fifofh;
6760 $restore_destroy_volumes->($cfg, $devinfo);
6764 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6766 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6768 eval { rescan
($vmid, 1); };
6771 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
6774 sub restore_tar_archive
{
6775 my ($archive, $vmid, $user, $opts) = @_;
6777 if ($archive ne '-') {
6778 my $firstfile = tar_archive_read_firstfile
($archive);
6779 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6780 if $firstfile ne 'qemu-server.conf';
6783 my $storecfg = PVE
::Storage
::config
();
6785 # avoid zombie disks when restoring over an existing VM -> cleanup first
6786 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6787 # skiplock=1 because qmrestore has set the 'create' lock itself already
6788 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6789 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6791 my $tocmd = "/usr/lib/qemu-server/qmextract";
6793 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6794 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6795 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6796 $tocmd .= ' --info' if $opts->{info
};
6798 # tar option "xf" does not autodetect compression when read from STDIN,
6799 # so we pipe to zcat
6800 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6801 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6803 my $tmpdir = "/var/tmp/vzdumptmp$$";
6806 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6807 local $ENV{VZDUMP_VMID
} = $vmid;
6808 local $ENV{VZDUMP_USER
} = $user;
6810 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6811 my $new_conf_raw = '';
6813 # disable interrupts (always do cleanups)
6817 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6825 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6827 if ($archive eq '-') {
6828 print "extracting archive from STDIN\n";
6829 run_command
($cmd, input
=> "<&STDIN");
6831 print "extracting archive '$archive'\n";
6835 return if $opts->{info
};
6839 my $statfile = "$tmpdir/qmrestore.stat";
6840 if (my $fd = IO
::File-
>new($statfile, "r")) {
6841 while (defined (my $line = <$fd>)) {
6842 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6843 $map->{$1} = $2 if $1;
6845 print STDERR
"unable to parse line in statfile - $line\n";
6851 my $confsrc = "$tmpdir/qemu-server.conf";
6853 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6855 my $cookie = { netcount
=> 0 };
6856 while (defined (my $line = <$srcfd>)) {
6857 $new_conf_raw .= restore_update_config_line
(
6868 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6874 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6876 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6878 eval { rescan
($vmid, 1); };
6882 sub foreach_storage_used_by_vm
{
6883 my ($conf, $func) = @_;
6887 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6888 my ($ds, $drive) = @_;
6889 return if drive_is_cdrom
($drive);
6891 my $volid = $drive->{file
};
6893 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6894 $sidhash->{$sid} = $sid if $sid;
6897 foreach my $sid (sort keys %$sidhash) {
6902 my $qemu_snap_storage = {
6905 sub do_snapshots_with_qemu
{
6906 my ($storecfg, $volid) = @_;
6908 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6909 my $scfg = $storecfg->{ids
}->{$storage_name};
6910 die "could not find storage '$storage_name'\n" if !defined($scfg);
6912 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6916 if ($volid =~ m/\.(qcow2|qed)$/){
6923 sub qga_check_running
{
6924 my ($vmid, $nowarn) = @_;
6926 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6928 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6934 sub template_create
{
6935 my ($vmid, $conf, $disk) = @_;
6937 my $storecfg = PVE
::Storage
::config
();
6939 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6940 my ($ds, $drive) = @_;
6942 return if drive_is_cdrom
($drive);
6943 return if $disk && $ds ne $disk;
6945 my $volid = $drive->{file
};
6946 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6948 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6949 $drive->{file
} = $voliddst;
6950 $conf->{$ds} = print_drive
($drive);
6951 PVE
::QemuConfig-
>write_config($vmid, $conf);
6955 sub convert_iscsi_path
{
6958 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6963 my $initiator_name = get_initiator_name
();
6965 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6966 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6969 die "cannot convert iscsi path '$path', unkown format\n";
6972 sub qemu_img_convert
{
6973 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6975 my $storecfg = PVE
::Storage
::config
();
6976 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6977 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6979 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6983 my $src_is_iscsi = 0;
6987 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6988 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6989 $src_format = qemu_img_format
($src_scfg, $src_volname);
6990 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6991 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6992 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6993 } elsif (-f
$src_volid) {
6994 $src_path = $src_volid;
6995 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7000 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7002 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7003 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7004 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7005 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7008 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7009 push @$cmd, '-l', "snapshot.name=$snapname"
7010 if $snapname && $src_format && $src_format eq "qcow2";
7011 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7012 push @$cmd, '-T', $cachemode if defined($cachemode);
7014 if ($src_is_iscsi) {
7015 push @$cmd, '--image-opts';
7016 $src_path = convert_iscsi_path
($src_path);
7017 } elsif ($src_format) {
7018 push @$cmd, '-f', $src_format;
7021 if ($dst_is_iscsi) {
7022 push @$cmd, '--target-image-opts';
7023 $dst_path = convert_iscsi_path
($dst_path);
7025 push @$cmd, '-O', $dst_format;
7028 push @$cmd, $src_path;
7030 if (!$dst_is_iscsi && $is_zero_initialized) {
7031 push @$cmd, "zeroinit:$dst_path";
7033 push @$cmd, $dst_path;
7038 if($line =~ m/\((\S+)\/100\
%\)/){
7040 my $transferred = int($size * $percent / 100);
7041 my $total_h = render_bytes
($size, 1);
7042 my $transferred_h = render_bytes
($transferred, 1);
7044 print "transferred $transferred_h of $total_h ($percent%)\n";
7049 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7051 die "copy failed: $err" if $err;
7054 sub qemu_img_format
{
7055 my ($scfg, $volname) = @_;
7057 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7064 sub qemu_drive_mirror
{
7065 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7067 $jobs = {} if !$jobs;
7071 $jobs->{"drive-$drive"} = {};
7073 if ($dst_volid =~ /^nbd:/) {
7074 $qemu_target = $dst_volid;
7077 my $storecfg = PVE
::Storage
::config
();
7078 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7080 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7082 $format = qemu_img_format
($dst_scfg, $dst_volname);
7084 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7086 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7089 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7090 $opts->{format
} = $format if $format;
7092 if (defined($src_bitmap)) {
7093 $opts->{sync
} = 'incremental';
7094 $opts->{bitmap
} = $src_bitmap;
7095 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7098 if (defined($bwlimit)) {
7099 $opts->{speed
} = $bwlimit * 1024;
7100 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7102 print "drive mirror is starting for drive-$drive\n";
7105 # if a job already runs for this device we get an error, catch it for cleanup
7106 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7108 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7110 die "mirroring error: $err\n";
7113 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7116 # $completion can be either
7117 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7118 # 'cancel': wait until all jobs are ready, block-job-cancel them
7119 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7120 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7121 sub qemu_drive_mirror_monitor
{
7122 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7124 $completion //= 'complete';
7128 my $err_complete = 0;
7130 my $starttime = time ();
7132 die "block job ('$op') timed out\n" if $err_complete > 300;
7134 my $stats = mon_cmd
($vmid, "query-block-jobs");
7137 my $running_jobs = {};
7138 for my $stat (@$stats) {
7139 next if $stat->{type
} ne $op;
7140 $running_jobs->{$stat->{device
}} = $stat;
7143 my $readycounter = 0;
7145 for my $job_id (sort keys %$jobs) {
7146 my $job = $running_jobs->{$job_id};
7148 my $vanished = !defined($job);
7149 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7150 if($complete || ($vanished && $completion eq 'auto')) {
7151 print "$job_id: $op-job finished\n";
7152 delete $jobs->{$job_id};
7156 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7158 my $busy = $job->{busy
};
7159 my $ready = $job->{ready
};
7160 if (my $total = $job->{len
}) {
7161 my $transferred = $job->{offset
} || 0;
7162 my $remaining = $total - $transferred;
7163 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7165 my $duration = $ctime - $starttime;
7166 my $total_h = render_bytes
($total, 1);
7167 my $transferred_h = render_bytes
($transferred, 1);
7169 my $status = sprintf(
7170 "transferred $transferred_h of $total_h ($percent%%) in %s",
7171 render_duration
($duration),
7176 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7178 $status .= ", ready";
7181 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
7182 $jobs->{$job_id}->{ready
} = $ready;
7185 $readycounter++ if $job->{ready
};
7188 last if scalar(keys %$jobs) == 0;
7190 if ($readycounter == scalar(keys %$jobs)) {
7191 print "all '$op' jobs are ready\n";
7193 # do the complete later (or has already been done)
7194 last if $completion eq 'skip' || $completion eq 'auto';
7196 if ($vmiddst && $vmiddst != $vmid) {
7197 my $agent_running = $qga && qga_check_running
($vmid);
7198 if ($agent_running) {
7199 print "freeze filesystem\n";
7200 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
7202 print "suspend vm\n";
7203 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
7206 # if we clone a disk for a new target vm, we don't switch the disk
7207 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
7209 if ($agent_running) {
7210 print "unfreeze filesystem\n";
7211 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
7213 print "resume vm\n";
7214 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
7220 for my $job_id (sort keys %$jobs) {
7221 # try to switch the disk if source and destination are on the same guest
7222 print "$job_id: Completing block job_id...\n";
7225 if ($completion eq 'complete') {
7226 $op = 'block-job-complete';
7227 } elsif ($completion eq 'cancel') {
7228 $op = 'block-job-cancel';
7230 die "invalid completion value: $completion\n";
7232 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
7233 if ($@ =~ m/cannot be completed/) {
7234 print "$job_id: block job cannot be completed, trying again.\n";
7237 print "$job_id: Completed successfully.\n";
7238 $jobs->{$job_id}->{complete
} = 1;
7249 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7250 die "block job ($op) error: $err";
7254 sub qemu_blockjobs_cancel
{
7255 my ($vmid, $jobs) = @_;
7257 foreach my $job (keys %$jobs) {
7258 print "$job: Cancelling block job\n";
7259 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
7260 $jobs->{$job}->{cancel
} = 1;
7264 my $stats = mon_cmd
($vmid, "query-block-jobs");
7266 my $running_jobs = {};
7267 foreach my $stat (@$stats) {
7268 $running_jobs->{$stat->{device
}} = $stat;
7271 foreach my $job (keys %$jobs) {
7273 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
7274 print "$job: Done.\n";
7275 delete $jobs->{$job};
7279 last if scalar(keys %$jobs) == 0;
7286 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7287 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7292 print "create linked clone of drive $drivename ($drive->{file})\n";
7293 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
7294 push @$newvollist, $newvolid;
7297 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
7298 $storeid = $storage if $storage;
7300 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
7302 print "create full clone of drive $drivename ($drive->{file})\n";
7305 if (drive_is_cloudinit
($drive)) {
7306 $name = "vm-$newvmid-cloudinit";
7307 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7308 if ($scfg->{path
}) {
7309 $name .= ".$dst_format";
7312 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
7313 } elsif ($drivename eq 'efidisk0') {
7314 $size = get_efivars_size
($conf);
7316 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
7318 $newvolid = PVE
::Storage
::vdisk_alloc
(
7319 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7321 push @$newvollist, $newvolid;
7323 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
7325 if (drive_is_cloudinit
($drive)) {
7326 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7327 # if this is the case, we have to complete any block-jobs still there from
7328 # previous drive-mirrors
7329 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7330 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
7335 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
7336 if (!$running || $snapname) {
7337 # TODO: handle bwlimits
7338 if ($drivename eq 'efidisk0') {
7339 # the relevant data on the efidisk may be smaller than the source
7340 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7341 # that is given by the OVMF_VARS.fd
7342 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
7343 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
7345 # better for Ceph if block size is not too small, see bug #3324
7348 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7349 "if=$src_path", "of=$dst_path"]);
7351 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
7355 my $kvmver = get_running_qemu_version
($vmid);
7356 if (!min_version
($kvmver, 2, 7)) {
7357 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7358 if $drive->{iothread
};
7361 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7362 $completion, $qga, $bwlimit);
7367 my ($size) = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
7370 $disk->{format
} = undef;
7371 $disk->{file
} = $newvolid;
7372 $disk->{size
} = $size if defined($size);
7377 sub get_running_qemu_version
{
7379 my $res = mon_cmd
($vmid, "query-version");
7380 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7383 sub qemu_use_old_bios_files
{
7384 my ($machine_type) = @_;
7386 return if !$machine_type;
7388 my $use_old_bios_files = undef;
7390 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7392 $use_old_bios_files = 1;
7394 my $version = extract_version
($machine_type, kvm_user_version
());
7395 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7396 # load new efi bios files on migration. So this hack is required to allow
7397 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7398 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7399 $use_old_bios_files = !min_version
($version, 2, 4);
7402 return ($use_old_bios_files, $machine_type);
7405 sub get_efivars_size
{
7407 my $arch = get_vm_arch
($conf);
7408 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7409 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
7410 return -s
$ovmf_vars;
7413 sub update_efidisk_size
{
7416 return if !defined($conf->{efidisk0
});
7418 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
7419 $disk->{size
} = get_efivars_size
($conf);
7420 $conf->{efidisk0
} = print_drive
($disk);
7425 sub create_efidisk
($$$$$) {
7426 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7428 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7429 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7431 my $vars_size_b = -s
$ovmf_vars;
7432 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7433 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7434 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7436 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7437 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7439 return ($volid, $size/1024);
7442 sub vm_iothreads_list
{
7445 my $res = mon_cmd
($vmid, 'query-iothreads');
7448 foreach my $iothread (@$res) {
7449 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7456 my ($conf, $drive) = @_;
7460 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7462 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7468 my $controller = int($drive->{index} / $maxdev);
7469 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
7473 return ($maxdev, $controller, $controller_prefix);
7476 sub windows_version
{
7479 return 0 if !$ostype;
7483 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7485 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7487 } elsif ($ostype =~ m/^win(\d+)$/) {
7494 sub resolve_dst_disk_format
{
7495 my ($storecfg, $storeid, $src_volname, $format) = @_;
7496 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7499 # if no target format is specified, use the source disk format as hint
7501 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7502 $format = qemu_img_format
($scfg, $src_volname);
7508 # test if requested format is supported - else use default
7509 my $supported = grep { $_ eq $format } @$validFormats;
7510 $format = $defFormat if !$supported;
7514 # NOTE: if this logic changes, please update docs & possibly gui logic
7515 sub find_vmstate_storage
{
7516 my ($conf, $storecfg) = @_;
7518 # first, return storage from conf if set
7519 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7521 my ($target, $shared, $local);
7523 foreach_storage_used_by_vm
($conf, sub {
7525 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7526 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7527 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7530 # second, use shared storage where VM has at least one disk
7531 # third, use local storage where VM has at least one disk
7532 # fall back to local storage
7533 $target = $shared // $local // 'local';
7539 my ($uuid, $uuid_str);
7540 UUID
::generate
($uuid);
7541 UUID
::unparse
($uuid, $uuid_str);
7545 sub generate_smbios1_uuid
{
7546 return "uuid=".generate_uuid
();
7552 mon_cmd
($vmid, 'nbd-server-stop');
7555 sub create_reboot_request
{
7557 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7558 or die "failed to create reboot trigger file: $!\n";
7562 sub clear_reboot_request
{
7564 my $path = "/run/qemu-server/$vmid.reboot";
7567 $res = unlink($path);
7568 die "could not remove reboot request for $vmid: $!"
7569 if !$res && $! != POSIX
::ENOENT
;
7574 sub bootorder_from_legacy
{
7575 my ($conf, $bootcfg) = @_;
7577 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
7578 my $bootindex_hash = {};
7580 foreach my $o (split(//, $boot)) {
7581 $bootindex_hash->{$o} = $i*100;
7587 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7588 my ($ds, $drive) = @_;
7590 if (drive_is_cdrom
($drive, 1)) {
7591 if ($bootindex_hash->{d
}) {
7592 $bootorder->{$ds} = $bootindex_hash->{d
};
7593 $bootindex_hash->{d
} += 1;
7595 } elsif ($bootindex_hash->{c
}) {
7596 $bootorder->{$ds} = $bootindex_hash->{c
}
7597 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
7598 $bootindex_hash->{c
} += 1;
7602 if ($bootindex_hash->{n
}) {
7603 for (my $i = 0; $i < $MAX_NETS; $i++) {
7604 my $netname = "net$i";
7605 next if !$conf->{$netname};
7606 $bootorder->{$netname} = $bootindex_hash->{n
};
7607 $bootindex_hash->{n
} += 1;
7614 # Generate default device list for 'boot: order=' property. Matches legacy
7615 # default boot order, but with explicit device names. This is important, since
7616 # the fallback for when neither 'order' nor the old format is specified relies
7617 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7618 sub get_default_bootdevices
{
7624 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
7625 push @ret, $first if $first;
7628 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
7629 push @ret, $first if $first;
7632 for (my $i = 0; $i < $MAX_NETS; $i++) {
7633 my $netname = "net$i";
7634 next if !$conf->{$netname};
7635 push @ret, $netname;
7642 sub device_bootorder
{
7645 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
7647 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
7650 if (!defined($boot) || $boot->{legacy
}) {
7651 $bootorder = bootorder_from_legacy
($conf, $boot);
7652 } elsif ($boot->{order
}) {
7653 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7654 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
7655 $bootorder->{$dev} = $i++;
7662 sub register_qmeventd_handle
{
7666 my $peer = "/var/run/qmeventd.sock";
7671 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
7673 if ($! != EINTR
&& $! != EAGAIN
) {
7674 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7677 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7678 . "after $count retries\n";
7683 # send handshake to mark VM as backing up
7684 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
7686 # return handle to be closed later when inhibit is no longer required
7690 # bash completion helper
7692 sub complete_backup_archives
{
7693 my ($cmdname, $pname, $cvalue) = @_;
7695 my $cfg = PVE
::Storage
::config
();
7699 if ($cvalue =~ m/^([^:]+):/) {
7703 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7706 foreach my $id (keys %$data) {
7707 foreach my $item (@{$data->{$id}}) {
7708 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7709 push @$res, $item->{volid
} if defined($item->{volid
});
7716 my $complete_vmid_full = sub {
7719 my $idlist = vmstatus
();
7723 foreach my $id (keys %$idlist) {
7724 my $d = $idlist->{$id};
7725 if (defined($running)) {
7726 next if $d->{template
};
7727 next if $running && $d->{status
} ne 'running';
7728 next if !$running && $d->{status
} eq 'running';
7737 return &$complete_vmid_full();
7740 sub complete_vmid_stopped
{
7741 return &$complete_vmid_full(0);
7744 sub complete_vmid_running
{
7745 return &$complete_vmid_full(1);
7748 sub complete_storage
{
7750 my $cfg = PVE
::Storage
::config
();
7751 my $ids = $cfg->{ids
};
7754 foreach my $sid (keys %$ids) {
7755 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7756 next if !$ids->{$sid}->{content
}->{images
};
7763 sub complete_migration_storage
{
7764 my ($cmd, $param, $current_value, $all_args) = @_;
7766 my $targetnode = @$all_args[1];
7768 my $cfg = PVE
::Storage
::config
();
7769 my $ids = $cfg->{ids
};
7772 foreach my $sid (keys %$ids) {
7773 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7774 next if !$ids->{$sid}->{content
}->{images
};
7783 my $qmpstatus = eval {
7784 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
7785 mon_cmd
($vmid, "query-status");
7788 return $qmpstatus && $qmpstatus->{status
} eq "paused";
7791 sub check_volume_storage_type
{
7792 my ($storecfg, $vol) = @_;
7794 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
7795 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7796 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
7798 die "storage '$storeid' does not support content-type '$vtype'\n"
7799 if !$scfg->{content
}->{$vtype};