1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
30 use PVE
::DataCenterConfig
;
31 use PVE
::Exception
qw(raise raise_param_exc);
32 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
34 use PVE
::JSONSchema
qw(get_standard_option);
36 use PVE
::RPCEnvironment
;
40 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
44 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
45 use PVE
::QemuServer
::Cloudinit
;
46 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
47 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
48 use PVE
::QemuServer
::Machine
;
49 use PVE
::QemuServer
::Memory
;
50 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
51 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
52 use PVE
::QemuServer
::USB
qw(parse_usb_device);
56 require PVE
::Network
::SDN
::Zones
;
60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
63 "$EDK2_FW_BASE/OVMF_CODE.fd",
64 "$EDK2_FW_BASE/OVMF_VARS.fd"
67 "$EDK2_FW_BASE/AAVMF_CODE.fd",
68 "$EDK2_FW_BASE/AAVMF_VARS.fd"
72 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
74 # Note about locking: we use flock on the config file protect
75 # against concurent actions.
76 # Aditionaly, we have a 'lock' setting in the config file. This
77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
78 # allowed when such lock is set. But you can ignore this kind of
79 # lock with the --skiplock flag.
81 cfs_register_file
('/qemu-server/',
85 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
86 description
=> "Some command save/restore state from this location.",
92 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
93 description
=> "Specifies the Qemu machine type.",
95 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
102 my ($map, $source) = @_;
104 return $source if !defined($map);
106 return $map->{entries
}->{$source}
107 if $map->{entries
} && defined($map->{entries
}->{$source});
109 return $map->{default} if $map->{default};
111 # identity (fallback)
115 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
116 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
118 format
=> 'storagepair-list',
122 #no warnings 'redefine';
125 my ($controller, $vmid, $option, $value) = @_;
127 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
128 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
134 $nodename_cache //= PVE
::INotify
::nodename
();
135 return $nodename_cache;
142 enum
=> [qw(i6300esb ib700)],
143 description
=> "Watchdog type to emulate.",
144 default => 'i6300esb',
149 enum
=> [qw(reset shutdown poweroff pause debug none)],
150 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
154 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
158 description
=> "Enable/disable Qemu GuestAgent.",
163 fstrim_cloned_disks
=> {
164 description
=> "Run fstrim after cloning/moving a disk.",
170 description
=> "Select the agent type",
174 enum
=> [qw(virtio isa)],
180 description
=> "Select the VGA type.",
185 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
188 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
200 description
=> "The size of the file in MB.",
204 pattern
=> '[a-zA-Z0-9\-]+',
206 format_description
=> 'string',
207 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
214 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
215 description
=> "Configure an audio device."
222 description
=> "Driver backend for the audio device."
226 my $spice_enhancements_fmt = {
231 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
235 enum
=> ['off', 'all', 'filter'],
238 description
=> "Enable video streaming. Uses compression for detected video streams."
245 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
247 description
=> "The file on the host to gather entropy from. In most"
248 . " cases /dev/urandom should be preferred over /dev/random"
249 . " to avoid entropy-starvation issues on the host. Using"
250 . " urandom does *not* decrease security in any meaningful"
251 . " way, as it's still seeded from real entropy, and the"
252 . " bytes provided will most likely be mixed with real"
253 . " entropy on the guest as well. /dev/hwrng can be used"
254 . " to pass through a hardware RNG from the host.",
258 description
=> "Maximum bytes of entropy injected into the guest every"
259 . " 'period' milliseconds. Prefer a lower value when using"
260 . " /dev/random as source. Use 0 to disable limiting"
261 . " (potentially dangerous!).",
264 # default is 1 KiB/s, provides enough entropy to the guest to avoid
265 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
266 # of overwhelming the host, provided we're reading from /dev/urandom
271 description
=> "Every 'period' milliseconds the entropy-injection quota"
272 . " is reset, allowing the guest to retrieve another"
273 . " 'max_bytes' of entropy.",
283 description
=> "Specifies whether a VM will be started during system bootup.",
289 description
=> "Automatic restart after crash (currently ignored).",
294 type
=> 'string', format
=> 'pve-hotplug-features',
295 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
296 default => 'network,disk,usb',
301 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
307 description
=> "Lock/unlock the VM.",
308 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
313 description
=> "Limit of CPU usage.",
314 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
322 description
=> "CPU weight for a VM.",
323 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
331 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
338 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
344 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
352 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
353 "It should not be necessary to set it.",
354 enum
=> PVE
::Tools
::kvmkeymaplist
(),
359 type
=> 'string', format
=> 'dns-name',
360 description
=> "Set a name for the VM. Only used on the configuration web interface.",
365 description
=> "SCSI controller model",
366 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
372 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
377 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
378 description
=> "Specify guest operating system.",
379 verbose_description
=> <<EODESC,
380 Specify guest operating system. This is used to enable special
381 optimization/features for specific operating systems:
384 other;; unspecified OS
385 wxp;; Microsoft Windows XP
386 w2k;; Microsoft Windows 2000
387 w2k3;; Microsoft Windows 2003
388 w2k8;; Microsoft Windows 2008
389 wvista;; Microsoft Windows Vista
390 win7;; Microsoft Windows 7
391 win8;; Microsoft Windows 8/2012/2012r2
392 win10;; Microsoft Windows 10/2016
393 l24;; Linux 2.4 Kernel
394 l26;; Linux 2.6 - 5.X Kernel
395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
401 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
402 pattern
=> '[acdn]{1,4}',
407 type
=> 'string', format
=> 'pve-qm-bootdisk',
408 description
=> "Enable booting from specified disk.",
409 pattern
=> '(ide|sata|scsi|virtio)\d+',
414 description
=> "The number of CPUs. Please use option -sockets instead.",
421 description
=> "The number of CPU sockets.",
428 description
=> "The number of cores per socket.",
435 description
=> "Enable/disable NUMA.",
441 description
=> "Enable/disable hugepages memory.",
442 enum
=> [qw(any 2 1024)],
447 description
=> "Number of hotplugged vcpus.",
454 description
=> "Enable/disable ACPI.",
459 description
=> "Enable/disable Qemu GuestAgent and its properties.",
461 format
=> $agent_fmt,
466 description
=> "Enable/disable KVM hardware virtualization.",
472 description
=> "Enable/disable time drift fix.",
478 description
=> "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
483 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
487 type
=> 'string', format
=> $vga_fmt,
488 description
=> "Configure the VGA hardware.",
489 verbose_description
=> "Configure the VGA Hardware. If you want to use ".
490 "high resolution modes (>= 1280x1024x16) you may need to increase " .
491 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
492 "is 'std' for all OS types besides some Windows versions (XP and " .
493 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
494 "display server. For win* OS you can select how many independent " .
495 "displays you want, Linux guests can add displays them self.\n".
496 "You can also run without any graphic card, using a serial device as terminal.",
500 type
=> 'string', format
=> 'pve-qm-watchdog',
501 description
=> "Create a virtual hardware watchdog device.",
502 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled" .
503 " (by a guest action), the watchdog must be periodically polled " .
504 "by an agent inside the guest or else the watchdog will reset " .
505 "the guest (or execute the respective action specified)",
510 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
511 description
=> "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
512 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
515 startup
=> get_standard_option
('pve-startup-order'),
519 description
=> "Enable/disable Template.",
525 description
=> "Arbitrary arguments passed to kvm.",
526 verbose_description
=> <<EODESCR,
527 Arbitrary arguments passed to kvm, for example:
529 args: -no-reboot -no-hpet
531 NOTE: this option is for experts only.
538 description
=> "Enable/disable the USB tablet device.",
539 verbose_description
=> "Enable/disable the USB tablet device. This device is " .
540 "usually needed to allow absolute mouse positioning with VNC. " .
541 "Else the mouse runs out of sync with normal VNC clients. " .
542 "If you're running lots of console-only guests on one host, " .
543 "you may consider disabling this to save some context switches. " .
544 "This is turned off by default if you use spice (-vga=qxl).",
549 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
553 migrate_downtime
=> {
556 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
562 type
=> 'string', format
=> 'pve-qm-ide',
563 typetext
=> '<volume>',
564 description
=> "This is an alias for option -ide2",
568 description
=> "Emulated CPU type.",
570 format
=> 'pve-vm-cpu-conf',
572 parent
=> get_standard_option
('pve-snapshot-name', {
574 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
578 description
=> "Timestamp for snapshots.",
584 type
=> 'string', format
=> 'pve-volume-id',
585 description
=> "Reference to a volume which stores the VM state. This is used internally for snapshots.",
587 vmstatestorage
=> get_standard_option
('pve-storage-id', {
588 description
=> "Default storage for VM state volumes/files.",
591 runningmachine
=> get_standard_option
('pve-qemu-machine', {
592 description
=> "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
595 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
598 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
599 format_description
=> 'QEMU -cpu parameter'
601 machine
=> get_standard_option
('pve-qemu-machine'),
603 description
=> "Virtual processor architecture. Defaults to the host.",
606 enum
=> [qw(x86_64 aarch64)],
609 description
=> "Specify SMBIOS type 1 fields.",
610 type
=> 'string', format
=> 'pve-qm-smbios1',
617 description
=> "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
623 enum
=> [ qw(seabios ovmf) ],
624 description
=> "Select BIOS implementation.",
625 default => 'seabios',
629 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
630 format_description
=> 'UUID',
631 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
632 verbose_description
=> "The VM generation ID (vmgenid) device exposes a".
633 " 128-bit integer value identifier to the guest OS. This allows to".
634 " notify the guest operating system when the virtual machine is".
635 " executed with a different configuration (e.g. snapshot execution".
636 " or creation from a template). The guest operating system notices".
637 " the change, and is then able to react as appropriate by marking".
638 " its copies of distributed databases as dirty, re-initializing its".
639 " random number generator, etc.\n".
640 "Note that auto-creation only works when done throug API/CLI create".
641 " or update methods, but not when manually editing the config file.",
642 default => "1 (autogenerated)",
647 format
=> 'pve-volume-id',
649 description
=> "Script that will be executed during various steps in the vms lifetime.",
653 format
=> $ivshmem_fmt,
654 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
659 format
=> $audio_fmt,
660 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
663 spice_enhancements
=> {
665 format
=> $spice_enhancements_fmt,
666 description
=> "Configure additional enhancements for SPICE.",
670 type
=> 'string', format
=> 'pve-tag-list',
671 description
=> 'Tags of the VM. This is only meta information.',
677 description
=> "Configure a VirtIO-based Random Number Generator.",
686 description
=> 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
687 format
=> 'pve-volume-id',
688 format_description
=> 'volume',
693 description
=> 'Specify a custom file containing all network data passed to the VM via cloud-init.',
694 format
=> 'pve-volume-id',
695 format_description
=> 'volume',
700 description
=> 'Specify a custom file containing all user data passed to the VM via cloud-init.',
701 format
=> 'pve-volume-id',
702 format_description
=> 'volume',
705 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
707 my $confdesc_cloudinit = {
711 description
=> 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
712 enum
=> ['configdrive2', 'nocloud'],
717 description
=> "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
722 description
=> 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
727 description
=> 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
728 format
=> 'pve-qm-cicustom',
733 description
=> "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
737 type
=> 'string', format
=> 'address-list',
738 description
=> "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
743 format
=> 'urlencoded',
744 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
748 # what about other qemu settings ?
750 #machine => 'string',
763 ##soundhw => 'string',
765 while (my ($k, $v) = each %$confdesc) {
766 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
769 my $MAX_USB_DEVICES = 5;
771 my $MAX_SERIAL_PORTS = 4;
772 my $MAX_PARALLEL_PORTS = 3;
778 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
779 description
=> "CPUs accessing this NUMA node.",
780 format_description
=> "id[-id];...",
784 description
=> "Amount of memory this NUMA node provides.",
789 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
790 description
=> "Host NUMA nodes to use.",
791 format_description
=> "id[-id];...",
796 enum
=> [qw(preferred bind interleave)],
797 description
=> "NUMA allocation policy.",
801 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
804 type
=> 'string', format
=> $numa_fmt,
805 description
=> "NUMA topology.",
807 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
809 for (my $i = 0; $i < $MAX_NUMA; $i++) {
810 $confdesc->{"numa$i"} = $numadesc;
813 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
814 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
815 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
816 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
818 my $net_fmt_bridge_descr = <<__EOD__;
819 Bridge to attach the network device to. The Proxmox VE standard bridge
822 If you do not specify a bridge, we create a kvm user (NATed) network
823 device, which provides DHCP and DNS services. The following addresses
830 The DHCP server assign addresses to the guest starting from 10.0.2.15.
834 macaddr
=> get_standard_option
('mac-addr', {
835 description
=> "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
839 description
=> "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
840 enum
=> $nic_model_list,
843 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
846 description
=> $net_fmt_bridge_descr,
847 format_description
=> 'bridge',
848 pattern
=> '[-_.\w\d]+',
853 minimum
=> 0, maximum
=> 16,
854 description
=> 'Number of packet queues to be used on the device.',
860 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
865 minimum
=> 1, maximum
=> 4094,
866 description
=> 'VLAN tag to apply to packets on this interface.',
871 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
872 description
=> 'VLAN trunks to pass through this interface.',
873 format_description
=> 'vlanid[;vlanid...]',
878 description
=> 'Whether this interface should be protected by the firewall.',
883 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
888 minimum
=> 1, maximum
=> 65520,
889 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
896 type
=> 'string', format
=> $net_fmt,
897 description
=> "Specify network devices.",
900 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
905 format
=> 'pve-ipv4-config',
906 format_description
=> 'IPv4Format/CIDR',
907 description
=> 'IPv4 address in CIDR format.',
914 format_description
=> 'GatewayIPv4',
915 description
=> 'Default gateway for IPv4 traffic.',
921 format
=> 'pve-ipv6-config',
922 format_description
=> 'IPv6Format/CIDR',
923 description
=> 'IPv6 address in CIDR format.',
930 format_description
=> 'GatewayIPv6',
931 description
=> 'Default gateway for IPv6 traffic.',
936 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
939 type
=> 'string', format
=> 'pve-qm-ipconfig',
940 description
=> <<'EODESCR',
941 cloud-init: Specify IP addresses and gateways for the corresponding interface.
943 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
945 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
946 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
948 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
951 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
953 for (my $i = 0; $i < $MAX_NETS; $i++) {
954 $confdesc->{"net$i"} = $netdesc;
955 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
958 foreach my $key (keys %$confdesc_cloudinit) {
959 $confdesc->{$key} = $confdesc_cloudinit->{$key};
962 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
963 sub verify_volume_id_or_qm_path
{
964 my ($volid, $noerr) = @_;
966 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
970 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
971 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
973 return undef if $noerr;
982 type
=> 'string', format
=> 'pve-qm-usb-device',
983 format_description
=> 'HOSTUSBDEVICE|spice',
984 description
=> <<EODESCR,
985 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
987 'bus-port(.port)*' (decimal numbers) or
988 'vendor_id:product_id' (hexadeciaml numbers) or
991 You can use the 'lsusb -t' command to list existing usb devices.
993 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
995 The value 'spice' can be used to add a usb redirection devices for spice.
1001 description
=> "Specifies whether if given host option is a USB3 device or port.",
1008 type
=> 'string', format
=> $usb_fmt,
1009 description
=> "Configure an USB device (n is 0 to 4).",
1011 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1016 pattern
=> '(/dev/.+|socket)',
1017 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1018 verbose_description
=> <<EODESCR,
1019 Create a serial device inside the VM (n is 0 to 3), and pass through a
1020 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1021 host side (use 'qm terminal' to open a terminal connection).
1023 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1025 CAUTION: Experimental! User reported problems with this option.
1032 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1033 description
=> "Map host parallel devices (n is 0 to 2).",
1034 verbose_description
=> <<EODESCR,
1035 Map host parallel devices (n is 0 to 2).
1037 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1039 CAUTION: Experimental! User reported problems with this option.
1043 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1044 $confdesc->{"parallel$i"} = $paralleldesc;
1047 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1048 $confdesc->{"serial$i"} = $serialdesc;
1051 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1052 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1055 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1056 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1059 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1060 $confdesc->{"usb$i"} = $usbdesc;
1063 my $kvm_api_version = 0;
1066 return $kvm_api_version if $kvm_api_version;
1068 open my $fh, '<', '/dev/kvm'
1071 # 0xae00 => KVM_GET_API_VERSION
1072 $kvm_api_version = ioctl($fh, 0xae00, 0);
1074 return $kvm_api_version;
1077 my $kvm_user_version = {};
1080 sub kvm_user_version
{
1083 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1084 my $st = stat($binary);
1086 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1087 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1088 $cachedmtime == $st->mtime;
1090 $kvm_user_version->{$binary} = 'unknown';
1091 $kvm_mtime->{$binary} = $st->mtime;
1095 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1096 $kvm_user_version->{$binary} = $2;
1100 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1103 return $kvm_user_version->{$binary};
1107 sub kernel_has_vhost_net
{
1108 return -c
'/dev/vhost-net';
1113 return defined($confdesc->{$key});
1117 sub get_cdrom_path
{
1119 return $cdrom_path if $cdrom_path;
1121 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1122 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1123 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1127 my ($storecfg, $vmid, $cdrom) = @_;
1129 if ($cdrom eq 'cdrom') {
1130 return get_cdrom_path
();
1131 } elsif ($cdrom eq 'none') {
1133 } elsif ($cdrom =~ m
|^/|) {
1136 return PVE
::Storage
::path
($storecfg, $cdrom);
1140 # try to convert old style file names to volume IDs
1141 sub filename_to_volume_id
{
1142 my ($vmid, $file, $media) = @_;
1144 if (!($file eq 'none' || $file eq 'cdrom' ||
1145 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1147 return undef if $file =~ m
|/|;
1149 if ($media && $media eq 'cdrom') {
1150 $file = "local:iso/$file";
1152 $file = "local:$vmid/$file";
1159 sub verify_media_type
{
1160 my ($opt, $vtype, $media) = @_;
1165 if ($media eq 'disk') {
1167 } elsif ($media eq 'cdrom') {
1170 die "internal error";
1173 return if ($vtype eq $etype);
1175 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1178 sub cleanup_drive_path
{
1179 my ($opt, $storecfg, $drive) = @_;
1181 # try to convert filesystem paths to volume IDs
1183 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1184 ($drive->{file
} !~ m
|^/dev/.+|) &&
1185 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1186 ($drive->{file
} !~ m/^\d+$/)) {
1187 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1188 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1189 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1190 verify_media_type
($opt, $vtype, $drive->{media
});
1191 $drive->{file
} = $volid;
1194 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1197 sub parse_hotplug_features
{
1202 return $res if $data eq '0';
1204 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1206 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1207 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1210 die "invalid hotplug feature '$feature'\n";
1216 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1217 sub pve_verify_hotplug_features
{
1218 my ($value, $noerr) = @_;
1220 return $value if parse_hotplug_features
($value);
1222 return undef if $noerr;
1224 die "unable to parse hotplug option\n";
1228 my($fh, $noerr) = @_;
1231 my $SG_GET_VERSION_NUM = 0x2282;
1233 my $versionbuf = "\x00" x
8;
1234 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1236 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1239 my $version = unpack("I", $versionbuf);
1240 if ($version < 30000) {
1241 die "scsi generic interface too old\n" if !$noerr;
1245 my $buf = "\x00" x
36;
1246 my $sensebuf = "\x00" x
8;
1247 my $cmd = pack("C x3 C x1", 0x12, 36);
1249 # see /usr/include/scsi/sg.h
1250 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1252 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1253 length($sensebuf), 0, length($buf), $buf,
1254 $cmd, $sensebuf, 6000);
1256 $ret = ioctl($fh, $SG_IO, $packet);
1258 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1262 my @res = unpack($sg_io_hdr_t, $packet);
1263 if ($res[17] || $res[18]) {
1264 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1269 (my $byte0, my $byte1, $res->{vendor
},
1270 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1272 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1273 $res->{type
} = $byte0 & 31;
1281 my $fh = IO
::File-
>new("+<$path") || return undef;
1282 my $res = scsi_inquiry
($fh, 1);
1288 sub print_tabletdevice_full
{
1289 my ($conf, $arch) = @_;
1291 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1293 # we use uhci for old VMs because tablet driver was buggy in older qemu
1295 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1301 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1304 sub print_keyboarddevice_full
{
1305 my ($conf, $arch, $machine) = @_;
1307 return undef if $arch ne 'aarch64';
1309 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1312 sub print_drivedevice_full
{
1313 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1318 if ($drive->{interface
} eq 'virtio') {
1319 my $pciaddr = print_pci_addr
("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1320 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1321 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread
};
1322 } elsif ($drive->{interface
} eq 'scsi') {
1324 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1325 my $unit = $drive->{index} % $maxdev;
1326 my $devicetype = 'hd';
1328 if (drive_is_cdrom
($drive)) {
1331 if ($drive->{file
} =~ m
|^/|) {
1332 $path = $drive->{file
};
1333 if (my $info = path_is_scsi
($path)) {
1334 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1335 $devicetype = 'block';
1336 } elsif ($info->{type
} == 1) { # tape
1337 $devicetype = 'generic';
1341 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1344 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1345 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, kvm_user_version
());
1346 if ($path =~ m/^iscsi\:\/\
// &&
1347 !min_version
($version, 4, 1)) {
1348 $devicetype = 'generic';
1352 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1353 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1355 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1358 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1359 $device .= ",rotation_rate=1";
1361 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1363 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1364 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1365 my $controller = int($drive->{index} / $maxdev);
1366 my $unit = $drive->{index} % $maxdev;
1367 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1369 $device = "ide-$devicetype";
1370 if ($drive->{interface
} eq 'ide') {
1371 $device .= ",bus=ide.$controller,unit=$unit";
1373 $device .= ",bus=ahci$controller.$unit";
1375 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1377 if ($devicetype eq 'hd') {
1378 if (my $model = $drive->{model
}) {
1379 $model = URI
::Escape
::uri_unescape
($model);
1380 $device .= ",model=$model";
1382 if ($drive->{ssd
}) {
1383 $device .= ",rotation_rate=1";
1386 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1387 } elsif ($drive->{interface
} eq 'usb') {
1389 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1391 die "unsupported interface type";
1394 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1396 if (my $serial = $drive->{serial
}) {
1397 $serial = URI
::Escape
::uri_unescape
($serial);
1398 $device .= ",serial=$serial";
1405 sub get_initiator_name
{
1408 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return undef;
1409 while (defined(my $line = <$fh>)) {
1410 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1419 sub print_drive_commandline_full
{
1420 my ($storecfg, $vmid, $drive) = @_;
1423 my $volid = $drive->{file
};
1426 if (drive_is_cdrom
($drive)) {
1427 $path = get_iso_path
($storecfg, $vmid, $volid);
1429 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1431 $path = PVE
::Storage
::path
($storecfg, $volid);
1432 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1433 $format = qemu_img_format
($scfg, $volname);
1441 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1442 foreach my $o (@qemu_drive_options) {
1443 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1446 # snapshot only accepts on|off
1447 if (defined($drive->{snapshot
})) {
1448 my $v = $drive->{snapshot
} ?
'on' : 'off';
1449 $opts .= ",snapshot=$v";
1452 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1453 my ($dir, $qmpname) = @$type;
1454 if (my $v = $drive->{"mbps$dir"}) {
1455 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1457 if (my $v = $drive->{"mbps${dir}_max"}) {
1458 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1460 if (my $v = $drive->{"bps${dir}_max_length"}) {
1461 $opts .= ",throttling.bps$qmpname-max-length=$v";
1463 if (my $v = $drive->{"iops${dir}"}) {
1464 $opts .= ",throttling.iops$qmpname=$v";
1466 if (my $v = $drive->{"iops${dir}_max"}) {
1467 $opts .= ",throttling.iops$qmpname-max=$v";
1469 if (my $v = $drive->{"iops${dir}_max_length"}) {
1470 $opts .= ",throttling.iops$qmpname-max-length=$v";
1474 $opts .= ",format=$format" if $format && !$drive->{format
};
1476 my $cache_direct = 0;
1478 if (my $cache = $drive->{cache
}) {
1479 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1480 } elsif (!drive_is_cdrom
($drive)) {
1481 $opts .= ",cache=none";
1485 # aio native works only with O_DIRECT
1486 if (!$drive->{aio
}) {
1488 $opts .= ",aio=native";
1490 $opts .= ",aio=threads";
1494 if (!drive_is_cdrom
($drive)) {
1496 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1497 $detectzeroes = 'off';
1498 } elsif ($drive->{discard
}) {
1499 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1501 # This used to be our default with discard not being specified:
1502 $detectzeroes = 'on';
1504 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1507 my $pathinfo = $path ?
"file=$path," : '';
1509 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1512 sub print_netdevice_full
{
1513 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1515 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
1517 my $device = $net->{model
};
1518 if ($net->{model
} eq 'virtio') {
1519 $device = 'virtio-net-pci';
1522 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1523 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1524 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1525 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1526 my $vectors = $net->{queues
} * 2 + 2;
1527 $tmpstr .= ",vectors=$vectors,mq=on";
1529 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1531 if (my $mtu = $net->{mtu
}) {
1532 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1533 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1536 } elsif ($mtu < 576) {
1537 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1538 } elsif ($mtu > $bridge_mtu) {
1539 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1541 $tmpstr .= ",host_mtu=$mtu";
1543 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1547 if ($use_old_bios_files) {
1549 if ($device eq 'virtio-net-pci') {
1550 $romfile = 'pxe-virtio.rom';
1551 } elsif ($device eq 'e1000') {
1552 $romfile = 'pxe-e1000.rom';
1553 } elsif ($device eq 'ne2k') {
1554 $romfile = 'pxe-ne2k_pci.rom';
1555 } elsif ($device eq 'pcnet') {
1556 $romfile = 'pxe-pcnet.rom';
1557 } elsif ($device eq 'rtl8139') {
1558 $romfile = 'pxe-rtl8139.rom';
1560 $tmpstr .= ",romfile=$romfile" if $romfile;
1566 sub print_netdev_full
{
1567 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1570 if ($netid =~ m/^net(\d+)$/) {
1574 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1576 my $ifname = "tap${vmid}i$i";
1578 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1579 die "interface name '$ifname' is too long (max 15 character)\n"
1580 if length($ifname) >= 16;
1582 my $vhostparam = '';
1583 if (is_native
($arch)) {
1584 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1587 my $vmname = $conf->{name
} || "vm$vmid";
1590 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1592 if ($net->{bridge
}) {
1593 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1595 $netdev = "type=user,id=$netid,hostname=$vmname";
1598 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1604 'cirrus' => 'cirrus-vga',
1606 'vmware' => 'vmware-svga',
1607 'virtio' => 'virtio-vga',
1610 sub print_vga_device
{
1611 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1613 my $type = $vga_map->{$vga->{type
}};
1614 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1615 $type = 'virtio-gpu';
1617 my $vgamem_mb = $vga->{memory
};
1619 my $max_outputs = '';
1621 $type = $id ?
'qxl' : 'qxl-vga';
1623 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1624 # set max outputs so linux can have up to 4 qxl displays with one device
1625 if (min_version
($machine_version, 4, 1)) {
1626 $max_outputs = ",max_outputs=4";
1631 die "no devicetype for $vga->{type}\n" if !$type;
1635 if ($vga->{type
} eq 'virtio') {
1636 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1637 $memory = ",max_hostmem=$bytes";
1639 # from https://www.spice-space.org/multiple-monitors.html
1640 $memory = ",vgamem_mb=$vga->{memory}";
1641 my $ram = $vgamem_mb * 4;
1642 my $vram = $vgamem_mb * 2;
1643 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1645 $memory = ",vgamem_mb=$vga->{memory}";
1647 } elsif ($qxlnum && $id) {
1648 $memory = ",ram_size=67108864,vram_size=33554432";
1651 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1652 my $vgaid = "vga" . ($id // '');
1655 if ($q35 && $vgaid eq 'vga') {
1656 # the first display uses pcie.0 bus on q35 machines
1657 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1659 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1662 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1665 sub parse_number_sets
{
1668 foreach my $part (split(/;/, $set)) {
1669 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1670 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1671 push @$res, [ $1, $2 ];
1673 die "invalid range: $part\n";
1682 my $res = PVE
::JSONSchema
::parse_property_string
($numa_fmt, $data);
1683 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1684 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1688 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1692 my $res = eval { PVE
::JSONSchema
::parse_property_string
($net_fmt, $data) };
1697 if (!defined($res->{macaddr
})) {
1698 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1699 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1704 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1705 sub parse_ipconfig
{
1708 my $res = eval { PVE
::JSONSchema
::parse_property_string
($ipconfig_fmt, $data) };
1714 if ($res->{gw
} && !$res->{ip
}) {
1715 warn 'gateway specified without specifying an IP address';
1718 if ($res->{gw6
} && !$res->{ip6
}) {
1719 warn 'IPv6 gateway specified without specifying an IPv6 address';
1722 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1723 warn 'gateway specified together with DHCP';
1726 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1728 warn "IPv6 gateway specified together with $res->{ip6} address";
1732 if (!$res->{ip
} && !$res->{ip6
}) {
1733 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1742 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1745 sub add_random_macs
{
1746 my ($settings) = @_;
1748 foreach my $opt (keys %$settings) {
1749 next if $opt !~ m/^net(\d+)$/;
1750 my $net = parse_net
($settings->{$opt});
1752 $settings->{$opt} = print_net
($net);
1756 sub vm_is_volid_owner
{
1757 my ($storecfg, $vmid, $volid) = @_;
1759 if ($volid !~ m
|^/|) {
1761 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1762 if ($owner && ($owner == $vmid)) {
1770 sub vmconfig_register_unused_drive
{
1771 my ($storecfg, $vmid, $conf, $drive) = @_;
1773 if (drive_is_cloudinit
($drive)) {
1774 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1776 } elsif (!drive_is_cdrom
($drive)) {
1777 my $volid = $drive->{file
};
1778 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1779 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1784 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1788 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1789 format_description
=> 'UUID',
1790 description
=> "Set SMBIOS1 UUID.",
1795 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1796 format_description
=> 'Base64 encoded string',
1797 description
=> "Set SMBIOS1 version.",
1802 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1803 format_description
=> 'Base64 encoded string',
1804 description
=> "Set SMBIOS1 serial number.",
1809 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1810 format_description
=> 'Base64 encoded string',
1811 description
=> "Set SMBIOS1 manufacturer.",
1816 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1817 format_description
=> 'Base64 encoded string',
1818 description
=> "Set SMBIOS1 product ID.",
1823 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1824 format_description
=> 'Base64 encoded string',
1825 description
=> "Set SMBIOS1 SKU string.",
1830 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1831 format_description
=> 'Base64 encoded string',
1832 description
=> "Set SMBIOS1 family string.",
1837 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
1845 my $res = eval { PVE
::JSONSchema
::parse_property_string
($smbios1_fmt, $data) };
1852 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
1855 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
1857 sub parse_watchdog
{
1860 return undef if !$value;
1862 my $res = eval { PVE
::JSONSchema
::parse_property_string
($watchdog_fmt, $value) };
1867 sub parse_guest_agent
{
1870 return {} if !defined($value->{agent
});
1872 my $res = eval { PVE
::JSONSchema
::parse_property_string
($agent_fmt, $value->{agent
}) };
1875 # if the agent is disabled ignore the other potentially set properties
1876 return {} if !$res->{enabled
};
1883 return {} if !$value;
1884 my $res = eval { PVE
::JSONSchema
::parse_property_string
($vga_fmt, $value) };
1892 return undef if !$value;
1894 my $res = eval { PVE
::JSONSchema
::parse_property_string
($rng_fmt, $value) };
1899 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
1900 sub verify_usb_device
{
1901 my ($value, $noerr) = @_;
1903 return $value if parse_usb_device
($value);
1905 return undef if $noerr;
1907 die "unable to parse usb device\n";
1910 # add JSON properties for create and set function
1911 sub json_config_properties
{
1914 foreach my $opt (keys %$confdesc) {
1915 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1916 $opt eq 'runningmachine' || $opt eq 'runningcpu';
1917 $prop->{$opt} = $confdesc->{$opt};
1923 # return copy of $confdesc_cloudinit to generate documentation
1924 sub cloudinit_config_properties
{
1926 return dclone
($confdesc_cloudinit);
1930 my ($key, $value) = @_;
1932 die "unknown setting '$key'\n" if !$confdesc->{$key};
1934 my $type = $confdesc->{$key}->{type
};
1936 if (!defined($value)) {
1937 die "got undefined value\n";
1940 if ($value =~ m/[\n\r]/) {
1941 die "property contains a line feed\n";
1944 if ($type eq 'boolean') {
1945 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1946 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1947 die "type check ('boolean') failed - got '$value'\n";
1948 } elsif ($type eq 'integer') {
1949 return int($1) if $value =~ m/^(\d+)$/;
1950 die "type check ('integer') failed - got '$value'\n";
1951 } elsif ($type eq 'number') {
1952 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1953 die "type check ('number') failed - got '$value'\n";
1954 } elsif ($type eq 'string') {
1955 if (my $fmt = $confdesc->{$key}->{format
}) {
1956 PVE
::JSONSchema
::check_format
($fmt, $value);
1959 $value =~ s/^\"(.*)\"$/$1/;
1962 die "internal error"
1967 my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
1969 my $conf = PVE
::QemuConfig-
>load_config($vmid);
1971 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
1973 if ($conf->{template
}) {
1974 # check if any base image is still used by a linked clone
1975 PVE
::QemuConfig-
>foreach_volume($conf, sub {
1976 my ($ds, $drive) = @_;
1977 return if drive_is_cdrom
($drive);
1979 my $volid = $drive->{file
};
1980 return if !$volid || $volid =~ m
|^/|;
1982 die "base volume '$volid' is still in use by linked cloned\n"
1983 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
1988 # only remove disks owned by this VM
1989 PVE
::QemuConfig-
>foreach_volume($conf, sub {
1990 my ($ds, $drive) = @_;
1991 return if drive_is_cdrom
($drive, 1);
1993 my $volid = $drive->{file
};
1994 return if !$volid || $volid =~ m
|^/|;
1996 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
1997 return if !$path || !$owner || ($owner != $vmid);
1999 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2000 warn "Could not remove disk '$volid', check manually: $@" if $@;
2003 # also remove unused disk
2004 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2005 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2006 my ($volid, $sid, $volname, $d) = @_;
2007 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2011 if (defined $replacement_conf) {
2012 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2014 PVE
::QemuConfig-
>destroy_config($vmid);
2018 sub parse_vm_config
{
2019 my ($filename, $raw) = @_;
2021 return undef if !defined($raw);
2024 digest
=> Digest
::SHA
::sha1_hex
($raw),
2029 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2030 || die "got strange filename '$filename'";
2038 my @lines = split(/\n/, $raw);
2039 foreach my $line (@lines) {
2040 next if $line =~ m/^\s*$/;
2042 if ($line =~ m/^\[PENDING\]\s*$/i) {
2043 $section = 'pending';
2044 if (defined($descr)) {
2046 $conf->{description
} = $descr;
2049 $conf = $res->{$section} = {};
2052 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2054 if (defined($descr)) {
2056 $conf->{description
} = $descr;
2059 $conf = $res->{snapshots
}->{$section} = {};
2063 if ($line =~ m/^\#(.*)\s*$/) {
2064 $descr = '' if !defined($descr);
2065 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2069 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2070 $descr = '' if !defined($descr);
2071 $descr .= PVE
::Tools
::decode_text
($2);
2072 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2073 $conf->{snapstate
} = $1;
2074 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2077 $conf->{$key} = $value;
2078 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2080 if ($section eq 'pending') {
2081 $conf->{delete} = $value; # we parse this later
2083 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2085 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2088 eval { $value = check_type
($key, $value); };
2090 warn "vm $vmid - unable to parse value of '$key' - $@";
2092 $key = 'ide2' if $key eq 'cdrom';
2093 my $fmt = $confdesc->{$key}->{format
};
2094 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2095 my $v = parse_drive
($key, $value);
2096 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2097 $v->{file
} = $volid;
2098 $value = print_drive
($v);
2100 warn "vm $vmid - unable to parse value of '$key'\n";
2105 $conf->{$key} = $value;
2110 if (defined($descr)) {
2112 $conf->{description
} = $descr;
2114 delete $res->{snapstate
}; # just to be sure
2119 sub write_vm_config
{
2120 my ($filename, $conf) = @_;
2122 delete $conf->{snapstate
}; # just to be sure
2124 if ($conf->{cdrom
}) {
2125 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2126 $conf->{ide2
} = $conf->{cdrom
};
2127 delete $conf->{cdrom
};
2130 # we do not use 'smp' any longer
2131 if ($conf->{sockets
}) {
2132 delete $conf->{smp
};
2133 } elsif ($conf->{smp
}) {
2134 $conf->{sockets
} = $conf->{smp
};
2135 delete $conf->{cores
};
2136 delete $conf->{smp
};
2139 my $used_volids = {};
2141 my $cleanup_config = sub {
2142 my ($cref, $pending, $snapname) = @_;
2144 foreach my $key (keys %$cref) {
2145 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2146 $key eq 'snapstate' || $key eq 'pending';
2147 my $value = $cref->{$key};
2148 if ($key eq 'delete') {
2149 die "propertry 'delete' is only allowed in [PENDING]\n"
2151 # fixme: check syntax?
2154 eval { $value = check_type
($key, $value); };
2155 die "unable to parse value of '$key' - $@" if $@;
2157 $cref->{$key} = $value;
2159 if (!$snapname && is_valid_drivename
($key)) {
2160 my $drive = parse_drive
($key, $value);
2161 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2166 &$cleanup_config($conf);
2168 &$cleanup_config($conf->{pending
}, 1);
2170 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2171 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2172 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2175 # remove 'unusedX' settings if we re-add a volume
2176 foreach my $key (keys %$conf) {
2177 my $value = $conf->{$key};
2178 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2179 delete $conf->{$key};
2183 my $generate_raw_config = sub {
2184 my ($conf, $pending) = @_;
2188 # add description as comment to top of file
2189 if (defined(my $descr = $conf->{description
})) {
2191 foreach my $cl (split(/\n/, $descr)) {
2192 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2195 $raw .= "#\n" if $pending;
2199 foreach my $key (sort keys %$conf) {
2200 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2201 $raw .= "$key: $conf->{$key}\n";
2206 my $raw = &$generate_raw_config($conf);
2208 if (scalar(keys %{$conf->{pending
}})){
2209 $raw .= "\n[PENDING]\n";
2210 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2213 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2214 $raw .= "\n[$snapname]\n";
2215 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2225 # we use static defaults from our JSON schema configuration
2226 foreach my $key (keys %$confdesc) {
2227 if (defined(my $default = $confdesc->{$key}->{default})) {
2228 $res->{$key} = $default;
2236 my $vmlist = PVE
::Cluster
::get_vmlist
();
2238 return $res if !$vmlist || !$vmlist->{ids
};
2239 my $ids = $vmlist->{ids
};
2240 my $nodename = nodename
();
2242 foreach my $vmid (keys %$ids) {
2243 my $d = $ids->{$vmid};
2244 next if !$d->{node
} || $d->{node
} ne $nodename;
2245 next if !$d->{type
} || $d->{type
} ne 'qemu';
2246 $res->{$vmid}->{exists} = 1;
2251 # test if VM uses local resources (to prevent migration)
2252 sub check_local_resources
{
2253 my ($conf, $noerr) = @_;
2257 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2258 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2260 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2262 foreach my $k (keys %$conf) {
2263 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2264 # sockets are safe: they will recreated be on the target side post-migrate
2265 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2266 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2269 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2274 # check if used storages are available on all nodes (use by migrate)
2275 sub check_storage_availability
{
2276 my ($storecfg, $conf, $node) = @_;
2278 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2279 my ($ds, $drive) = @_;
2281 my $volid = $drive->{file
};
2284 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2287 # check if storage is available on both nodes
2288 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2289 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2293 # list nodes where all VM images are available (used by has_feature API)
2295 my ($conf, $storecfg) = @_;
2297 my $nodelist = PVE
::Cluster
::get_nodelist
();
2298 my $nodehash = { map { $_ => 1 } @$nodelist };
2299 my $nodename = nodename
();
2301 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2302 my ($ds, $drive) = @_;
2304 my $volid = $drive->{file
};
2307 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2309 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2310 if ($scfg->{disable
}) {
2312 } elsif (my $avail = $scfg->{nodes
}) {
2313 foreach my $node (keys %$nodehash) {
2314 delete $nodehash->{$node} if !$avail->{$node};
2316 } elsif (!$scfg->{shared
}) {
2317 foreach my $node (keys %$nodehash) {
2318 delete $nodehash->{$node} if $node ne $nodename
2327 sub check_local_storage_availability
{
2328 my ($conf, $storecfg) = @_;
2330 my $nodelist = PVE
::Cluster
::get_nodelist
();
2331 my $nodehash = { map { $_ => {} } @$nodelist };
2333 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2334 my ($ds, $drive) = @_;
2336 my $volid = $drive->{file
};
2339 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2341 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2343 if ($scfg->{disable
}) {
2344 foreach my $node (keys %$nodehash) {
2345 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2347 } elsif (my $avail = $scfg->{nodes
}) {
2348 foreach my $node (keys %$nodehash) {
2349 if (!$avail->{$node}) {
2350 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2357 foreach my $node (values %$nodehash) {
2358 if (my $unavail = $node->{unavailable_storages
}) {
2359 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2366 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2368 my ($vmid, $nocheck, $node) = @_;
2370 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2371 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2376 my $vzlist = config_list
();
2378 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2380 while (defined(my $de = $fd->read)) {
2381 next if $de !~ m/^(\d+)\.pid$/;
2383 next if !defined($vzlist->{$vmid});
2384 if (my $pid = check_running
($vmid)) {
2385 $vzlist->{$vmid}->{pid
} = $pid;
2392 our $vmstatus_return_properties = {
2393 vmid
=> get_standard_option
('pve-vmid'),
2395 description
=> "Qemu process status.",
2397 enum
=> ['stopped', 'running'],
2400 description
=> "Maximum memory in bytes.",
2403 renderer
=> 'bytes',
2406 description
=> "Root disk size in bytes.",
2409 renderer
=> 'bytes',
2412 description
=> "VM name.",
2417 description
=> "Qemu QMP agent status.",
2422 description
=> "PID of running qemu process.",
2427 description
=> "Uptime.",
2430 renderer
=> 'duration',
2433 description
=> "Maximum usable CPUs.",
2438 description
=> "The current config lock, if any.",
2443 description
=> "The current configured tags, if any",
2449 my $last_proc_pid_stat;
2451 # get VM status information
2452 # This must be fast and should not block ($full == false)
2453 # We only query KVM using QMP if $full == true (this can be slow)
2455 my ($opt_vmid, $full) = @_;
2459 my $storecfg = PVE
::Storage
::config
();
2461 my $list = vzlist
();
2462 my $defaults = load_defaults
();
2464 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2466 my $cpucount = $cpuinfo->{cpus
} || 1;
2468 foreach my $vmid (keys %$list) {
2469 next if $opt_vmid && ($vmid ne $opt_vmid);
2471 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2473 my $d = { vmid
=> $vmid };
2474 $d->{pid
} = $list->{$vmid}->{pid
};
2476 # fixme: better status?
2477 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2479 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2480 if (defined($size)) {
2481 $d->{disk
} = 0; # no info available
2482 $d->{maxdisk
} = $size;
2488 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2489 * ($conf->{cores
} || $defaults->{cores
});
2490 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2491 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2493 $d->{name
} = $conf->{name
} || "VM $vmid";
2494 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2495 : $defaults->{memory
}*(1024*1024);
2497 if ($conf->{balloon
}) {
2498 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2499 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2500 : $defaults->{shares
};
2511 $d->{diskwrite
} = 0;
2513 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
2515 $d->{serial
} = 1 if conf_has_serial
($conf);
2516 $d->{lock} = $conf->{lock} if $conf->{lock};
2517 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2522 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2523 foreach my $dev (keys %$netdev) {
2524 next if $dev !~ m/^tap([1-9]\d*)i/;
2526 my $d = $res->{$vmid};
2529 $d->{netout
} += $netdev->{$dev}->{receive
};
2530 $d->{netin
} += $netdev->{$dev}->{transmit
};
2533 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
2534 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
2539 my $ctime = gettimeofday
;
2541 foreach my $vmid (keys %$list) {
2543 my $d = $res->{$vmid};
2544 my $pid = $d->{pid
};
2547 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2548 next if !$pstat; # not running
2550 my $used = $pstat->{utime} + $pstat->{stime
};
2552 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2554 if ($pstat->{vsize
}) {
2555 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2558 my $old = $last_proc_pid_stat->{$pid};
2560 $last_proc_pid_stat->{$pid} = {
2568 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2570 if ($dtime > 1000) {
2571 my $dutime = $used - $old->{used
};
2573 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2574 $last_proc_pid_stat->{$pid} = {
2580 $d->{cpu
} = $old->{cpu
};
2584 return $res if !$full;
2586 my $qmpclient = PVE
::QMPClient-
>new();
2588 my $ballooncb = sub {
2589 my ($vmid, $resp) = @_;
2591 my $info = $resp->{'return'};
2592 return if !$info->{max_mem
};
2594 my $d = $res->{$vmid};
2596 # use memory assigned to VM
2597 $d->{maxmem
} = $info->{max_mem
};
2598 $d->{balloon
} = $info->{actual
};
2600 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2601 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2602 $d->{freemem
} = $info->{free_mem
};
2605 $d->{ballooninfo
} = $info;
2608 my $blockstatscb = sub {
2609 my ($vmid, $resp) = @_;
2610 my $data = $resp->{'return'} || [];
2611 my $totalrdbytes = 0;
2612 my $totalwrbytes = 0;
2614 for my $blockstat (@$data) {
2615 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2616 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2618 $blockstat->{device
} =~ s/drive-//;
2619 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2621 $res->{$vmid}->{diskread
} = $totalrdbytes;
2622 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2625 my $statuscb = sub {
2626 my ($vmid, $resp) = @_;
2628 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2629 # this fails if ballon driver is not loaded, so this must be
2630 # the last commnand (following command are aborted if this fails).
2631 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2633 my $status = 'unknown';
2634 if (!defined($status = $resp->{'return'}->{status
})) {
2635 warn "unable to get VM status\n";
2639 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2642 foreach my $vmid (keys %$list) {
2643 next if $opt_vmid && ($vmid ne $opt_vmid);
2644 next if !$res->{$vmid}->{pid
}; # not running
2645 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2648 $qmpclient->queue_execute(undef, 2);
2650 foreach my $vmid (keys %$list) {
2651 next if $opt_vmid && ($vmid ne $opt_vmid);
2652 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2658 sub conf_has_serial
{
2661 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2662 if ($conf->{"serial$i"}) {
2670 sub conf_has_audio
{
2671 my ($conf, $id) = @_;
2674 my $audio = $conf->{"audio$id"};
2675 return undef if !defined($audio);
2677 my $audioproperties = PVE
::JSONSchema
::parse_property_string
($audio_fmt, $audio);
2678 my $audiodriver = $audioproperties->{driver
} // 'spice';
2681 dev
=> $audioproperties->{device
},
2682 dev_id
=> "audiodev$id",
2683 backend
=> $audiodriver,
2684 backend_id
=> "$audiodriver-backend${id}",
2689 my ($audio, $audiopciaddr, $machine_version) = @_;
2693 my $id = $audio->{dev_id
};
2695 if (min_version
($machine_version, 4, 2)) {
2696 $audiodev = ",audiodev=$audio->{backend_id}";
2699 if ($audio->{dev
} eq 'AC97') {
2700 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2701 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
2702 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2703 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2704 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2706 die "unkown audio device '$audio->{dev}', implement me!";
2709 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2714 sub vga_conf_has_spice
{
2717 my $vgaconf = parse_vga
($vga);
2718 my $vgatype = $vgaconf->{type
};
2719 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2726 return get_host_arch
() eq $arch;
2731 return $conf->{arch
} // get_host_arch
();
2734 my $default_machines = {
2739 sub get_vm_machine
{
2740 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2742 my $machine = $forcemachine || $conf->{machine
};
2744 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2746 $machine ||= $default_machines->{$arch};
2747 if ($add_pve_version) {
2748 $kvmversion //= kvm_user_version
();
2749 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
2750 $machine .= "+pve$pvever";
2754 if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2755 # for version-pinned machines that do not include a pve-version (e.g.
2756 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2757 $machine .= '+pve0';
2763 sub get_ovmf_files
($) {
2766 my $ovmf = $OVMF->{$arch}
2767 or die "no OVMF images known for architecture '$arch'\n";
2773 aarch64
=> '/usr/bin/qemu-system-aarch64',
2774 x86_64
=> '/usr/bin/qemu-system-x86_64',
2776 sub get_command_for_arch
($) {
2778 return '/usr/bin/kvm' if is_native
($arch);
2780 my $cmd = $Arch2Qemu->{$arch}
2781 or die "don't know how to emulate architecture '$arch'\n";
2785 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2786 # to use in a QEMU command line (-cpu element), first array_intersect the result
2787 # of query_supported_ with query_understood_. This is necessary because:
2789 # a) query_understood_ returns flags the host cannot use and
2790 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2791 # flags, but CPU settings - with most of them being flags. Those settings
2792 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
2794 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2795 # expensive. If you need the value returned from this, you can get it much
2796 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2797 # $accel being 'kvm' or 'tcg'.
2799 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2800 # changes, automatically populating pmxcfs.
2802 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2803 # since kvm and tcg machines support different flags
2805 sub query_supported_cpu_flags
{
2808 $arch //= get_host_arch
();
2809 my $default_machine = $default_machines->{$arch};
2813 # FIXME: Once this is merged, the code below should work for ARM as well:
2814 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2815 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2818 my $kvm_supported = defined(kvm_version
());
2819 my $qemu_cmd = get_command_for_arch
($arch);
2821 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
2823 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2824 my $query_supported_run_qemu = sub {
2830 '-machine', $default_machine,
2832 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2833 '-mon', 'chardev=qmp,mode=control',
2834 '-pidfile', $pidfile,
2839 push @$cmd, '-accel', 'tcg';
2842 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
2843 die "QEMU flag querying VM exited with code " . $rc if $rc;
2846 my $cmd_result = mon_cmd
(
2848 'query-cpu-model-expansion',
2850 model
=> { name
=> 'host' }
2853 my $props = $cmd_result->{model
}->{props
};
2854 foreach my $prop (keys %$props) {
2855 next if $props->{$prop} ne '1';
2856 # QEMU returns some flags multiple times, with '_', '.' or '-'
2857 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2858 # We only keep those with underscores, to match /proc/cpuinfo
2859 $prop =~ s/\.|-/_/g;
2860 $flags->{$prop} = 1;
2865 # force stop with 10 sec timeout and 'nocheck'
2866 # always stop, even if QMP failed
2867 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
2871 return [ sort keys %$flags ];
2874 # We need to query QEMU twice, since KVM and TCG have different supported flags
2875 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
2876 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
2877 warn "warning: failed querying supported tcg flags: $@\n" if $@;
2879 if ($kvm_supported) {
2880 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
2881 warn "warning: failed querying supported kvm flags: $@\n" if $@;
2888 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2889 my $understood_cpu_flag_dir = "/usr/share/kvm";
2890 sub query_understood_cpu_flags
{
2891 my $arch = get_host_arch
();
2892 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2894 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2897 my $raw = file_get_contents
($filepath);
2898 $raw =~ s/^\s+|\s+$//g;
2899 my @flags = split(/\s+/, $raw);
2904 sub config_to_command
{
2905 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2908 my $globalFlags = [];
2909 my $machineFlags = [];
2914 my $ostype = $conf->{ostype
};
2915 my $winversion = windows_version
($ostype);
2916 my $kvm = $conf->{kvm
};
2917 my $nodename = nodename
();
2919 my $arch = get_vm_arch
($conf);
2920 my $kvm_binary = get_command_for_arch
($arch);
2921 my $kvmver = kvm_user_version
($kvm_binary);
2923 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2924 $kvmver //= "undefined";
2925 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2928 my $add_pve_version = min_version
($kvmver, 4, 1);
2930 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
2931 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, $kvmver);
2932 $kvm //= 1 if is_native
($arch);
2934 $machine_version =~ m/(\d+)\.(\d+)/;
2935 my ($machine_major, $machine_minor) = ($1, $2);
2937 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
2938 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
2939 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
2940 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2941 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
2942 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
2943 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2944 ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2947 # if a specific +pve version is required for a feature, use $version_guard
2948 # instead of min_version to allow machines to be run with the minimum
2950 my $required_pve_version = 0;
2951 my $version_guard = sub {
2952 my ($major, $minor, $pve) = @_;
2953 return 0 if !min_version
($machine_version, $major, $minor, $pve);
2954 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
2955 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
2956 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2961 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2962 if !defined kvm_version
();
2965 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2966 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
2967 my $use_old_bios_files = undef;
2968 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
2970 my $cpuunits = defined($conf->{cpuunits
}) ?
2971 $conf->{cpuunits
} : $defaults->{cpuunits
};
2973 push @$cmd, $kvm_binary;
2975 push @$cmd, '-id', $vmid;
2977 my $vmname = $conf->{name
} || "vm$vmid";
2979 push @$cmd, '-name', $vmname;
2983 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
2984 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2985 push @$cmd, '-mon', "chardev=qmp,mode=control";
2987 if (min_version
($machine_version, 2, 12)) {
2988 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
2989 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
2992 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
2994 push @$cmd, '-daemonize';
2996 if ($conf->{smbios1
}) {
2997 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
2998 if ($smbios_conf->{base64
}) {
2999 # Do not pass base64 flag to qemu
3000 delete $smbios_conf->{base64
};
3001 my $smbios_string = "";
3002 foreach my $key (keys %$smbios_conf) {
3004 if ($key eq "uuid") {
3005 $value = $smbios_conf->{uuid
}
3007 $value = decode_base64
($smbios_conf->{$key});
3009 # qemu accepts any binary data, only commas need escaping by double comma
3011 $smbios_string .= "," . $key . "=" . $value if $value;
3013 push @$cmd, '-smbios', "type=1" . $smbios_string;
3015 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3019 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3020 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3021 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3023 my ($path, $format);
3024 if (my $efidisk = $conf->{efidisk0
}) {
3025 my $d = parse_drive
('efidisk0', $efidisk);
3026 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3027 $format = $d->{format
};
3029 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3030 if (!defined($format)) {
3031 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3032 $format = qemu_img_format
($scfg, $volname);
3036 die "efidisk format must be specified\n"
3037 if !defined($format);
3040 warn "no efidisk configured! Using temporary efivars disk.\n";
3041 $path = "/tmp/$vmid-ovmf.fd";
3042 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3048 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3049 $size_str = ",size=" . (-s
$ovmf_vars);
3052 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3053 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3058 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3059 if (min_version
($machine_version, 4, 0)) {
3060 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3062 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3066 if ($conf->{vmgenid
}) {
3067 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3070 # add usb controllers
3071 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
($conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3072 push @$devices, @usbcontrollers if @usbcontrollers;
3073 my $vga = parse_vga
($conf->{vga
});
3075 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3076 $vga->{type
} = 'qxl' if $qxlnum;
3078 if (!$vga->{type
}) {
3079 if ($arch eq 'aarch64') {
3080 $vga->{type
} = 'virtio';
3081 } elsif (min_version
($machine_version, 2, 9)) {
3082 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3084 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3088 # enable absolute mouse coordinates (needed by vnc)
3090 if (defined($conf->{tablet
})) {
3091 $tablet = $conf->{tablet
};
3093 $tablet = $defaults->{tablet
};
3094 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3095 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3099 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3100 my $kbd = print_keyboarddevice_full
($conf, $arch);
3101 push @$devices, '-device', $kbd if defined($kbd);
3104 # host pci device passthrough
3105 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3106 $vmid, $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type);
3109 my $usb_dev_features = {};
3110 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3112 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
($conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features);
3113 push @$devices, @usbdevices if @usbdevices;
3115 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3116 if (my $path = $conf->{"serial$i"}) {
3117 if ($path eq 'socket') {
3118 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3119 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3120 # On aarch64, serial0 is the UART device. Qemu only allows
3121 # connecting UART devices via the '-serial' command line, as
3122 # the device has a fixed slot on the hardware...
3123 if ($arch eq 'aarch64' && $i == 0) {
3124 push @$devices, '-serial', "chardev:serial$i";
3126 push @$devices, '-device', "isa-serial,chardev=serial$i";
3129 die "no such serial device\n" if ! -c
$path;
3130 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3131 push @$devices, '-device', "isa-serial,chardev=serial$i";
3137 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3138 if (my $path = $conf->{"parallel$i"}) {
3139 die "no such parallel device\n" if ! -c
$path;
3140 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3141 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3142 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3146 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3147 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3148 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3149 push @$devices, @$audio_devs;
3153 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3154 $sockets = $conf->{sockets
} if $conf->{sockets
};
3156 my $cores = $conf->{cores
} || 1;
3158 my $maxcpus = $sockets * $cores;
3160 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3162 my $allowed_vcpus = $cpuinfo->{cpus
};
3164 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3165 if ($allowed_vcpus < $maxcpus);
3167 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3169 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3170 for (my $i = 2; $i <= $vcpus; $i++) {
3171 my $cpustr = print_cpu_device
($conf,$i);
3172 push @$cmd, '-device', $cpustr;
3177 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3179 push @$cmd, '-nodefaults';
3181 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
3183 my $bootindex_hash = {};
3185 foreach my $o (split(//, $bootorder)) {
3186 $bootindex_hash->{$o} = $i*100;
3190 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3192 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3194 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3196 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3197 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3198 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3199 push @$cmd, '-vnc', "unix:$socket,password";
3201 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3202 push @$cmd, '-nographic';
3206 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3207 my $useLocaltime = $conf->{localtime};
3209 if ($winversion >= 5) { # windows
3210 $useLocaltime = 1 if !defined($conf->{localtime});
3212 # use time drift fix when acpi is enabled
3213 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3214 $tdf = 1 if !defined($conf->{tdf
});
3218 if ($winversion >= 6) {
3219 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3220 push @$cmd, '-no-hpet';
3223 push @$rtcFlags, 'driftfix=slew' if $tdf;
3225 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3226 push @$rtcFlags, "base=$conf->{startdate}";
3227 } elsif ($useLocaltime) {
3228 push @$rtcFlags, 'base=localtime';
3232 push @$cmd, '-cpu', $forcecpu;
3234 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3237 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3239 push @$cmd, '-S' if $conf->{freeze
};
3241 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3243 my $guest_agent = parse_guest_agent
($conf);
3245 if ($guest_agent->{enabled
}) {
3246 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3247 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3249 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3250 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3251 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3252 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3253 } elsif ($guest_agent->{type
} eq 'isa') {
3254 push @$devices, '-device', "isa-serial,chardev=qga0";
3258 my $rng = parse_rng
($conf->{rng0
}) if $conf->{rng0
};
3259 if ($rng && &$version_guard(4, 1, 2)) {
3260 check_rng_source
($rng->{source
});
3262 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3263 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3264 my $limiter_str = "";
3266 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3269 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3270 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3271 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3279 for (my $i = 1; $i < $qxlnum; $i++){
3280 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3283 # assume other OS works like Linux
3284 my ($ram, $vram) = ("134217728", "67108864");
3285 if ($vga->{memory
}) {
3286 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3287 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3289 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3290 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3294 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3296 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3297 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3298 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3300 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3301 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3302 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3304 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3305 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3307 my $spice_enhancement = PVE
::JSONSchema
::parse_property_string
($spice_enhancements_fmt, $conf->{spice_enhancements
} // '');
3308 if ($spice_enhancement->{foldersharing
}) {
3309 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3310 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3313 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3314 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming
};
3315 push @$devices, '-spice', "$spice_opts";
3318 # enable balloon by default, unless explicitly disabled
3319 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3320 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3321 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3324 if ($conf->{watchdog
}) {
3325 my $wdopts = parse_watchdog
($conf->{watchdog
});
3326 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3327 my $watchdog = $wdopts->{model
} || 'i6300esb';
3328 push @$devices, '-device', "$watchdog$pciaddr";
3329 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3333 my $scsicontroller = {};
3334 my $ahcicontroller = {};
3335 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3337 # Add iscsi initiator name if available
3338 if (my $initiator = get_initiator_name
()) {
3339 push @$devices, '-iscsi', "initiator-name=$initiator";
3342 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3343 my ($ds, $drive) = @_;
3345 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3346 push @$vollist, $drive->{file
};
3349 # ignore efidisk here, already added in bios/fw handling code above
3350 return if $drive->{interface
} eq 'efidisk';
3352 $use_virtio = 1 if $ds =~ m/^virtio/;
3354 if (drive_is_cdrom
($drive)) {
3355 if ($bootindex_hash->{d
}) {
3356 $drive->{bootindex
} = $bootindex_hash->{d
};
3357 $bootindex_hash->{d
} += 1;
3360 if ($bootindex_hash->{c
}) {
3361 $drive->{bootindex
} = $bootindex_hash->{c
} if $conf->{bootdisk
} && ($conf->{bootdisk
} eq $ds);
3362 $bootindex_hash->{c
} += 1;
3366 if ($drive->{interface
} eq 'virtio'){
3367 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3370 if ($drive->{interface
} eq 'scsi') {
3372 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3374 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3375 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3377 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3378 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3381 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3382 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3383 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3384 } elsif ($drive->{iothread
}) {
3385 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3389 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3390 $queues = ",num_queues=$drive->{queues}";
3393 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3394 $scsicontroller->{$controller}=1;
3397 if ($drive->{interface
} eq 'sata') {
3398 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3399 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3400 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3401 $ahcicontroller->{$controller}=1;
3404 my $drive_cmd = print_drive_commandline_full
($storecfg, $vmid, $drive);
3405 push @$devices, '-drive',$drive_cmd;
3406 push @$devices, '-device', print_drivedevice_full
($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3409 for (my $i = 0; $i < $MAX_NETS; $i++) {
3410 next if !$conf->{"net$i"};
3411 my $d = parse_net
($conf->{"net$i"});
3414 $use_virtio = 1 if $d->{model
} eq 'virtio';
3416 if ($bootindex_hash->{n
}) {
3417 $d->{bootindex
} = $bootindex_hash->{n
};
3418 $bootindex_hash->{n
} += 1;
3421 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, "net$i");
3422 push @$devices, '-netdev', $netdevfull;
3424 my $netdevicefull = print_netdevice_full
($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3425 push @$devices, '-device', $netdevicefull;
3428 if ($conf->{ivshmem
}) {
3429 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3433 $bus = print_pcie_addr
("ivshmem");
3435 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3438 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3439 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3441 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3442 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3445 # pci.4 is nested in pci.1
3446 $bridges->{1} = 1 if $bridges->{4};
3450 if (min_version
($machine_version, 2, 3)) {
3455 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3459 for my $k (sort {$b cmp $a} keys %$bridges) {
3460 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3463 if ($k == 2 && $legacy_igd) {
3466 $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
3468 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3470 # add after -readconfig pve-q35.cfg
3471 splice @$devices, 2, 0, '-device', $devstr;
3473 unshift @$devices, '-device', $devstr if $k > 0;
3478 push @$machineFlags, 'accel=tcg';
3481 my $machine_type_min = $machine_type;
3482 if ($add_pve_version) {
3483 $machine_type_min =~ s/\+pve\d+$//;
3484 $machine_type_min .= "+pve$required_pve_version";
3486 push @$machineFlags, "type=${machine_type_min}";
3488 push @$cmd, @$devices;
3489 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3490 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3491 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3493 if (my $vmstate = $conf->{vmstate
}) {
3494 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3495 push @$vollist, $vmstate;
3496 push @$cmd, '-loadstate', $statepath;
3497 print "activating and using '$vmstate' as vmstate\n";
3501 if ($conf->{args
}) {
3502 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3506 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3509 sub check_rng_source
{
3512 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3513 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3516 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3517 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
3518 # Needs to abort, otherwise QEMU crashes on first rng access.
3519 # Note that rng_current cannot be changed to 'none' manually, so
3520 # once the VM is past this point, it is no longer an issue.
3521 die "Cannot start VM with passed-through RNG device: '/dev/hwrng'"
3522 . " exists, but '$rng_current' is set to 'none'. Ensure that"
3523 . " a compatible hardware-RNG is attached to the host.\n";
3530 my $res = mon_cmd
($vmid, 'query-spice');
3532 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3535 sub vm_devices_list
{
3538 my $res = mon_cmd
($vmid, 'query-pci');
3539 my $devices_to_check = [];
3541 foreach my $pcibus (@$res) {
3542 push @$devices_to_check, @{$pcibus->{devices
}},
3545 while (@$devices_to_check) {
3547 for my $d (@$devices_to_check) {
3548 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3549 next if !$d->{'pci_bridge'};
3551 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3552 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3554 $devices_to_check = $to_check;
3557 my $resblock = mon_cmd
($vmid, 'query-block');
3558 foreach my $block (@$resblock) {
3559 if($block->{device
} =~ m/^drive-(\S+)/){
3564 my $resmice = mon_cmd
($vmid, 'query-mice');
3565 foreach my $mice (@$resmice) {
3566 if ($mice->{name
} eq 'QEMU HID Tablet') {
3567 $devices->{tablet
} = 1;
3572 # for usb devices there is no query-usb
3573 # but we can iterate over the entries in
3574 # qom-list path=/machine/peripheral
3575 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3576 foreach my $per (@$resperipheral) {
3577 if ($per->{name
} =~ m/^usb\d+$/) {
3578 $devices->{$per->{name
}} = 1;
3586 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3588 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3590 my $devices_list = vm_devices_list
($vmid);
3591 return 1 if defined($devices_list->{$deviceid});
3593 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3595 if ($deviceid eq 'tablet') {
3597 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
3599 } elsif ($deviceid eq 'keyboard') {
3601 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
3603 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3605 die "usb hotplug currently not reliable\n";
3606 # since we can't reliably hot unplug all added usb devices
3607 # and usb passthrough disables live migration
3608 # we disable usb hotplugging for now
3609 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device));
3611 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3613 qemu_iothread_add
($vmid, $deviceid, $device);
3615 qemu_driveadd
($storecfg, $vmid, $device);
3616 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3618 qemu_deviceadd
($vmid, $devicefull);
3619 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3621 eval { qemu_drivedel
($vmid, $deviceid); };
3626 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3629 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3630 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3631 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3633 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3635 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3636 qemu_iothread_add
($vmid, $deviceid, $device);
3637 $devicefull .= ",iothread=iothread-$deviceid";
3640 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3641 $devicefull .= ",num_queues=$device->{queues}";
3644 qemu_deviceadd
($vmid, $devicefull);
3645 qemu_deviceaddverify
($vmid, $deviceid);
3647 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3649 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3650 qemu_driveadd
($storecfg, $vmid, $device);
3652 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3653 eval { qemu_deviceadd
($vmid, $devicefull); };
3655 eval { qemu_drivedel
($vmid, $deviceid); };
3660 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3662 return undef if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
3664 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
3665 my $use_old_bios_files = undef;
3666 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3668 my $netdevicefull = print_netdevice_full
($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3669 qemu_deviceadd
($vmid, $netdevicefull);
3671 qemu_deviceaddverify
($vmid, $deviceid);
3672 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
3675 eval { qemu_netdevdel
($vmid, $deviceid); };
3680 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3683 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3684 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3686 qemu_deviceadd
($vmid, $devicefull);
3687 qemu_deviceaddverify
($vmid, $deviceid);
3690 die "can't hotplug device '$deviceid'\n";
3696 # fixme: this should raise exceptions on error!
3697 sub vm_deviceunplug
{
3698 my ($vmid, $conf, $deviceid) = @_;
3700 my $devices_list = vm_devices_list
($vmid);
3701 return 1 if !defined($devices_list->{$deviceid});
3703 die "can't unplug bootdisk" if $conf->{bootdisk
} && $conf->{bootdisk
} eq $deviceid;
3705 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3707 qemu_devicedel
($vmid, $deviceid);
3709 } elsif ($deviceid =~ m/^usb\d+$/) {
3711 die "usb hotplug currently not reliable\n";
3712 # when unplugging usb devices this way,
3713 # there may be remaining usb controllers/hubs
3714 # so we disable it for now
3715 qemu_devicedel
($vmid, $deviceid);
3716 qemu_devicedelverify
($vmid, $deviceid);
3718 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3720 qemu_devicedel
($vmid, $deviceid);
3721 qemu_devicedelverify
($vmid, $deviceid);
3722 qemu_drivedel
($vmid, $deviceid);
3723 qemu_iothread_del
($conf, $vmid, $deviceid);
3725 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3727 qemu_devicedel
($vmid, $deviceid);
3728 qemu_devicedelverify
($vmid, $deviceid);
3729 qemu_iothread_del
($conf, $vmid, $deviceid);
3731 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3733 qemu_devicedel
($vmid, $deviceid);
3734 qemu_drivedel
($vmid, $deviceid);
3735 qemu_deletescsihw
($conf, $vmid, $deviceid);
3737 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3739 qemu_devicedel
($vmid, $deviceid);
3740 qemu_devicedelverify
($vmid, $deviceid);
3741 qemu_netdevdel
($vmid, $deviceid);
3744 die "can't unplug device '$deviceid'\n";
3750 sub qemu_deviceadd
{
3751 my ($vmid, $devicefull) = @_;
3753 $devicefull = "driver=".$devicefull;
3754 my %options = split(/[=,]/, $devicefull);
3756 mon_cmd
($vmid, "device_add" , %options);
3759 sub qemu_devicedel
{
3760 my ($vmid, $deviceid) = @_;
3762 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
3765 sub qemu_iothread_add
{
3766 my($vmid, $deviceid, $device) = @_;
3768 if ($device->{iothread
}) {
3769 my $iothreads = vm_iothreads_list
($vmid);
3770 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3774 sub qemu_iothread_del
{
3775 my($conf, $vmid, $deviceid) = @_;
3777 my $confid = $deviceid;
3778 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3779 $confid = 'scsi' . $1;
3781 my $device = parse_drive
($confid, $conf->{$confid});
3782 if ($device->{iothread
}) {
3783 my $iothreads = vm_iothreads_list
($vmid);
3784 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3788 sub qemu_objectadd
{
3789 my($vmid, $objectid, $qomtype) = @_;
3791 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
3796 sub qemu_objectdel
{
3797 my($vmid, $objectid) = @_;
3799 mon_cmd
($vmid, "object-del", id
=> $objectid);
3805 my ($storecfg, $vmid, $device) = @_;
3807 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device);
3808 $drive =~ s/\\/\\\\/g;
3809 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
3811 # If the command succeeds qemu prints: "OK
"
3812 return 1 if $ret =~ m/OK/s;
3814 die "adding drive failed
: $ret\n";
3818 my($vmid, $deviceid) = @_;
3820 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
3823 return 1 if $ret eq "";
3825 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3826 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3828 die "deleting drive
$deviceid failed
: $ret\n";
3831 sub qemu_deviceaddverify {
3832 my ($vmid, $deviceid) = @_;
3834 for (my $i = 0; $i <= 5; $i++) {
3835 my $devices_list = vm_devices_list($vmid);
3836 return 1 if defined($devices_list->{$deviceid});
3840 die "error on hotplug device
'$deviceid'\n";
3844 sub qemu_devicedelverify {
3845 my ($vmid, $deviceid) = @_;
3847 # need to verify that the device is correctly removed as device_del
3848 # is async and empty return is not reliable
3850 for (my $i = 0; $i <= 5; $i++) {
3851 my $devices_list = vm_devices_list($vmid);
3852 return 1 if !defined($devices_list->{$deviceid});
3856 die "error on hot-unplugging device
'$deviceid'\n";
3859 sub qemu_findorcreatescsihw {
3860 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3862 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3864 my $scsihwid="$controller_prefix$controller";
3865 my $devices_list = vm_devices_list($vmid);
3867 if(!defined($devices_list->{$scsihwid})) {
3868 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3874 sub qemu_deletescsihw {
3875 my ($conf, $vmid, $opt) = @_;
3877 my $device = parse_drive($opt, $conf->{$opt});
3879 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3880 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
3884 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3886 my $devices_list = vm_devices_list($vmid);
3887 foreach my $opt (keys %{$devices_list}) {
3888 if (is_valid_drivename($opt)) {
3889 my $drive = parse_drive($opt, $conf->{$opt});
3890 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3896 my $scsihwid="scsihw
$controller";
3898 vm_deviceunplug($vmid, $conf, $scsihwid);
3903 sub qemu_add_pci_bridge {
3904 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3910 print_pci_addr($device, $bridges, $arch, $machine_type);
3912 while (my ($k, $v) = each %$bridges) {
3915 return 1 if !defined($bridgeid) || $bridgeid < 1;
3917 my $bridge = "pci
.$bridgeid";
3918 my $devices_list = vm_devices_list($vmid);
3920 if (!defined($devices_list->{$bridge})) {
3921 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3927 sub qemu_set_link_status {
3928 my ($vmid, $device, $up) = @_;
3930 mon_cmd($vmid, "set_link
", name => $device,
3931 up => $up ? JSON::true : JSON::false);
3934 sub qemu_netdevadd {
3935 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
3937 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
3938 my %options = split(/[=,]/, $netdev);
3940 if (defined(my $vhost = $options{vhost})) {
3941 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
3944 if (defined(my $queues = $options{queues})) {
3945 $options{queues} = $queues + 0;
3948 mon_cmd($vmid, "netdev_add
", %options);
3952 sub qemu_netdevdel {
3953 my ($vmid, $deviceid) = @_;
3955 mon_cmd($vmid, "netdev_del
", id => $deviceid);
3958 sub qemu_usb_hotplug {
3959 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3963 # remove the old one first
3964 vm_deviceunplug($vmid, $conf, $deviceid);
3966 # check if xhci controller is necessary and available
3967 if ($device->{usb3}) {
3969 my $devicelist = vm_devices_list($vmid);
3971 if (!$devicelist->{xhci}) {
3972 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
3973 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
3976 my $d = parse_usb_device($device->{host});
3977 $d->{usb3} = $device->{usb3};
3980 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
3983 sub qemu_cpu_hotplug {
3984 my ($vmid, $conf, $vcpus) = @_;
3986 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
3989 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3990 $sockets = $conf->{sockets} if $conf->{sockets};
3991 my $cores = $conf->{cores} || 1;
3992 my $maxcpus = $sockets * $cores;
3994 $vcpus = $maxcpus if !$vcpus;
3996 die "you can
't add more vcpus than maxcpus\n"
3997 if $vcpus > $maxcpus;
3999 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4001 if ($vcpus < $currentvcpus) {
4003 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4005 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4006 qemu_devicedel($vmid, "cpu$i");
4008 my $currentrunningvcpus = undef;
4010 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4011 last if scalar(@{$currentrunningvcpus}) == $i-1;
4012 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4016 #update conf after each succesfull cpu unplug
4017 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4018 PVE::QemuConfig->write_config($vmid, $conf);
4021 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4027 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4028 die "vcpus in running vm does not match its configuration\n"
4029 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4031 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4033 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4034 my $cpustr = print_cpu_device($conf, $i);
4035 qemu_deviceadd($vmid, $cpustr);
4038 my $currentrunningvcpus = undef;
4040 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4041 last if scalar(@{$currentrunningvcpus}) == $i;
4042 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4046 #update conf after each succesfull cpu hotplug
4047 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4048 PVE::QemuConfig->write_config($vmid, $conf);
4052 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4053 mon_cmd($vmid, "cpu-add", id => int($i));
4058 sub qemu_block_set_io_throttle {
4059 my ($vmid, $deviceid,
4060 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4061 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4062 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4063 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4065 return if !check_running($vmid) ;
4067 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4069 bps_rd => int($bps_rd),
4070 bps_wr => int($bps_wr),
4072 iops_rd => int($iops_rd),
4073 iops_wr => int($iops_wr),
4074 bps_max => int($bps_max),
4075 bps_rd_max => int($bps_rd_max),
4076 bps_wr_max => int($bps_wr_max),
4077 iops_max => int($iops_max),
4078 iops_rd_max => int($iops_rd_max),
4079 iops_wr_max => int($iops_wr_max),
4080 bps_max_length => int($bps_max_length),
4081 bps_rd_max_length => int($bps_rd_max_length),
4082 bps_wr_max_length => int($bps_wr_max_length),
4083 iops_max_length => int($iops_max_length),
4084 iops_rd_max_length => int($iops_rd_max_length),
4085 iops_wr_max_length => int($iops_wr_max_length),
4090 # old code, only used to shutdown old VM after update
4092 my ($fh, $timeout) = @_;
4094 my $sel = new IO::Select;
4101 while (scalar (@ready = $sel->can_read($timeout))) {
4103 if ($count = $fh->sysread($buf, 8192)) {
4104 if ($buf =~ /^(.*)\(qemu\) $/s) {
4111 if (!defined($count)) {
4118 die "monitor read timeout\n" if !scalar(@ready);
4123 sub qemu_block_resize {
4124 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4126 my $running = check_running($vmid);
4128 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4130 return if !$running;
4132 my $padding = (1024 - $size % 1024) % 1024;
4133 $size = $size + $padding;
4135 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4139 sub qemu_volume_snapshot {
4140 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4142 my $running = check_running($vmid);
4144 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4145 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4147 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4151 sub qemu_volume_snapshot_delete {
4152 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4154 my $running = check_running($vmid);
4159 my $conf = PVE::QemuConfig->load_config($vmid);
4160 PVE::QemuConfig->foreach_volume($conf, sub {
4161 my ($ds, $drive) = @_;
4162 $running = 1 if $drive->{file} eq $volid;
4166 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4167 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4169 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4173 sub set_migration_caps {
4179 "auto-converge" => 1,
4181 "x-rdma-pin-all" => 0,
4186 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4188 for my $supported_capability (@$supported_capabilities) {
4190 capability => $supported_capability->{capability},
4191 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4195 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4199 my ($conf, $func, @param) = @_;
4203 my $test_volid = sub {
4204 my ($key, $drive, $snapname) = @_;
4206 my $volid = $drive->{file};
4209 $volhash->{$volid}->{cdrom} //= 1;
4210 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4212 my $replicate = $drive->{replicate} // 1;
4213 $volhash->{$volid}->{replicate} //= 0;
4214 $volhash->{$volid}->{replicate} = 1 if $replicate;
4216 $volhash->{$volid}->{shared} //= 0;
4217 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4219 $volhash->{$volid}->{referenced_in_config} //= 0;
4220 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4222 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4223 if defined($snapname);
4225 my $size = $drive->{size};
4226 $volhash->{$volid}->{size} //= $size if $size;
4228 $volhash->{$volid}->{is_vmstate} //= 0;
4229 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4231 $volhash->{$volid}->{is_unused} //= 0;
4232 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4235 my $include_opts = {
4236 extra_keys => ['vmstate
'],
4237 include_unused => 1,
4240 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4241 foreach my $snapname (keys %{$conf->{snapshots}}) {
4242 my $snap = $conf->{snapshots}->{$snapname};
4243 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4246 foreach my $volid (keys %$volhash) {
4247 &$func($volid, $volhash->{$volid}, @param);
4251 my $fast_plug_option = {
4259 'vmstatestorage
' => 1,
4264 # hotplug changes in [PENDING]
4265 # $selection hash can be used to only apply specified options, for
4266 # example: { cores => 1 } (only apply changed 'cores
')
4267 # $errors ref is used to return error messages
4268 sub vmconfig_hotplug_pending {
4269 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4271 my $defaults = load_defaults();
4272 my $arch = get_vm_arch($conf);
4273 my $machine_type = get_vm_machine($conf, undef, $arch);
4275 # commit values which do not have any impact on running VM first
4276 # Note: those option cannot raise errors, we we do not care about
4277 # $selection and always apply them.
4279 my $add_error = sub {
4280 my ($opt, $msg) = @_;
4281 $errors->{$opt} = "hotplug problem - $msg";
4285 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4286 if ($fast_plug_option->{$opt}) {
4287 $conf->{$opt} = $conf->{pending}->{$opt};
4288 delete $conf->{pending}->{$opt};
4294 PVE::QemuConfig->write_config($vmid, $conf);
4297 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4299 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4300 foreach my $opt (sort keys %$pending_delete_hash) {
4301 next if $selection && !$selection->{$opt};
4302 my $force = $pending_delete_hash->{$opt}->{force};
4304 if ($opt eq 'hotplug
') {
4305 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4306 } elsif ($opt eq 'tablet
') {
4307 die "skip\n" if !$hotplug_features->{usb};
4308 if ($defaults->{tablet}) {
4309 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4310 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4311 if $arch eq 'aarch64
';
4313 vm_deviceunplug($vmid, $conf, 'tablet
');
4314 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4316 } elsif ($opt =~ m/^usb\d+/) {
4318 # since we cannot reliably hot unplug usb devices
4319 # we are disabling it
4320 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4321 vm_deviceunplug($vmid, $conf, $opt);
4322 } elsif ($opt eq 'vcpus
') {
4323 die "skip\n" if !$hotplug_features->{cpu};
4324 qemu_cpu_hotplug($vmid, $conf, undef);
4325 } elsif ($opt eq 'balloon
') {
4326 # enable balloon device is not hotpluggable
4327 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4328 # here we reset the ballooning value to memory
4329 my $balloon = $conf->{memory} || $defaults->{memory};
4330 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4331 } elsif ($fast_plug_option->{$opt}) {
4333 } elsif ($opt =~ m/^net(\d+)$/) {
4334 die "skip\n" if !$hotplug_features->{network};
4335 vm_deviceunplug($vmid, $conf, $opt);
4336 } elsif (is_valid_drivename($opt)) {
4337 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4338 vm_deviceunplug($vmid, $conf, $opt);
4339 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4340 } elsif ($opt =~ m/^memory$/) {
4341 die "skip\n" if !$hotplug_features->{memory};
4342 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4343 } elsif ($opt eq 'cpuunits
') {
4344 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4345 } elsif ($opt eq 'cpulimit
') {
4346 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4352 &$add_error($opt, $err) if $err ne "skip\n";
4354 delete $conf->{$opt};
4355 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4359 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4360 $apply_pending_cloudinit = sub {
4361 return if $apply_pending_cloudinit_done; # once is enough
4362 $apply_pending_cloudinit_done = 1; # once is enough
4364 my ($key, $value) = @_;
4366 my @cloudinit_opts = keys %$confdesc_cloudinit;
4367 foreach my $opt (keys %{$conf->{pending}}) {
4368 next if !grep { $_ eq $opt } @cloudinit_opts;
4369 $conf->{$opt} = delete $conf->{pending}->{$opt};
4372 my $new_conf = { %$conf };
4373 $new_conf->{$key} = $value;
4374 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4377 foreach my $opt (keys %{$conf->{pending}}) {
4378 next if $selection && !$selection->{$opt};
4379 my $value = $conf->{pending}->{$opt};
4381 if ($opt eq 'hotplug
') {
4382 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4383 } elsif ($opt eq 'tablet
') {
4384 die "skip\n" if !$hotplug_features->{usb};
4386 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4387 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4388 if $arch eq 'aarch64
';
4389 } elsif ($value == 0) {
4390 vm_deviceunplug($vmid, $conf, 'tablet
');
4391 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4393 } elsif ($opt =~ m/^usb\d+$/) {
4395 # since we cannot reliably hot unplug usb devices
4396 # we are disabling it
4397 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4398 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4399 die "skip\n" if !$d;
4400 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4401 } elsif ($opt eq 'vcpus
') {
4402 die "skip\n" if !$hotplug_features->{cpu};
4403 qemu_cpu_hotplug($vmid, $conf, $value);
4404 } elsif ($opt eq 'balloon
') {
4405 # enable/disable balloning device is not hotpluggable
4406 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4407 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4408 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4410 # allow manual ballooning if shares is set to zero
4411 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4412 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4413 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4415 } elsif ($opt =~ m/^net(\d+)$/) {
4416 # some changes can be done without hotplug
4417 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4418 $vmid, $opt, $value, $arch, $machine_type);
4419 } elsif (is_valid_drivename($opt)) {
4420 die "skip\n" if $opt eq 'efidisk0
';
4421 # some changes can be done without hotplug
4422 my $drive = parse_drive($opt, $value);
4423 if (drive_is_cloudinit($drive)) {
4424 &$apply_pending_cloudinit($opt, $value);
4426 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4427 $vmid, $opt, $value, $arch, $machine_type);
4428 } elsif ($opt =~ m/^memory$/) { #dimms
4429 die "skip\n" if !$hotplug_features->{memory};
4430 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4431 } elsif ($opt eq 'cpuunits
') {
4432 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4433 } elsif ($opt eq 'cpulimit
') {
4434 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4435 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4437 die "skip\n"; # skip non-hot-pluggable options
4441 &$add_error($opt, $err) if $err ne "skip\n";
4443 $conf->{$opt} = $value;
4444 delete $conf->{pending}->{$opt};
4448 PVE::QemuConfig->write_config($vmid, $conf);
4451 sub try_deallocate_drive {
4452 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4454 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4455 my $volid = $drive->{file};
4456 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4457 my $sid = PVE::Storage::parse_volume_id($volid);
4458 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4460 # check if the disk is really unused
4461 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4462 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4463 PVE::Storage::vdisk_free($storecfg, $volid);
4466 # If vm is not owner of this disk remove from config
4474 sub vmconfig_delete_or_detach_drive {
4475 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4477 my $drive = parse_drive($opt, $conf->{$opt});
4479 my $rpcenv = PVE::RPCEnvironment::get();
4480 my $authuser = $rpcenv->get_user();
4483 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4484 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4486 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4492 sub vmconfig_apply_pending {
4493 my ($vmid, $conf, $storecfg, $errors) = @_;
4495 my $add_apply_error = sub {
4496 my ($opt, $msg) = @_;
4497 my $err_msg = "unable to apply pending change $opt : $msg";
4498 $errors->{$opt} = $err_msg;
4504 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4505 foreach my $opt (sort keys %$pending_delete_hash) {
4506 my $force = $pending_delete_hash->{$opt}->{force};
4508 if ($opt =~ m/^unused/) {
4509 die "internal error";
4510 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4511 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4515 $add_apply_error->($opt, $err);
4517 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4518 delete $conf->{$opt};
4522 PVE::QemuConfig->cleanup_pending($conf);
4524 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4525 next if $opt eq 'delete'; # just to be sure
4527 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4528 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4532 $add_apply_error->($opt, $err);
4534 $conf->{$opt} = delete $conf->{pending}->{$opt};
4538 # write all changes at once to avoid unnecessary i/o
4539 PVE::QemuConfig->write_config($vmid, $conf);
4542 sub vmconfig_update_net {
4543 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4545 my $newnet = parse_net($value);
4547 if ($conf->{$opt}) {
4548 my $oldnet = parse_net($conf->{$opt});
4550 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4551 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4552 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4553 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4555 # for non online change, we try to hot-unplug
4556 die "skip\n" if !$hotplug;
4557 vm_deviceunplug($vmid, $conf, $opt);
4560 die "internal error" if $opt !~ m/net(\d+)/;
4561 my $iface = "tap${vmid}i$1";
4563 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4564 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4565 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4566 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4567 PVE::Network::tap_unplug($iface);
4570 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4572 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4574 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4575 # Rate can be applied on its own but any change above needs to
4576 # include the rate in tap_plug since OVS resets everything.
4577 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4580 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4581 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4589 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4595 sub vmconfig_update_disk {
4596 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4598 my $drive = parse_drive($opt, $value);
4600 if ($conf->{$opt}) {
4602 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
4604 my $media = $drive->{media} || 'disk
';
4605 my $oldmedia = $old_drive->{media} || 'disk
';
4606 die "unable to change media type\n" if $media ne $oldmedia;
4608 if (!drive_is_cdrom($old_drive)) {
4610 if ($drive->{file} ne $old_drive->{file}) {
4612 die "skip\n" if !$hotplug;
4614 # unplug and register as unused
4615 vm_deviceunplug($vmid, $conf, $opt);
4616 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4619 # update existing disk
4621 # skip non hotpluggable value
4622 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4623 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4624 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4625 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4626 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4631 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4632 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4633 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4634 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4635 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4636 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4637 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4638 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4639 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4640 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4641 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4642 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4643 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4644 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4645 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4646 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4647 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4648 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4650 qemu_block_set_io_throttle($vmid,"drive-$opt",
4651 ($drive->{mbps} || 0)*1024*1024,
4652 ($drive->{mbps_rd} || 0)*1024*1024,
4653 ($drive->{mbps_wr} || 0)*1024*1024,
4654 $drive->{iops} || 0,
4655 $drive->{iops_rd} || 0,
4656 $drive->{iops_wr} || 0,
4657 ($drive->{mbps_max} || 0)*1024*1024,
4658 ($drive->{mbps_rd_max} || 0)*1024*1024,
4659 ($drive->{mbps_wr_max} || 0)*1024*1024,
4660 $drive->{iops_max} || 0,
4661 $drive->{iops_rd_max} || 0,
4662 $drive->{iops_wr_max} || 0,
4663 $drive->{bps_max_length} || 1,
4664 $drive->{bps_rd_max_length} || 1,
4665 $drive->{bps_wr_max_length} || 1,
4666 $drive->{iops_max_length} || 1,
4667 $drive->{iops_rd_max_length} || 1,
4668 $drive->{iops_wr_max_length} || 1);
4677 if ($drive->{file} eq 'none
') {
4678 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4679 if (drive_is_cloudinit($old_drive)) {
4680 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4683 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4685 # force eject if locked
4686 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4689 mon_cmd($vmid, "blockdev-change-medium",
4690 id => "$opt", filename => "$path");
4699 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4701 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4702 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4705 # called in locked context by incoming migration
4706 sub vm_migrate_get_nbd_disks {
4707 my ($storecfg, $conf, $replicated_volumes) = @_;
4709 my $local_volumes = {};
4710 PVE::QemuConfig->foreach_volume($conf, sub {
4711 my ($ds, $drive) = @_;
4713 return if drive_is_cdrom($drive);
4715 my $volid = $drive->{file};
4719 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4721 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4722 return if $scfg->{shared};
4724 # replicated disks re-use existing state via bitmap
4725 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4726 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4728 return $local_volumes;
4731 # called in locked context by incoming migration
4732 sub vm_migrate_alloc_nbd_disks {
4733 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4738 foreach my $opt (sort keys %$source_volumes) {
4739 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4741 if ($use_existing) {
4742 $nbd->{$opt}->{drivestr} = print_drive($drive);
4743 $nbd->{$opt}->{volid} = $volid;
4744 $nbd->{$opt}->{replicated} = 1;
4748 # If a remote storage is specified and the format of the original
4749 # volume is not available there, fall back to the default format.
4750 # Otherwise use the same format as the original.
4751 if (!$storagemap->{identity}) {
4752 $storeid = map_storage($storagemap, $storeid);
4753 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4754 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4755 my $fileFormat = qemu_img_format($scfg, $volname);
4756 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4758 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4759 $format = qemu_img_format($scfg, $volname);
4762 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4763 my $newdrive = $drive;
4764 $newdrive->{format} = $format;
4765 $newdrive->{file} = $newvolid;
4766 my $drivestr = print_drive($newdrive);
4767 $nbd->{$opt}->{drivestr} = $drivestr;
4768 $nbd->{$opt}->{volid} = $newvolid;
4774 # see vm_start_nolock for parameters, additionally:
4776 # storagemap = parsed storage map for allocating NBD disks
4778 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4780 return PVE::QemuConfig->lock_config($vmid, sub {
4781 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4783 die "you can't start a vm
if it
's a template\n" if PVE::QemuConfig->is_template($conf);
4785 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended
');
4787 PVE::QemuConfig->check_lock($conf)
4788 if !($params->{skiplock} || $has_suspended_lock);
4790 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4792 die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4794 if (my $storagemap = $migrate_opts->{storagemap}) {
4795 my $replicated = $migrate_opts->{replicated_volumes};
4796 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4797 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4799 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4800 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4804 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4810 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
4811 # skiplock => 0/1, skip checking for config lock
4812 # forcemachine => to force Qemu machine (rollback/migration)
4813 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
4814 # timeout => in seconds
4815 # paused => start VM in paused state (backup)
4816 # resume => resume from hibernation
4818 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4819 # migratedfrom => source node
4820 # spice_ticket => used for spice migration, passed via tunnel/stdin
4821 # network => CIDR of migration network
4822 # type => secure/insecure - tunnel over encrypted connection or plain-text
4823 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
4824 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4825 sub vm_start_nolock {
4826 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4828 my $statefile = $params->{statefile};
4829 my $resume = $params->{resume};
4831 my $migratedfrom = $migrate_opts->{migratedfrom};
4832 my $migration_type = $migrate_opts->{type};
4836 # clean up leftover reboot request files
4837 eval { clear_reboot_request($vmid); };
4840 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4841 vmconfig_apply_pending($vmid, $conf, $storecfg);
4842 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4845 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4847 my $defaults = load_defaults();
4849 # set environment variable useful inside network script
4850 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4852 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
4854 my $forcemachine = $params->{forcemachine};
4855 my $forcecpu = $params->{forcecpu};
4857 # enforce machine and CPU type on suspended vm to ensure HW compatibility
4858 $forcemachine = $conf->{runningmachine};
4859 $forcecpu = $conf->{runningcpu};
4860 print "Resuming suspended VM\n";
4863 my ($cmd, $vollist, $spice_port) =
4864 config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4867 my $get_migration_ip = sub {
4868 my ($nodename) = @_;
4870 return $migration_ip if defined($migration_ip);
4872 my $cidr = $migrate_opts->{network};
4874 if (!defined($cidr)) {
4875 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
4876 $cidr = $dc_conf->{migration}->{network};
4879 if (defined($cidr)) {
4880 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4882 die "could not get IP: no address configured on local " .
4883 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4885 die "could not get IP: multiple addresses configured on local " .
4886 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4888 $migration_ip = @$ips[0];
4891 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4892 if !defined($migration_ip);
4894 return $migration_ip;
4899 if ($statefile eq 'tcp
') {
4900 my $localip = "localhost";
4901 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
4902 my $nodename = nodename();
4904 if (!defined($migration_type)) {
4905 if (defined($datacenterconf->{migration}->{type})) {
4906 $migration_type = $datacenterconf->{migration}->{type};
4908 $migration_type = 'secure
';
4912 if ($migration_type eq 'insecure
') {
4913 $localip = $get_migration_ip->($nodename);
4914 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4917 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4918 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4919 $migrate_uri = "tcp:${localip}:${migrate_port}";
4920 push @$cmd, '-incoming
', $migrate_uri;
4923 } elsif ($statefile eq 'unix
') {
4924 # should be default for secure migrations as a ssh TCP forward
4925 # tunnel is not deterministic reliable ready and fails regurarly
4926 # to set up in time, so use UNIX socket forwards
4927 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4928 unlink $socket_addr;
4930 $migrate_uri = "unix:$socket_addr";
4932 push @$cmd, '-incoming
', $migrate_uri;
4935 } elsif (-e $statefile) {
4936 push @$cmd, '-loadstate
', $statefile;
4938 my $statepath = PVE::Storage::path($storecfg, $statefile);
4939 push @$vollist, $statefile;
4940 push @$cmd, '-loadstate
', $statepath;
4942 } elsif ($params->{paused}) {
4947 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
4948 my $d = parse_hostpci($conf->{"hostpci$i"});
4950 my $pcidevices = $d->{pciid};
4951 foreach my $pcidevice (@$pcidevices) {
4952 my $pciid = $pcidevice->{id};
4954 my $info = PVE::SysFSTools::pci_device_info("$pciid");
4955 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4956 die "no pci device info for device '$pciid'\n" if !$info;
4959 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4960 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4962 die "can't unbind
/bind pci group to vfio
'$pciid'\n"
4963 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4964 die "can
't reset pci device '$pciid'\n"
4965 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4970 PVE::Storage::activate_volumes($storecfg, $vollist);
4973 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
4974 outfunc => sub {}, errfunc => sub {});
4976 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
4977 # timeout should be more than enough here...
4978 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4980 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4981 : $defaults->{cpuunits};
4983 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
4985 timeout => $statefile ? undef : $start_timeout,
4990 # when migrating, prefix QEMU output so other side can pick up any
4991 # errors that might occur and show the user
4992 if ($migratedfrom) {
4993 $run_params{quiet} = 1;
4994 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
4998 Slice => 'qemu
.slice
',
5000 CPUShares => $cpuunits
5003 if (my $cpulimit = $conf->{cpulimit}) {
5004 $properties{CPUQuota} = int($cpulimit * 100);
5006 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5008 my $run_qemu = sub {
5009 PVE::Tools::run_fork sub {
5010 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5012 my $exitcode = run_command($cmd, %run_params);
5013 die "QEMU exited with code $exitcode\n" if $exitcode;
5017 if ($conf->{hugepages}) {
5020 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5021 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5023 PVE::QemuServer::Memory::hugepages_mount();
5024 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5026 eval { $run_qemu->() };
5028 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5032 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5034 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5037 eval { $run_qemu->() };
5041 # deactivate volumes if start fails
5042 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5043 die "start failed: $err";
5046 print "migration listens on $migrate_uri\n" if $migrate_uri;
5047 $res->{migrate_uri} = $migrate_uri;
5049 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5050 eval { mon_cmd($vmid, "cont"); };
5054 #start nbd server for storage migration
5055 if (my $nbd = $migrate_opts->{nbd}) {
5056 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5058 my $migrate_storage_uri;
5059 # nbd_protocol_version > 0 for unix socket support
5060 if ($nbd_protocol_version > 0 && $migration_type eq 'secure
') {
5061 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5062 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix
', data => { path => $socket_path } } );
5063 $migrate_storage_uri = "nbd:unix:$socket_path";
5065 my $nodename = nodename();
5066 my $localip = $get_migration_ip->($nodename);
5067 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5068 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5070 mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet
', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5071 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5072 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5075 $res->{migrate_storage_uri} = $migrate_storage_uri;
5077 foreach my $opt (sort keys %$nbd) {
5078 my $drivestr = $nbd->{$opt}->{drivestr};
5079 my $volid = $nbd->{$opt}->{volid};
5080 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5081 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5082 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5083 print "re-using replicated volume: $opt - $volid\n"
5084 if $nbd->{$opt}->{replicated};
5086 $res->{drives}->{$opt} = $nbd->{$opt};
5087 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5091 if ($migratedfrom) {
5093 set_migration_caps($vmid);
5098 print "spice listens on port $spice_port\n";
5099 $res->{spice_port} = $spice_port;
5100 if ($migrate_opts->{spice_ticket}) {
5101 mon_cmd($vmid, "set_password", protocol => 'spice
', password => $migrate_opts->{spice_ticket});
5102 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5107 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5108 if !$statefile && $conf->{balloon};
5110 foreach my $opt (keys %$conf) {
5111 next if $opt !~ m/^net\d+$/;
5112 my $nicconf = parse_net($conf->{$opt});
5113 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5117 mon_cmd($vmid, 'qom-set
',
5118 path => "machine/peripheral/balloon0",
5119 property => "guest-stats-polling-interval",
5120 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5123 print "Resumed VM, removing state\n";
5124 if (my $vmstate = $conf->{vmstate}) {
5125 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5126 PVE::Storage::vdisk_free($storecfg, $vmstate);
5128 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5129 PVE
::QemuConfig-
>write_config($vmid, $conf);
5132 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5137 sub vm_commandline
{
5138 my ($storecfg, $vmid, $snapname) = @_;
5140 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5145 my $snapshot = $conf->{snapshots
}->{$snapname};
5146 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5148 # check for machine or CPU overrides in snapshot
5149 $forcemachine = $snapshot->{runningmachine
};
5150 $forcecpu = $snapshot->{runningcpu
};
5152 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5157 my $defaults = load_defaults
();
5159 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults,
5160 $forcemachine, $forcecpu);
5162 return PVE
::Tools
::cmd2string
($cmd);
5166 my ($vmid, $skiplock) = @_;
5168 PVE
::QemuConfig-
>lock_config($vmid, sub {
5170 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5172 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5174 mon_cmd
($vmid, "system_reset");
5178 sub get_vm_volumes
{
5182 foreach_volid
($conf, sub {
5183 my ($volid, $attr) = @_;
5185 return if $volid =~ m
|^/|;
5187 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5190 push @$vollist, $volid;
5196 sub vm_stop_cleanup
{
5197 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5202 my $vollist = get_vm_volumes
($conf);
5203 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5206 foreach my $ext (qw(mon qmp pid vnc qga)) {
5207 unlink "/var/run/qemu-server/${vmid}.$ext";
5210 if ($conf->{ivshmem
}) {
5211 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5212 # just delete it for now, VMs which have this already open do not
5213 # are affected, but new VMs will get a separated one. If this
5214 # becomes an issue we either add some sort of ref-counting or just
5215 # add a "don't delete on stop" flag to the ivshmem format.
5216 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5219 foreach my $key (keys %$conf) {
5220 next if $key !~ m/^hostpci(\d+)$/;
5221 my $hostpciindex = $1;
5222 my $d = parse_hostpci
($conf->{$key});
5223 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5225 foreach my $pci (@{$d->{pciid
}}) {
5226 my $pciid = $pci->{id
};
5227 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5231 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5233 warn $@ if $@; # avoid errors - just warn
5236 # call only in locked context
5238 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5240 my $pid = check_running
($vmid, $nocheck);
5245 $conf = PVE
::QemuConfig-
>load_config($vmid);
5246 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5247 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5248 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5249 $timeout = $opts->{down
} if $opts->{down
};
5251 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5256 if (defined($conf) && parse_guest_agent
($conf)->{enabled
}) {
5257 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5259 mon_cmd
($vmid, "system_powerdown");
5262 mon_cmd
($vmid, "quit");
5268 $timeout = 60 if !defined($timeout);
5271 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5276 if ($count >= $timeout) {
5278 warn "VM still running - terminating now with SIGTERM\n";
5281 die "VM quit/powerdown failed - got timeout\n";
5284 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5288 if (!check_running
($vmid, $nocheck)) {
5289 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5293 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5296 die "VM quit/powerdown failed\n";
5304 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5309 if ($count >= $timeout) {
5310 warn "VM still running - terminating now with SIGKILL\n";
5315 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5318 # Note: use $nocheck to skip tests if VM configuration file exists.
5319 # We need that when migration VMs to other nodes (files already moved)
5320 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5322 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5324 $force = 1 if !defined($force) && !$shutdown;
5327 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5328 kill 15, $pid if $pid;
5329 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5330 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5334 PVE
::QemuConfig-
>lock_config($vmid, sub {
5335 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5340 my ($vmid, $timeout) = @_;
5342 PVE
::QemuConfig-
>lock_config($vmid, sub {
5345 # only reboot if running, as qmeventd starts it again on a stop event
5346 return if !check_running
($vmid);
5348 create_reboot_request
($vmid);
5350 my $storecfg = PVE
::Storage
::config
();
5351 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5355 # avoid that the next normal shutdown will be confused for a reboot
5356 clear_reboot_request
($vmid);
5362 # note: if using the statestorage parameter, the caller has to check privileges
5364 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5371 PVE
::QemuConfig-
>lock_config($vmid, sub {
5373 $conf = PVE
::QemuConfig-
>load_config($vmid);
5375 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5376 PVE
::QemuConfig-
>check_lock($conf)
5377 if !($skiplock || $is_backing_up);
5379 die "cannot suspend to disk during backup\n"
5380 if $is_backing_up && $includestate;
5382 if ($includestate) {
5383 $conf->{lock} = 'suspending';
5384 my $date = strftime
("%Y-%m-%d", localtime(time()));
5385 $storecfg = PVE
::Storage
::config
();
5386 if (!$statestorage) {
5387 $statestorage = find_vmstate_storage
($conf, $storecfg);
5388 # check permissions for the storage
5389 my $rpcenv = PVE
::RPCEnvironment
::get
();
5390 if ($rpcenv->{type
} ne 'cli') {
5391 my $authuser = $rpcenv->get_user();
5392 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5397 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5398 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5399 PVE
::QemuConfig-
>write_config($vmid, $conf);
5401 mon_cmd
($vmid, "stop");
5405 if ($includestate) {
5407 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5410 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5412 my $state = mon_cmd
($vmid, "query-savevm");
5413 if (!$state->{status
}) {
5414 die "savevm not active\n";
5415 } elsif ($state->{status
} eq 'active') {
5418 } elsif ($state->{status
} eq 'completed') {
5419 print "State saved, quitting\n";
5421 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5422 die "query-savevm failed with error '$state->{error}'\n"
5424 die "query-savevm returned status '$state->{status}'\n";
5430 PVE
::QemuConfig-
>lock_config($vmid, sub {
5431 $conf = PVE
::QemuConfig-
>load_config($vmid);
5433 # cleanup, but leave suspending lock, to indicate something went wrong
5435 mon_cmd
($vmid, "savevm-end");
5436 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5437 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5438 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5439 PVE
::QemuConfig-
>write_config($vmid, $conf);
5445 die "lock changed unexpectedly\n"
5446 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5448 mon_cmd
($vmid, "quit");
5449 $conf->{lock} = 'suspended';
5450 PVE
::QemuConfig-
>write_config($vmid, $conf);
5456 my ($vmid, $skiplock, $nocheck) = @_;
5458 PVE
::QemuConfig-
>lock_config($vmid, sub {
5459 my $res = mon_cmd
($vmid, 'query-status');
5460 my $resume_cmd = 'cont';
5462 if ($res->{status
} && $res->{status
} eq 'suspended') {
5463 $resume_cmd = 'system_wakeup';
5468 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5470 PVE
::QemuConfig-
>check_lock($conf)
5471 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5474 mon_cmd
($vmid, $resume_cmd);
5479 my ($vmid, $skiplock, $key) = @_;
5481 PVE
::QemuConfig-
>lock_config($vmid, sub {
5483 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5485 # there is no qmp command, so we use the human monitor command
5486 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5487 die $res if $res ne '';
5491 # vzdump restore implementaion
5493 sub tar_archive_read_firstfile
{
5494 my $archive = shift;
5496 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5498 # try to detect archive type first
5499 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5500 die "unable to open file '$archive'\n";
5501 my $firstfile = <$fh>;
5505 die "ERROR: archive contaions no data\n" if !$firstfile;
5511 sub tar_restore_cleanup
{
5512 my ($storecfg, $statfile) = @_;
5514 print STDERR
"starting cleanup\n";
5516 if (my $fd = IO
::File-
>new($statfile, "r")) {
5517 while (defined(my $line = <$fd>)) {
5518 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5521 if ($volid =~ m
|^/|) {
5522 unlink $volid || die 'unlink failed\n';
5524 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5526 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5528 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5530 print STDERR
"unable to parse line in statfile - $line";
5537 sub restore_file_archive
{
5538 my ($archive, $vmid, $user, $opts) = @_;
5540 return restore_vma_archive
($archive, $vmid, $user, $opts)
5543 my $info = PVE
::Storage
::archive_info
($archive);
5544 my $format = $opts->{format
} // $info->{format
};
5545 my $comp = $info->{compression
};
5547 # try to detect archive format
5548 if ($format eq 'tar') {
5549 return restore_tar_archive
($archive, $vmid, $user, $opts);
5551 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5555 # hepler to remove disks that will not be used after restore
5556 my $restore_cleanup_oldconf = sub {
5557 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5559 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
5560 my ($ds, $drive) = @_;
5562 return if drive_is_cdrom
($drive, 1);
5564 my $volid = $drive->{file
};
5565 return if !$volid || $volid =~ m
|^/|;
5567 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
5568 return if !$path || !$owner || ($owner != $vmid);
5570 # Note: only delete disk we want to restore
5571 # other volumes will become unused
5572 if ($virtdev_hash->{$ds}) {
5573 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
5580 # delete vmstate files, after the restore we have no snapshots anymore
5581 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5582 my $snap = $oldconf->{snapshots
}->{$snapname};
5583 if ($snap->{vmstate
}) {
5584 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
5592 # Helper to parse vzdump backup device hints
5594 # $rpcenv: Environment, used to ckeck storage permissions
5595 # $user: User ID, to check storage permissions
5596 # $storecfg: Storage configuration
5597 # $fh: the file handle for reading the configuration
5598 # $devinfo: should contain device sizes for all backu-up'ed devices
5599 # $options: backup options (pool, default storage)
5601 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5602 my $parse_backup_hints = sub {
5603 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5605 my $virtdev_hash = {};
5607 while (defined(my $line = <$fh>)) {
5608 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5609 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5610 die "archive does not contain data for drive '$virtdev'\n"
5611 if !$devinfo->{$devname};
5613 if (defined($options->{storage
})) {
5614 $storeid = $options->{storage
} || 'local';
5615 } elsif (!$storeid) {
5618 $format = 'raw' if !$format;
5619 $devinfo->{$devname}->{devname
} = $devname;
5620 $devinfo->{$devname}->{virtdev
} = $virtdev;
5621 $devinfo->{$devname}->{format
} = $format;
5622 $devinfo->{$devname}->{storeid
} = $storeid;
5624 # check permission on storage
5625 my $pool = $options->{pool
}; # todo: do we need that?
5626 if ($user ne 'root@pam') {
5627 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5630 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5631 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5633 my $drive = parse_drive
($virtdev, $2);
5634 if (drive_is_cloudinit
($drive)) {
5635 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
5636 $storeid = $options->{storage
} if defined ($options->{storage
});
5637 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5638 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
5640 $virtdev_hash->{$virtdev} = {
5642 storeid
=> $storeid,
5643 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
5650 return $virtdev_hash;
5653 # Helper to allocate and activate all volumes required for a restore
5655 # $storecfg: Storage configuration
5656 # $virtdev_hash: as returned by parse_backup_hints()
5658 # Returns: { $virtdev => $volid }
5659 my $restore_allocate_devices = sub {
5660 my ($storecfg, $virtdev_hash, $vmid) = @_;
5663 foreach my $virtdev (sort keys %$virtdev_hash) {
5664 my $d = $virtdev_hash->{$virtdev};
5665 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5666 my $storeid = $d->{storeid
};
5667 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5669 # test if requested format is supported
5670 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5671 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
5672 $d->{format
} = $defFormat if !$supported;
5675 if ($d->{is_cloudinit
}) {
5676 $name = "vm-$vmid-cloudinit";
5677 $name .= ".$d->{format}" if $d->{format
} ne 'raw';
5680 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
5682 print STDERR
"new volume ID is '$volid'\n";
5683 $d->{volid
} = $volid;
5685 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
5687 $map->{$virtdev} = $volid;
5693 my $restore_update_config_line = sub {
5694 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5696 return if $line =~ m/^\#qmdump\#/;
5697 return if $line =~ m/^\#vzdump\#/;
5698 return if $line =~ m/^lock:/;
5699 return if $line =~ m/^unused\d+:/;
5700 return if $line =~ m/^parent:/;
5702 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5703 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5704 # try to convert old 1.X settings
5705 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5706 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5707 my ($model, $macaddr) = split(/\=/, $devconfig);
5708 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5711 bridge
=> "vmbr$ind",
5712 macaddr
=> $macaddr,
5714 my $netstr = print_net
($net);
5716 print $outfd "net$cookie->{netcount}: $netstr\n";
5717 $cookie->{netcount
}++;
5719 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5720 my ($id, $netstr) = ($1, $2);
5721 my $net = parse_net
($netstr);
5722 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5723 $netstr = print_net
($net);
5724 print $outfd "$id: $netstr\n";
5725 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5728 my $di = parse_drive
($virtdev, $value);
5729 if (defined($di->{backup
}) && !$di->{backup
}) {
5730 print $outfd "#$line";
5731 } elsif ($map->{$virtdev}) {
5732 delete $di->{format
}; # format can change on restore
5733 $di->{file
} = $map->{$virtdev};
5734 $value = print_drive
($di);
5735 print $outfd "$virtdev: $value\n";
5739 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5741 if ($vmgenid ne '0') {
5742 # always generate a new vmgenid if there was a valid one setup
5743 $vmgenid = generate_uuid
();
5745 print $outfd "vmgenid: $vmgenid\n";
5746 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5747 my ($uuid, $uuid_str);
5748 UUID
::generate
($uuid);
5749 UUID
::unparse
($uuid, $uuid_str);
5750 my $smbios1 = parse_smbios1
($2);
5751 $smbios1->{uuid
} = $uuid_str;
5752 print $outfd $1.print_smbios1
($smbios1)."\n";
5758 my $restore_deactivate_volumes = sub {
5759 my ($storecfg, $devinfo) = @_;
5762 foreach my $devname (keys %$devinfo) {
5763 my $volid = $devinfo->{$devname}->{volid
};
5764 push @$vollist, $volid if $volid;
5767 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5770 my $restore_destroy_volumes = sub {
5771 my ($storecfg, $devinfo) = @_;
5773 foreach my $devname (keys %$devinfo) {
5774 my $volid = $devinfo->{$devname}->{volid
};
5777 if ($volid =~ m
|^/|) {
5778 unlink $volid || die 'unlink failed\n';
5780 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5782 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5784 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5789 my ($cfg, $vmid) = @_;
5791 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5793 my $volid_hash = {};
5794 foreach my $storeid (keys %$info) {
5795 foreach my $item (@{$info->{$storeid}}) {
5796 next if !($item->{volid
} && $item->{size
});
5797 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5798 $volid_hash->{$item->{volid
}} = $item;
5805 sub update_disk_config
{
5806 my ($vmid, $conf, $volid_hash) = @_;
5809 my $prefix = "VM $vmid";
5811 # used and unused disks
5812 my $referenced = {};
5814 # Note: it is allowed to define multiple storages with same path (alias), so
5815 # we need to check both 'volid' and real 'path' (two different volid can point
5816 # to the same path).
5818 my $referencedpath = {};
5821 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5822 my ($opt, $drive) = @_;
5824 my $volid = $drive->{file
};
5827 # mark volid as "in-use" for next step
5828 $referenced->{$volid} = 1;
5829 if ($volid_hash->{$volid} &&
5830 (my $path = $volid_hash->{$volid}->{path
})) {
5831 $referencedpath->{$path} = 1;
5834 return if drive_is_cdrom
($drive);
5835 return if !$volid_hash->{$volid};
5837 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volid_hash->{$volid}->{size
});
5838 if (defined($updated)) {
5840 $conf->{$opt} = print_drive
($updated);
5841 print "$prefix ($opt): $msg\n";
5845 # remove 'unusedX' entry if volume is used
5846 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
5847 my ($opt, $drive) = @_;
5849 my $volid = $drive->{file
};
5852 my $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
5853 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5854 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5856 delete $conf->{$opt};
5859 $referenced->{$volid} = 1;
5860 $referencedpath->{$path} = 1 if $path;
5863 foreach my $volid (sort keys %$volid_hash) {
5864 next if $volid =~ m/vm-$vmid-state-/;
5865 next if $referenced->{$volid};
5866 my $path = $volid_hash->{$volid}->{path
};
5867 next if !$path; # just to be sure
5868 next if $referencedpath->{$path};
5870 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
5871 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5872 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5879 my ($vmid, $nolock, $dryrun) = @_;
5881 my $cfg = PVE
::Storage
::config
();
5883 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5884 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5885 foreach my $stor (keys %{$cfg->{ids
}}) {
5886 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
5889 print "rescan volumes...\n";
5890 my $volid_hash = scan_volids
($cfg, $vmid);
5892 my $updatefn = sub {
5895 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5897 PVE
::QemuConfig-
>check_lock($conf);
5900 foreach my $volid (keys %$volid_hash) {
5901 my $info = $volid_hash->{$volid};
5902 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
5905 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
5907 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
5910 if (defined($vmid)) {
5914 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
5917 my $vmlist = config_list
();
5918 foreach my $vmid (keys %$vmlist) {
5922 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
5928 sub restore_proxmox_backup_archive
{
5929 my ($archive, $vmid, $user, $options) = @_;
5931 my $storecfg = PVE
::Storage
::config
();
5933 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
5934 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5936 my $server = $scfg->{server
};
5937 my $datastore = $scfg->{datastore
};
5938 my $username = $scfg->{username
} // 'root@pam';
5939 my $fingerprint = $scfg->{fingerprint
};
5940 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
5942 my $repo = "$username\@$server:$datastore";
5944 # This is only used for `pbs-restore`!
5945 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
5946 local $ENV{PBS_PASSWORD
} = $password;
5947 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
5949 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
5950 PVE
::Storage
::parse_volname
($storecfg, $archive);
5952 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
5954 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
5956 my $tmpdir = "/var/tmp/vzdumptmp$$";
5960 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
5961 my $tmpfn = "$conffile.$$.tmp";
5962 # disable interrupts (always do cleanups)
5966 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
5968 # Note: $oldconf is undef if VM does not exists
5969 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
5970 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
5972 my $rpcenv = PVE
::RPCEnvironment
::get
();
5981 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
5983 my $cfgfn = "$tmpdir/qemu-server.conf";
5984 my $firewall_config_fn = "$tmpdir/fw.conf";
5985 my $index_fn = "$tmpdir/index.json";
5987 my $cmd = "restore";
5989 my $param = [$pbs_backup_name, "index.json", $index_fn];
5990 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
5991 my $index = PVE
::Tools
::file_get_contents
($index_fn);
5992 $index = decode_json
($index);
5994 # print Dumper($index);
5995 foreach my $info (@{$index->{files
}}) {
5996 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
5998 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
5999 $devinfo->{$devname}->{size
} = $1;
6001 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6006 my $is_qemu_server_backup = scalar(grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}});
6007 if (!$is_qemu_server_backup) {
6008 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6010 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6012 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6013 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6015 if ($has_firewall_config) {
6016 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6017 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6019 my $pve_firewall_dir = '/etc/pve/firewall';
6020 mkdir $pve_firewall_dir; # make sure the dir exists
6021 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6024 my $fh = IO
::File-
>new($cfgfn, "r") ||
6025 die "unable to read qemu-server.conf - $!\n";
6027 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6029 # fixme: rate limit?
6031 # create empty/temp config
6032 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6034 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6037 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6039 foreach my $virtdev (sort keys %$virtdev_hash) {
6040 my $d = $virtdev_hash->{$virtdev};
6041 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6043 my $volid = $d->{volid
};
6045 my $path = PVE
::Storage
::path
($storecfg, $volid);
6047 # This is the ONLY user of the PBS_ env vars set on top of this function!
6048 my $pbs_restore_cmd = [
6049 '/usr/bin/pbs-restore',
6050 '--repository', $repo,
6052 "$d->{devname}.img.fidx",
6057 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6058 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6060 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6061 push @$pbs_restore_cmd, '--skip-zero';
6064 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6065 print "restore proxmox backup image: $dbg_cmdstring\n";
6066 run_command
($pbs_restore_cmd);
6069 $fh->seek(0, 0) || die "seek failed - $!\n";
6071 my $outfd = new IO
::File
($tmpfn, "w") ||
6072 die "unable to write config for VM $vmid\n";
6074 my $cookie = { netcount
=> 0 };
6075 while (defined(my $line = <$fh>)) {
6076 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique
});
6084 $restore_deactivate_volumes->($storecfg, $devinfo);
6090 $restore_destroy_volumes->($storecfg, $devinfo);
6094 rename($tmpfn, $conffile) ||
6095 die "unable to commit configuration file '$conffile'\n";
6097 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6099 eval { rescan
($vmid, 1); };
6103 sub restore_vma_archive
{
6104 my ($archive, $vmid, $user, $opts, $comp) = @_;
6106 my $readfrom = $archive;
6108 my $cfg = PVE
::Storage
::config
();
6110 my $bwlimit = $opts->{bwlimit
};
6112 my $dbg_cmdstring = '';
6113 my $add_pipe = sub {
6115 push @$commands, $cmd;
6116 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6117 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6122 if ($archive eq '-') {
6125 # If we use a backup from a PVE defined storage we also consider that
6126 # storage's rate limit:
6127 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6128 if (defined($volid)) {
6129 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6130 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6132 print STDERR
"applying read rate limit: $readlimit\n";
6133 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6134 $add_pipe->($cstream);
6140 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6141 my $cmd = $info->{decompressor
};
6142 push @$cmd, $readfrom;
6146 my $tmpdir = "/var/tmp/vzdumptmp$$";
6149 # disable interrupts (always do cleanups)
6153 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6155 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6156 POSIX
::mkfifo
($mapfifo, 0600);
6159 my $openfifo = sub {
6160 open($fifofh, '>', $mapfifo) || die $!;
6163 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6170 my $rpcenv = PVE
::RPCEnvironment
::get
();
6172 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6173 my $tmpfn = "$conffile.$$.tmp";
6175 # Note: $oldconf is undef if VM does not exist
6176 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6177 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6181 my $print_devmap = sub {
6182 my $cfgfn = "$tmpdir/qemu-server.conf";
6184 # we can read the config - that is already extracted
6185 my $fh = IO
::File-
>new($cfgfn, "r") ||
6186 die "unable to read qemu-server.conf - $!\n";
6188 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6190 my $pve_firewall_dir = '/etc/pve/firewall';
6191 mkdir $pve_firewall_dir; # make sure the dir exists
6192 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6195 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6197 foreach my $key (keys %storage_limits) {
6198 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6200 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6201 $storage_limits{$key} = $limit * 1024;
6204 foreach my $devname (keys %$devinfo) {
6205 die "found no device mapping information for device '$devname'\n"
6206 if !$devinfo->{$devname}->{virtdev
};
6209 # create empty/temp config
6211 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6212 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6216 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6218 # print restore information to $fifofh
6219 foreach my $virtdev (sort keys %$virtdev_hash) {
6220 my $d = $virtdev_hash->{$virtdev};
6221 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6223 my $storeid = $d->{storeid
};
6224 my $volid = $d->{volid
};
6227 if (my $limit = $storage_limits{$storeid}) {
6228 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6231 my $write_zeros = 1;
6232 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6236 my $path = PVE
::Storage
::path
($cfg, $volid);
6238 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6240 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6243 $fh->seek(0, 0) || die "seek failed - $!\n";
6245 my $outfd = new IO
::File
($tmpfn, "w") ||
6246 die "unable to write config for VM $vmid\n";
6248 my $cookie = { netcount
=> 0 };
6249 while (defined(my $line = <$fh>)) {
6250 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6263 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6264 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6266 $oldtimeout = alarm($timeout);
6273 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6274 my ($dev_id, $size, $devname) = ($1, $2, $3);
6275 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6276 } elsif ($line =~ m/^CTIME: /) {
6277 # we correctly received the vma config, so we can disable
6278 # the timeout now for disk allocation (set to 10 minutes, so
6279 # that we always timeout if something goes wrong)
6282 print $fifofh "done\n";
6283 my $tmp = $oldtimeout || 0;
6284 $oldtimeout = undef;
6290 print "restore vma archive: $dbg_cmdstring\n";
6291 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6295 alarm($oldtimeout) if $oldtimeout;
6297 $restore_deactivate_volumes->($cfg, $devinfo);
6304 $restore_destroy_volumes->($cfg, $devinfo);
6308 rename($tmpfn, $conffile) ||
6309 die "unable to commit configuration file '$conffile'\n";
6311 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6313 eval { rescan
($vmid, 1); };
6317 sub restore_tar_archive
{
6318 my ($archive, $vmid, $user, $opts) = @_;
6320 if ($archive ne '-') {
6321 my $firstfile = tar_archive_read_firstfile
($archive);
6322 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6323 if $firstfile ne 'qemu-server.conf';
6326 my $storecfg = PVE
::Storage
::config
();
6328 # avoid zombie disks when restoring over an existing VM -> cleanup first
6329 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6330 # skiplock=1 because qmrestore has set the 'create' lock itself already
6331 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6332 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6334 my $tocmd = "/usr/lib/qemu-server/qmextract";
6336 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6337 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6338 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6339 $tocmd .= ' --info' if $opts->{info
};
6341 # tar option "xf" does not autodetect compression when read from STDIN,
6342 # so we pipe to zcat
6343 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6344 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6346 my $tmpdir = "/var/tmp/vzdumptmp$$";
6349 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6350 local $ENV{VZDUMP_VMID
} = $vmid;
6351 local $ENV{VZDUMP_USER
} = $user;
6353 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6354 my $tmpfn = "$conffile.$$.tmp";
6356 # disable interrupts (always do cleanups)
6360 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6368 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6370 if ($archive eq '-') {
6371 print "extracting archive from STDIN\n";
6372 run_command
($cmd, input
=> "<&STDIN");
6374 print "extracting archive '$archive'\n";
6378 return if $opts->{info
};
6382 my $statfile = "$tmpdir/qmrestore.stat";
6383 if (my $fd = IO
::File-
>new($statfile, "r")) {
6384 while (defined (my $line = <$fd>)) {
6385 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6386 $map->{$1} = $2 if $1;
6388 print STDERR
"unable to parse line in statfile - $line\n";
6394 my $confsrc = "$tmpdir/qemu-server.conf";
6396 my $srcfd = new IO
::File
($confsrc, "r") ||
6397 die "unable to open file '$confsrc'\n";
6399 my $outfd = new IO
::File
($tmpfn, "w") ||
6400 die "unable to write config for VM $vmid\n";
6402 my $cookie = { netcount
=> 0 };
6403 while (defined (my $line = <$srcfd>)) {
6404 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6412 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6418 rename $tmpfn, $conffile ||
6419 die "unable to commit configuration file '$conffile'\n";
6421 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6423 eval { rescan
($vmid, 1); };
6427 sub foreach_storage_used_by_vm
{
6428 my ($conf, $func) = @_;
6432 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6433 my ($ds, $drive) = @_;
6434 return if drive_is_cdrom
($drive);
6436 my $volid = $drive->{file
};
6438 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6439 $sidhash->{$sid} = $sid if $sid;
6442 foreach my $sid (sort keys %$sidhash) {
6447 my $qemu_snap_storage = {
6450 sub do_snapshots_with_qemu
{
6451 my ($storecfg, $volid) = @_;
6453 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6454 my $scfg = $storecfg->{ids
}->{$storage_name};
6456 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6460 if ($volid =~ m/\.(qcow2|qed)$/){
6467 sub qga_check_running
{
6468 my ($vmid, $nowarn) = @_;
6470 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6472 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6478 sub template_create
{
6479 my ($vmid, $conf, $disk) = @_;
6481 my $storecfg = PVE
::Storage
::config
();
6483 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6484 my ($ds, $drive) = @_;
6486 return if drive_is_cdrom
($drive);
6487 return if $disk && $ds ne $disk;
6489 my $volid = $drive->{file
};
6490 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6492 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6493 $drive->{file
} = $voliddst;
6494 $conf->{$ds} = print_drive
($drive);
6495 PVE
::QemuConfig-
>write_config($vmid, $conf);
6499 sub convert_iscsi_path
{
6502 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6507 my $initiator_name = get_initiator_name
();
6509 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6510 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6513 die "cannot convert iscsi path '$path', unkown format\n";
6516 sub qemu_img_convert
{
6517 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6519 my $storecfg = PVE
::Storage
::config
();
6520 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6521 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6523 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6527 my $src_is_iscsi = 0;
6531 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6532 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6533 $src_format = qemu_img_format
($src_scfg, $src_volname);
6534 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6535 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6536 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6537 } elsif (-f
$src_volid) {
6538 $src_path = $src_volid;
6539 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6544 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6546 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6547 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6548 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6549 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6552 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6553 push @$cmd, '-l', "snapshot.name=$snapname"
6554 if $snapname && $src_format && $src_format eq "qcow2";
6555 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6556 push @$cmd, '-T', $cachemode if defined($cachemode);
6558 if ($src_is_iscsi) {
6559 push @$cmd, '--image-opts';
6560 $src_path = convert_iscsi_path
($src_path);
6561 } elsif ($src_format) {
6562 push @$cmd, '-f', $src_format;
6565 if ($dst_is_iscsi) {
6566 push @$cmd, '--target-image-opts';
6567 $dst_path = convert_iscsi_path
($dst_path);
6569 push @$cmd, '-O', $dst_format;
6572 push @$cmd, $src_path;
6574 if (!$dst_is_iscsi && $is_zero_initialized) {
6575 push @$cmd, "zeroinit:$dst_path";
6577 push @$cmd, $dst_path;
6582 if($line =~ m/\((\S+)\/100\
%\)/){
6584 my $transferred = int($size * $percent / 100);
6585 my $remaining = $size - $transferred;
6587 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6592 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6594 die "copy failed: $err" if $err;
6597 sub qemu_img_format
{
6598 my ($scfg, $volname) = @_;
6600 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6607 sub qemu_drive_mirror
{
6608 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6610 $jobs = {} if !$jobs;
6614 $jobs->{"drive-$drive"} = {};
6616 if ($dst_volid =~ /^nbd:/) {
6617 $qemu_target = $dst_volid;
6620 my $storecfg = PVE
::Storage
::config
();
6621 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6623 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6625 $format = qemu_img_format
($dst_scfg, $dst_volname);
6627 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6629 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6632 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6633 $opts->{format
} = $format if $format;
6635 if (defined($src_bitmap)) {
6636 $opts->{sync
} = 'incremental';
6637 $opts->{bitmap
} = $src_bitmap;
6638 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6641 if (defined($bwlimit)) {
6642 $opts->{speed
} = $bwlimit * 1024;
6643 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6645 print "drive mirror is starting for drive-$drive\n";
6648 # if a job already runs for this device we get an error, catch it for cleanup
6649 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
6651 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6653 die "mirroring error: $err\n";
6656 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
6659 # $completion can be either
6660 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6661 # 'cancel': wait until all jobs are ready, block-job-cancel them
6662 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6663 sub qemu_drive_mirror_monitor
{
6664 my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6666 $completion //= 'complete';
6669 my $err_complete = 0;
6672 die "storage migration timed out\n" if $err_complete > 300;
6674 my $stats = mon_cmd
($vmid, "query-block-jobs");
6676 my $running_mirror_jobs = {};
6677 foreach my $stat (@$stats) {
6678 next if $stat->{type
} ne 'mirror';
6679 $running_mirror_jobs->{$stat->{device
}} = $stat;
6682 my $readycounter = 0;
6684 foreach my $job (keys %$jobs) {
6686 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6687 print "$job : finished\n";
6688 delete $jobs->{$job};
6692 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6694 my $busy = $running_mirror_jobs->{$job}->{busy
};
6695 my $ready = $running_mirror_jobs->{$job}->{ready
};
6696 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6697 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6698 my $remaining = $total - $transferred;
6699 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6701 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6704 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6707 last if scalar(keys %$jobs) == 0;
6709 if ($readycounter == scalar(keys %$jobs)) {
6710 print "all mirroring jobs are ready \n";
6711 last if $completion eq 'skip'; #do the complete later
6713 if ($vmiddst && $vmiddst != $vmid) {
6714 my $agent_running = $qga && qga_check_running
($vmid);
6715 if ($agent_running) {
6716 print "freeze filesystem\n";
6717 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6719 print "suspend vm\n";
6720 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6723 # if we clone a disk for a new target vm, we don't switch the disk
6724 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6726 if ($agent_running) {
6727 print "unfreeze filesystem\n";
6728 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6730 print "resume vm\n";
6731 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6737 foreach my $job (keys %$jobs) {
6738 # try to switch the disk if source and destination are on the same guest
6739 print "$job: Completing block job...\n";
6742 if ($completion eq 'complete') {
6743 $op = 'block-job-complete';
6744 } elsif ($completion eq 'cancel') {
6745 $op = 'block-job-cancel';
6747 die "invalid completion value: $completion\n";
6749 eval { mon_cmd
($vmid, $op, device
=> $job) };
6750 if ($@ =~ m/cannot be completed/) {
6751 print "$job: Block job cannot be completed, try again.\n";
6754 print "$job: Completed successfully.\n";
6755 $jobs->{$job}->{complete
} = 1;
6766 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6767 die "mirroring error: $err";
6772 sub qemu_blockjobs_cancel
{
6773 my ($vmid, $jobs) = @_;
6775 foreach my $job (keys %$jobs) {
6776 print "$job: Cancelling block job\n";
6777 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6778 $jobs->{$job}->{cancel
} = 1;
6782 my $stats = mon_cmd
($vmid, "query-block-jobs");
6784 my $running_jobs = {};
6785 foreach my $stat (@$stats) {
6786 $running_jobs->{$stat->{device
}} = $stat;
6789 foreach my $job (keys %$jobs) {
6791 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6792 print "$job: Done.\n";
6793 delete $jobs->{$job};
6797 last if scalar(keys %$jobs) == 0;
6804 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6805 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6810 print "create linked clone of drive $drivename ($drive->{file})\n";
6811 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6812 push @$newvollist, $newvolid;
6815 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6816 $storeid = $storage if $storage;
6818 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6819 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
6821 print "create full clone of drive $drivename ($drive->{file})\n";
6823 if (drive_is_cloudinit
($drive)) {
6824 $name = "vm-$newvmid-cloudinit";
6825 $name .= ".$dst_format" if $dst_format ne 'raw';
6827 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
6828 } elsif ($drivename eq 'efidisk0') {
6829 $size = get_efivars_size
($conf);
6831 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6832 push @$newvollist, $newvolid;
6834 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
6836 if (drive_is_cloudinit
($drive)) {
6840 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6841 if (!$running || $snapname) {
6842 # TODO: handle bwlimits
6843 if ($drivename eq 'efidisk0') {
6844 # the relevant data on the efidisk may be smaller than the source
6845 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6846 # that is given by the OVMF_VARS.fd
6847 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
6848 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
6849 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6851 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
6855 my $kvmver = get_running_qemu_version
($vmid);
6856 if (!min_version
($kvmver, 2, 7)) {
6857 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6858 if $drive->{iothread
};
6861 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6866 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
6869 $disk->{format
} = undef;
6870 $disk->{file
} = $newvolid;
6871 $disk->{size
} = $size;
6876 sub get_running_qemu_version
{
6878 my $res = mon_cmd
($vmid, "query-version");
6879 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6882 sub qemu_use_old_bios_files
{
6883 my ($machine_type) = @_;
6885 return if !$machine_type;
6887 my $use_old_bios_files = undef;
6889 if ($machine_type =~ m/^(\S+)\.pxe$/) {
6891 $use_old_bios_files = 1;
6893 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, kvm_user_version
());
6894 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6895 # load new efi bios files on migration. So this hack is required to allow
6896 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6897 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6898 $use_old_bios_files = !min_version
($version, 2, 4);
6901 return ($use_old_bios_files, $machine_type);
6904 sub get_efivars_size
{
6906 my $arch = get_vm_arch
($conf);
6907 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
6908 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
6909 return -s
$ovmf_vars;
6912 sub update_efidisk_size
{
6915 return if !defined($conf->{efidisk0
});
6917 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
6918 $disk->{size
} = get_efivars_size
($conf);
6919 $conf->{efidisk0
} = print_drive
($disk);
6924 sub create_efidisk
($$$$$) {
6925 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6927 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
6928 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
6930 my $vars_size_b = -s
$ovmf_vars;
6931 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
6932 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6933 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6935 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
6936 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
6938 return ($volid, $size/1024);
6941 sub vm_iothreads_list
{
6944 my $res = mon_cmd
($vmid, 'query-iothreads');
6947 foreach my $iothread (@$res) {
6948 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
6955 my ($conf, $drive) = @_;
6959 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
6961 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
6967 my $controller = int($drive->{index} / $maxdev);
6968 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single') ?
"virtioscsi" : "scsihw";
6970 return ($maxdev, $controller, $controller_prefix);
6973 sub windows_version
{
6976 return 0 if !$ostype;
6980 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6982 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6984 } elsif ($ostype =~ m/^win(\d+)$/) {
6991 sub resolve_dst_disk_format
{
6992 my ($storecfg, $storeid, $src_volname, $format) = @_;
6993 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6996 # if no target format is specified, use the source disk format as hint
6998 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6999 $format = qemu_img_format
($scfg, $src_volname);
7005 # test if requested format is supported - else use default
7006 my $supported = grep { $_ eq $format } @$validFormats;
7007 $format = $defFormat if !$supported;
7011 # NOTE: if this logic changes, please update docs & possibly gui logic
7012 sub find_vmstate_storage
{
7013 my ($conf, $storecfg) = @_;
7015 # first, return storage from conf if set
7016 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7018 my ($target, $shared, $local);
7020 foreach_storage_used_by_vm
($conf, sub {
7022 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7023 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7024 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7027 # second, use shared storage where VM has at least one disk
7028 # third, use local storage where VM has at least one disk
7029 # fall back to local storage
7030 $target = $shared // $local // 'local';
7036 my ($uuid, $uuid_str);
7037 UUID
::generate
($uuid);
7038 UUID
::unparse
($uuid, $uuid_str);
7042 sub generate_smbios1_uuid
{
7043 return "uuid=".generate_uuid
();
7049 mon_cmd
($vmid, 'nbd-server-stop');
7052 sub create_reboot_request
{
7054 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7055 or die "failed to create reboot trigger file: $!\n";
7059 sub clear_reboot_request
{
7061 my $path = "/run/qemu-server/$vmid.reboot";
7064 $res = unlink($path);
7065 die "could not remove reboot request for $vmid: $!"
7066 if !$res && $! != POSIX
::ENOENT
;
7071 # bash completion helper
7073 sub complete_backup_archives
{
7074 my ($cmdname, $pname, $cvalue) = @_;
7076 my $cfg = PVE
::Storage
::config
();
7080 if ($cvalue =~ m/^([^:]+):/) {
7084 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7087 foreach my $id (keys %$data) {
7088 foreach my $item (@{$data->{$id}}) {
7089 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7090 push @$res, $item->{volid
} if defined($item->{volid
});
7097 my $complete_vmid_full = sub {
7100 my $idlist = vmstatus
();
7104 foreach my $id (keys %$idlist) {
7105 my $d = $idlist->{$id};
7106 if (defined($running)) {
7107 next if $d->{template
};
7108 next if $running && $d->{status
} ne 'running';
7109 next if !$running && $d->{status
} eq 'running';
7118 return &$complete_vmid_full();
7121 sub complete_vmid_stopped
{
7122 return &$complete_vmid_full(0);
7125 sub complete_vmid_running
{
7126 return &$complete_vmid_full(1);
7129 sub complete_storage
{
7131 my $cfg = PVE
::Storage
::config
();
7132 my $ids = $cfg->{ids
};
7135 foreach my $sid (keys %$ids) {
7136 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7137 next if !$ids->{$sid}->{content
}->{images
};
7144 sub complete_migration_storage
{
7145 my ($cmd, $param, $current_value, $all_args) = @_;
7147 my $targetnode = @$all_args[1];
7149 my $cfg = PVE
::Storage
::config
();
7150 my $ids = $cfg->{ids
};
7153 foreach my $sid (keys %$ids) {
7154 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7155 next if !$ids->{$sid}->{content
}->{images
};