1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
30 use PVE
::DataCenterConfig
;
31 use PVE
::Exception
qw(raise raise_param_exc);
32 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
34 use PVE
::JSONSchema
qw(get_standard_option);
36 use PVE
::RPCEnvironment
;
40 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
44 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
45 use PVE
::QemuServer
::Cloudinit
;
46 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
47 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
48 use PVE
::QemuServer
::Machine
;
49 use PVE
::QemuServer
::Memory
;
50 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
51 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port);
52 use PVE
::QemuServer
::USB
qw(parse_usb_device);
56 require PVE
::Network
::SDN
::Zones
;
60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
63 "$EDK2_FW_BASE/OVMF_CODE.fd",
64 "$EDK2_FW_BASE/OVMF_VARS.fd"
67 "$EDK2_FW_BASE/AAVMF_CODE.fd",
68 "$EDK2_FW_BASE/AAVMF_VARS.fd"
72 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
74 # Note about locking: we use flock on the config file protect
75 # against concurent actions.
76 # Aditionaly, we have a 'lock' setting in the config file. This
77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
78 # allowed when such lock is set. But you can ignore this kind of
79 # lock with the --skiplock flag.
81 cfs_register_file
('/qemu-server/',
85 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
86 description
=> "Some command save/restore state from this location.",
92 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
93 description
=> "Specifies the Qemu machine type.",
95 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
102 my ($map, $source) = @_;
104 return $source if !defined($map);
106 return $map->{entries
}->{$source}
107 if $map->{entries
} && defined($map->{entries
}->{$source});
109 return $map->{default} if $map->{default};
111 # identity (fallback)
115 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
116 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
118 format
=> 'storagepair-list',
122 #no warnings 'redefine';
125 my ($controller, $vmid, $option, $value) = @_;
127 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
128 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
134 $nodename_cache //= PVE
::INotify
::nodename
();
135 return $nodename_cache;
142 enum
=> [qw(i6300esb ib700)],
143 description
=> "Watchdog type to emulate.",
144 default => 'i6300esb',
149 enum
=> [qw(reset shutdown poweroff pause debug none)],
150 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
154 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
158 description
=> "Enable/disable Qemu GuestAgent.",
163 fstrim_cloned_disks
=> {
164 description
=> "Run fstrim after cloning/moving a disk.",
170 description
=> "Select the agent type",
174 enum
=> [qw(virtio isa)],
180 description
=> "Select the VGA type.",
185 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
188 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
200 description
=> "The size of the file in MB.",
204 pattern
=> '[a-zA-Z0-9\-]+',
206 format_description
=> 'string',
207 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
214 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
215 description
=> "Configure an audio device."
222 description
=> "Driver backend for the audio device."
226 my $spice_enhancements_fmt = {
231 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
235 enum
=> ['off', 'all', 'filter'],
238 description
=> "Enable video streaming. Uses compression for detected video streams."
245 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
247 description
=> "The file on the host to gather entropy from. In most"
248 . " cases /dev/urandom should be preferred over /dev/random"
249 . " to avoid entropy-starvation issues on the host. Using"
250 . " urandom does *not* decrease security in any meaningful"
251 . " way, as it's still seeded from real entropy, and the"
252 . " bytes provided will most likely be mixed with real"
253 . " entropy on the guest as well. /dev/hwrng can be used"
254 . " to pass through a hardware RNG from the host.",
258 description
=> "Maximum bytes of entropy injected into the guest every"
259 . " 'period' milliseconds. Prefer a lower value when using"
260 . " /dev/random as source. Use 0 to disable limiting"
261 . " (potentially dangerous!).",
264 # default is 1 KiB/s, provides enough entropy to the guest to avoid
265 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
266 # of overwhelming the host, provided we're reading from /dev/urandom
271 description
=> "Every 'period' milliseconds the entropy-injection quota"
272 . " is reset, allowing the guest to retrieve another"
273 . " 'max_bytes' of entropy.",
283 description
=> "Specifies whether a VM will be started during system bootup.",
289 description
=> "Automatic restart after crash (currently ignored).",
294 type
=> 'string', format
=> 'pve-hotplug-features',
295 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
296 default => 'network,disk,usb',
301 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
307 description
=> "Lock/unlock the VM.",
308 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
313 description
=> "Limit of CPU usage.",
314 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
322 description
=> "CPU weight for a VM.",
323 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
331 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
338 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
344 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
352 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
353 "It should not be necessary to set it.",
354 enum
=> PVE
::Tools
::kvmkeymaplist
(),
359 type
=> 'string', format
=> 'dns-name',
360 description
=> "Set a name for the VM. Only used on the configuration web interface.",
365 description
=> "SCSI controller model",
366 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
372 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
377 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
378 description
=> "Specify guest operating system.",
379 verbose_description
=> <<EODESC,
380 Specify guest operating system. This is used to enable special
381 optimization/features for specific operating systems:
384 other;; unspecified OS
385 wxp;; Microsoft Windows XP
386 w2k;; Microsoft Windows 2000
387 w2k3;; Microsoft Windows 2003
388 w2k8;; Microsoft Windows 2008
389 wvista;; Microsoft Windows Vista
390 win7;; Microsoft Windows 7
391 win8;; Microsoft Windows 8/2012/2012r2
392 win10;; Microsoft Windows 10/2016
393 l24;; Linux 2.4 Kernel
394 l26;; Linux 2.6 - 5.X Kernel
395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
401 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
402 pattern
=> '[acdn]{1,4}',
407 type
=> 'string', format
=> 'pve-qm-bootdisk',
408 description
=> "Enable booting from specified disk.",
409 pattern
=> '(ide|sata|scsi|virtio)\d+',
414 description
=> "The number of CPUs. Please use option -sockets instead.",
421 description
=> "The number of CPU sockets.",
428 description
=> "The number of cores per socket.",
435 description
=> "Enable/disable NUMA.",
441 description
=> "Enable/disable hugepages memory.",
442 enum
=> [qw(any 2 1024)],
447 description
=> "Number of hotplugged vcpus.",
454 description
=> "Enable/disable ACPI.",
459 description
=> "Enable/disable Qemu GuestAgent and its properties.",
461 format
=> $agent_fmt,
466 description
=> "Enable/disable KVM hardware virtualization.",
472 description
=> "Enable/disable time drift fix.",
478 description
=> "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
483 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
487 type
=> 'string', format
=> $vga_fmt,
488 description
=> "Configure the VGA hardware.",
489 verbose_description
=> "Configure the VGA Hardware. If you want to use ".
490 "high resolution modes (>= 1280x1024x16) you may need to increase " .
491 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
492 "is 'std' for all OS types besides some Windows versions (XP and " .
493 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
494 "display server. For win* OS you can select how many independent " .
495 "displays you want, Linux guests can add displays them self.\n".
496 "You can also run without any graphic card, using a serial device as terminal.",
500 type
=> 'string', format
=> 'pve-qm-watchdog',
501 description
=> "Create a virtual hardware watchdog device.",
502 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled" .
503 " (by a guest action), the watchdog must be periodically polled " .
504 "by an agent inside the guest or else the watchdog will reset " .
505 "the guest (or execute the respective action specified)",
510 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
511 description
=> "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
512 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
515 startup
=> get_standard_option
('pve-startup-order'),
519 description
=> "Enable/disable Template.",
525 description
=> "Arbitrary arguments passed to kvm.",
526 verbose_description
=> <<EODESCR,
527 Arbitrary arguments passed to kvm, for example:
529 args: -no-reboot -no-hpet
531 NOTE: this option is for experts only.
538 description
=> "Enable/disable the USB tablet device.",
539 verbose_description
=> "Enable/disable the USB tablet device. This device is " .
540 "usually needed to allow absolute mouse positioning with VNC. " .
541 "Else the mouse runs out of sync with normal VNC clients. " .
542 "If you're running lots of console-only guests on one host, " .
543 "you may consider disabling this to save some context switches. " .
544 "This is turned off by default if you use spice (-vga=qxl).",
549 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
553 migrate_downtime
=> {
556 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
562 type
=> 'string', format
=> 'pve-qm-ide',
563 typetext
=> '<volume>',
564 description
=> "This is an alias for option -ide2",
568 description
=> "Emulated CPU type.",
570 format
=> 'pve-vm-cpu-conf',
572 parent
=> get_standard_option
('pve-snapshot-name', {
574 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
578 description
=> "Timestamp for snapshots.",
584 type
=> 'string', format
=> 'pve-volume-id',
585 description
=> "Reference to a volume which stores the VM state. This is used internally for snapshots.",
587 vmstatestorage
=> get_standard_option
('pve-storage-id', {
588 description
=> "Default storage for VM state volumes/files.",
591 runningmachine
=> get_standard_option
('pve-qemu-machine', {
592 description
=> "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
595 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
598 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
599 format_description
=> 'QEMU -cpu parameter'
601 machine
=> get_standard_option
('pve-qemu-machine'),
603 description
=> "Virtual processor architecture. Defaults to the host.",
606 enum
=> [qw(x86_64 aarch64)],
609 description
=> "Specify SMBIOS type 1 fields.",
610 type
=> 'string', format
=> 'pve-qm-smbios1',
617 description
=> "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
623 enum
=> [ qw(seabios ovmf) ],
624 description
=> "Select BIOS implementation.",
625 default => 'seabios',
629 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
630 format_description
=> 'UUID',
631 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
632 verbose_description
=> "The VM generation ID (vmgenid) device exposes a".
633 " 128-bit integer value identifier to the guest OS. This allows to".
634 " notify the guest operating system when the virtual machine is".
635 " executed with a different configuration (e.g. snapshot execution".
636 " or creation from a template). The guest operating system notices".
637 " the change, and is then able to react as appropriate by marking".
638 " its copies of distributed databases as dirty, re-initializing its".
639 " random number generator, etc.\n".
640 "Note that auto-creation only works when done throug API/CLI create".
641 " or update methods, but not when manually editing the config file.",
642 default => "1 (autogenerated)",
647 format
=> 'pve-volume-id',
649 description
=> "Script that will be executed during various steps in the vms lifetime.",
653 format
=> $ivshmem_fmt,
654 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
659 format
=> $audio_fmt,
660 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
663 spice_enhancements
=> {
665 format
=> $spice_enhancements_fmt,
666 description
=> "Configure additional enhancements for SPICE.",
670 type
=> 'string', format
=> 'pve-tag-list',
671 description
=> 'Tags of the VM. This is only meta information.',
677 description
=> "Configure a VirtIO-based Random Number Generator.",
686 description
=> 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
687 format
=> 'pve-volume-id',
688 format_description
=> 'volume',
693 description
=> 'Specify a custom file containing all network data passed to the VM via cloud-init.',
694 format
=> 'pve-volume-id',
695 format_description
=> 'volume',
700 description
=> 'Specify a custom file containing all user data passed to the VM via cloud-init.',
701 format
=> 'pve-volume-id',
702 format_description
=> 'volume',
705 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
707 my $confdesc_cloudinit = {
711 description
=> 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
712 enum
=> ['configdrive2', 'nocloud'],
717 description
=> "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
722 description
=> 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
727 description
=> 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
728 format
=> 'pve-qm-cicustom',
733 description
=> "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
737 type
=> 'string', format
=> 'address-list',
738 description
=> "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
743 format
=> 'urlencoded',
744 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
748 # what about other qemu settings ?
750 #machine => 'string',
763 ##soundhw => 'string',
765 while (my ($k, $v) = each %$confdesc) {
766 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
769 my $MAX_USB_DEVICES = 5;
771 my $MAX_HOSTPCI_DEVICES = 16;
772 my $MAX_SERIAL_PORTS = 4;
773 my $MAX_PARALLEL_PORTS = 3;
779 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
780 description
=> "CPUs accessing this NUMA node.",
781 format_description
=> "id[-id];...",
785 description
=> "Amount of memory this NUMA node provides.",
790 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
791 description
=> "Host NUMA nodes to use.",
792 format_description
=> "id[-id];...",
797 enum
=> [qw(preferred bind interleave)],
798 description
=> "NUMA allocation policy.",
802 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
805 type
=> 'string', format
=> $numa_fmt,
806 description
=> "NUMA topology.",
808 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
810 for (my $i = 0; $i < $MAX_NUMA; $i++) {
811 $confdesc->{"numa$i"} = $numadesc;
814 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
815 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
816 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
817 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
819 my $net_fmt_bridge_descr = <<__EOD__;
820 Bridge to attach the network device to. The Proxmox VE standard bridge
823 If you do not specify a bridge, we create a kvm user (NATed) network
824 device, which provides DHCP and DNS services. The following addresses
831 The DHCP server assign addresses to the guest starting from 10.0.2.15.
835 macaddr
=> get_standard_option
('mac-addr', {
836 description
=> "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
840 description
=> "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
841 enum
=> $nic_model_list,
844 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
847 description
=> $net_fmt_bridge_descr,
848 format_description
=> 'bridge',
849 pattern
=> '[-_.\w\d]+',
854 minimum
=> 0, maximum
=> 16,
855 description
=> 'Number of packet queues to be used on the device.',
861 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
866 minimum
=> 1, maximum
=> 4094,
867 description
=> 'VLAN tag to apply to packets on this interface.',
872 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
873 description
=> 'VLAN trunks to pass through this interface.',
874 format_description
=> 'vlanid[;vlanid...]',
879 description
=> 'Whether this interface should be protected by the firewall.',
884 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
891 type
=> 'string', format
=> $net_fmt,
892 description
=> "Specify network devices.",
895 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
900 format
=> 'pve-ipv4-config',
901 format_description
=> 'IPv4Format/CIDR',
902 description
=> 'IPv4 address in CIDR format.',
909 format_description
=> 'GatewayIPv4',
910 description
=> 'Default gateway for IPv4 traffic.',
916 format
=> 'pve-ipv6-config',
917 format_description
=> 'IPv6Format/CIDR',
918 description
=> 'IPv6 address in CIDR format.',
925 format_description
=> 'GatewayIPv6',
926 description
=> 'Default gateway for IPv6 traffic.',
931 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
934 type
=> 'string', format
=> 'pve-qm-ipconfig',
935 description
=> <<'EODESCR',
936 cloud-init: Specify IP addresses and gateways for the corresponding interface.
938 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
940 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
941 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
943 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
946 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
948 for (my $i = 0; $i < $MAX_NETS; $i++) {
949 $confdesc->{"net$i"} = $netdesc;
950 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
953 foreach my $key (keys %$confdesc_cloudinit) {
954 $confdesc->{$key} = $confdesc_cloudinit->{$key};
957 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
958 sub verify_volume_id_or_qm_path
{
959 my ($volid, $noerr) = @_;
961 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
965 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
966 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
968 return undef if $noerr;
977 type
=> 'string', format
=> 'pve-qm-usb-device',
978 format_description
=> 'HOSTUSBDEVICE|spice',
979 description
=> <<EODESCR,
980 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
982 'bus-port(.port)*' (decimal numbers) or
983 'vendor_id:product_id' (hexadeciaml numbers) or
986 You can use the 'lsusb -t' command to list existing usb devices.
988 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
990 The value 'spice' can be used to add a usb redirection devices for spice.
996 description
=> "Specifies whether if given host option is a USB3 device or port.",
1003 type
=> 'string', format
=> $usb_fmt,
1004 description
=> "Configure an USB device (n is 0 to 4).",
1006 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1008 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1013 pattern
=> qr/$PCIRE(;$PCIRE)*/,
1014 format_description
=> 'HOSTPCIID[;HOSTPCIID2...]',
1015 description
=> <<EODESCR,
1016 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1017 of PCI virtual functions of the host. HOSTPCIID syntax is:
1019 'bus:dev.func' (hexadecimal numbers)
1021 You can us the 'lspci' command to list existing PCI devices.
1026 description
=> "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1032 pattern
=> '[^,;]+',
1033 format_description
=> 'string',
1034 description
=> "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1039 description
=> "Choose the PCI-express bus (needs the 'q35' machine model).",
1045 description
=> "Enable vfio-vga device support.",
1051 format_description
=> 'string',
1052 pattern
=> '[^/\.:]+',
1054 description
=> <<EODESCR
1055 The type of mediated device to use.
1056 An instance of this type will be created on startup of the VM and
1057 will be cleaned up when the VM stops.
1061 PVE
::JSONSchema
::register_format
('pve-qm-hostpci', $hostpci_fmt);
1065 type
=> 'string', format
=> 'pve-qm-hostpci',
1066 description
=> "Map host PCI devices into guest.",
1067 verbose_description
=> <<EODESCR,
1068 Map host PCI devices into guest.
1070 NOTE: This option allows direct access to host hardware. So it is no longer
1071 possible to migrate such machines - use with special care.
1073 CAUTION: Experimental! User reported problems with this option.
1076 PVE
::JSONSchema
::register_standard_option
("pve-qm-hostpci", $hostpcidesc);
1081 pattern
=> '(/dev/.+|socket)',
1082 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1083 verbose_description
=> <<EODESCR,
1084 Create a serial device inside the VM (n is 0 to 3), and pass through a
1085 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1086 host side (use 'qm terminal' to open a terminal connection).
1088 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1090 CAUTION: Experimental! User reported problems with this option.
1097 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1098 description
=> "Map host parallel devices (n is 0 to 2).",
1099 verbose_description
=> <<EODESCR,
1100 Map host parallel devices (n is 0 to 2).
1102 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1104 CAUTION: Experimental! User reported problems with this option.
1108 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1109 $confdesc->{"parallel$i"} = $paralleldesc;
1112 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1113 $confdesc->{"serial$i"} = $serialdesc;
1116 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
1117 $confdesc->{"hostpci$i"} = $hostpcidesc;
1120 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1121 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1124 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1125 $confdesc->{"usb$i"} = $usbdesc;
1128 my $kvm_api_version = 0;
1131 return $kvm_api_version if $kvm_api_version;
1133 open my $fh, '<', '/dev/kvm'
1136 # 0xae00 => KVM_GET_API_VERSION
1137 $kvm_api_version = ioctl($fh, 0xae00, 0);
1139 return $kvm_api_version;
1142 my $kvm_user_version = {};
1145 sub kvm_user_version
{
1148 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1149 my $st = stat($binary);
1151 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1152 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1153 $cachedmtime == $st->mtime;
1155 $kvm_user_version->{$binary} = 'unknown';
1156 $kvm_mtime->{$binary} = $st->mtime;
1160 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1161 $kvm_user_version->{$binary} = $2;
1165 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1168 return $kvm_user_version->{$binary};
1172 sub kernel_has_vhost_net
{
1173 return -c
'/dev/vhost-net';
1178 return defined($confdesc->{$key});
1182 sub get_cdrom_path
{
1184 return $cdrom_path if $cdrom_path;
1186 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1187 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1188 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1192 my ($storecfg, $vmid, $cdrom) = @_;
1194 if ($cdrom eq 'cdrom') {
1195 return get_cdrom_path
();
1196 } elsif ($cdrom eq 'none') {
1198 } elsif ($cdrom =~ m
|^/|) {
1201 return PVE
::Storage
::path
($storecfg, $cdrom);
1205 # try to convert old style file names to volume IDs
1206 sub filename_to_volume_id
{
1207 my ($vmid, $file, $media) = @_;
1209 if (!($file eq 'none' || $file eq 'cdrom' ||
1210 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1212 return undef if $file =~ m
|/|;
1214 if ($media && $media eq 'cdrom') {
1215 $file = "local:iso/$file";
1217 $file = "local:$vmid/$file";
1224 sub verify_media_type
{
1225 my ($opt, $vtype, $media) = @_;
1230 if ($media eq 'disk') {
1232 } elsif ($media eq 'cdrom') {
1235 die "internal error";
1238 return if ($vtype eq $etype);
1240 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1243 sub cleanup_drive_path
{
1244 my ($opt, $storecfg, $drive) = @_;
1246 # try to convert filesystem paths to volume IDs
1248 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1249 ($drive->{file
} !~ m
|^/dev/.+|) &&
1250 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1251 ($drive->{file
} !~ m/^\d+$/)) {
1252 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1253 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1254 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1255 verify_media_type
($opt, $vtype, $drive->{media
});
1256 $drive->{file
} = $volid;
1259 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1262 sub parse_hotplug_features
{
1267 return $res if $data eq '0';
1269 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1271 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1272 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1275 die "invalid hotplug feature '$feature'\n";
1281 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1282 sub pve_verify_hotplug_features
{
1283 my ($value, $noerr) = @_;
1285 return $value if parse_hotplug_features
($value);
1287 return undef if $noerr;
1289 die "unable to parse hotplug option\n";
1293 my($fh, $noerr) = @_;
1296 my $SG_GET_VERSION_NUM = 0x2282;
1298 my $versionbuf = "\x00" x
8;
1299 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1301 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1304 my $version = unpack("I", $versionbuf);
1305 if ($version < 30000) {
1306 die "scsi generic interface too old\n" if !$noerr;
1310 my $buf = "\x00" x
36;
1311 my $sensebuf = "\x00" x
8;
1312 my $cmd = pack("C x3 C x1", 0x12, 36);
1314 # see /usr/include/scsi/sg.h
1315 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1317 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1318 length($sensebuf), 0, length($buf), $buf,
1319 $cmd, $sensebuf, 6000);
1321 $ret = ioctl($fh, $SG_IO, $packet);
1323 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1327 my @res = unpack($sg_io_hdr_t, $packet);
1328 if ($res[17] || $res[18]) {
1329 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1334 (my $byte0, my $byte1, $res->{vendor
},
1335 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1337 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1338 $res->{type
} = $byte0 & 31;
1346 my $fh = IO
::File-
>new("+<$path") || return undef;
1347 my $res = scsi_inquiry
($fh, 1);
1353 sub print_tabletdevice_full
{
1354 my ($conf, $arch) = @_;
1356 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1358 # we use uhci for old VMs because tablet driver was buggy in older qemu
1360 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1366 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1369 sub print_keyboarddevice_full
{
1370 my ($conf, $arch, $machine) = @_;
1372 return undef if $arch ne 'aarch64';
1374 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1377 sub print_drivedevice_full
{
1378 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1383 if ($drive->{interface
} eq 'virtio') {
1384 my $pciaddr = print_pci_addr
("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1385 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1386 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread
};
1387 } elsif ($drive->{interface
} eq 'scsi') {
1389 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1390 my $unit = $drive->{index} % $maxdev;
1391 my $devicetype = 'hd';
1393 if (drive_is_cdrom
($drive)) {
1396 if ($drive->{file
} =~ m
|^/|) {
1397 $path = $drive->{file
};
1398 if (my $info = path_is_scsi
($path)) {
1399 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1400 $devicetype = 'block';
1401 } elsif ($info->{type
} == 1) { # tape
1402 $devicetype = 'generic';
1406 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1409 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1410 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, kvm_user_version
());
1411 if ($path =~ m/^iscsi\:\/\
// &&
1412 !min_version
($version, 4, 1)) {
1413 $devicetype = 'generic';
1417 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1418 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1420 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1423 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1424 $device .= ",rotation_rate=1";
1426 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1428 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1429 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1430 my $controller = int($drive->{index} / $maxdev);
1431 my $unit = $drive->{index} % $maxdev;
1432 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1434 $device = "ide-$devicetype";
1435 if ($drive->{interface
} eq 'ide') {
1436 $device .= ",bus=ide.$controller,unit=$unit";
1438 $device .= ",bus=ahci$controller.$unit";
1440 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1442 if ($devicetype eq 'hd') {
1443 if (my $model = $drive->{model
}) {
1444 $model = URI
::Escape
::uri_unescape
($model);
1445 $device .= ",model=$model";
1447 if ($drive->{ssd
}) {
1448 $device .= ",rotation_rate=1";
1451 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1452 } elsif ($drive->{interface
} eq 'usb') {
1454 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1456 die "unsupported interface type";
1459 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1461 if (my $serial = $drive->{serial
}) {
1462 $serial = URI
::Escape
::uri_unescape
($serial);
1463 $device .= ",serial=$serial";
1470 sub get_initiator_name
{
1473 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return undef;
1474 while (defined(my $line = <$fh>)) {
1475 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1484 sub print_drive_commandline_full
{
1485 my ($storecfg, $vmid, $drive) = @_;
1488 my $volid = $drive->{file
};
1491 if (drive_is_cdrom
($drive)) {
1492 $path = get_iso_path
($storecfg, $vmid, $volid);
1494 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1496 $path = PVE
::Storage
::path
($storecfg, $volid);
1497 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1498 $format = qemu_img_format
($scfg, $volname);
1506 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1507 foreach my $o (@qemu_drive_options) {
1508 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1511 # snapshot only accepts on|off
1512 if (defined($drive->{snapshot
})) {
1513 my $v = $drive->{snapshot
} ?
'on' : 'off';
1514 $opts .= ",snapshot=$v";
1517 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1518 my ($dir, $qmpname) = @$type;
1519 if (my $v = $drive->{"mbps$dir"}) {
1520 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1522 if (my $v = $drive->{"mbps${dir}_max"}) {
1523 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1525 if (my $v = $drive->{"bps${dir}_max_length"}) {
1526 $opts .= ",throttling.bps$qmpname-max-length=$v";
1528 if (my $v = $drive->{"iops${dir}"}) {
1529 $opts .= ",throttling.iops$qmpname=$v";
1531 if (my $v = $drive->{"iops${dir}_max"}) {
1532 $opts .= ",throttling.iops$qmpname-max=$v";
1534 if (my $v = $drive->{"iops${dir}_max_length"}) {
1535 $opts .= ",throttling.iops$qmpname-max-length=$v";
1539 $opts .= ",format=$format" if $format && !$drive->{format
};
1541 my $cache_direct = 0;
1543 if (my $cache = $drive->{cache
}) {
1544 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1545 } elsif (!drive_is_cdrom
($drive)) {
1546 $opts .= ",cache=none";
1550 # aio native works only with O_DIRECT
1551 if (!$drive->{aio
}) {
1553 $opts .= ",aio=native";
1555 $opts .= ",aio=threads";
1559 if (!drive_is_cdrom
($drive)) {
1561 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1562 $detectzeroes = 'off';
1563 } elsif ($drive->{discard
}) {
1564 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1566 # This used to be our default with discard not being specified:
1567 $detectzeroes = 'on';
1569 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1572 my $pathinfo = $path ?
"file=$path," : '';
1574 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1577 sub print_netdevice_full
{
1578 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1580 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
1582 my $device = $net->{model
};
1583 if ($net->{model
} eq 'virtio') {
1584 $device = 'virtio-net-pci';
1587 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1588 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1589 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1590 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1591 my $vectors = $net->{queues
} * 2 + 2;
1592 $tmpstr .= ",vectors=$vectors,mq=on";
1594 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1596 if ($use_old_bios_files) {
1598 if ($device eq 'virtio-net-pci') {
1599 $romfile = 'pxe-virtio.rom';
1600 } elsif ($device eq 'e1000') {
1601 $romfile = 'pxe-e1000.rom';
1602 } elsif ($device eq 'ne2k') {
1603 $romfile = 'pxe-ne2k_pci.rom';
1604 } elsif ($device eq 'pcnet') {
1605 $romfile = 'pxe-pcnet.rom';
1606 } elsif ($device eq 'rtl8139') {
1607 $romfile = 'pxe-rtl8139.rom';
1609 $tmpstr .= ",romfile=$romfile" if $romfile;
1615 sub print_netdev_full
{
1616 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1619 if ($netid =~ m/^net(\d+)$/) {
1623 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1625 my $ifname = "tap${vmid}i$i";
1627 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1628 die "interface name '$ifname' is too long (max 15 character)\n"
1629 if length($ifname) >= 16;
1631 my $vhostparam = '';
1632 if (is_native
($arch)) {
1633 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1636 my $vmname = $conf->{name
} || "vm$vmid";
1639 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1641 if ($net->{bridge
}) {
1642 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1644 $netdev = "type=user,id=$netid,hostname=$vmname";
1647 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1653 'cirrus' => 'cirrus-vga',
1655 'vmware' => 'vmware-svga',
1656 'virtio' => 'virtio-vga',
1659 sub print_vga_device
{
1660 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1662 my $type = $vga_map->{$vga->{type
}};
1663 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1664 $type = 'virtio-gpu';
1666 my $vgamem_mb = $vga->{memory
};
1668 my $max_outputs = '';
1670 $type = $id ?
'qxl' : 'qxl-vga';
1672 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1673 # set max outputs so linux can have up to 4 qxl displays with one device
1674 if (min_version
($machine_version, 4, 1)) {
1675 $max_outputs = ",max_outputs=4";
1680 die "no devicetype for $vga->{type}\n" if !$type;
1684 if ($vga->{type
} eq 'virtio') {
1685 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1686 $memory = ",max_hostmem=$bytes";
1688 # from https://www.spice-space.org/multiple-monitors.html
1689 $memory = ",vgamem_mb=$vga->{memory}";
1690 my $ram = $vgamem_mb * 4;
1691 my $vram = $vgamem_mb * 2;
1692 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1694 $memory = ",vgamem_mb=$vga->{memory}";
1696 } elsif ($qxlnum && $id) {
1697 $memory = ",ram_size=67108864,vram_size=33554432";
1700 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1701 my $vgaid = "vga" . ($id // '');
1704 if ($q35 && $vgaid eq 'vga') {
1705 # the first display uses pcie.0 bus on q35 machines
1706 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1708 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1711 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1714 sub parse_number_sets
{
1717 foreach my $part (split(/;/, $set)) {
1718 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1719 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1720 push @$res, [ $1, $2 ];
1722 die "invalid range: $part\n";
1731 my $res = PVE
::JSONSchema
::parse_property_string
($numa_fmt, $data);
1732 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1733 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1740 return undef if !$value;
1742 my $res = PVE
::JSONSchema
::parse_property_string
($hostpci_fmt, $value);
1744 my @idlist = split(/;/, $res->{host
});
1745 delete $res->{host
};
1746 foreach my $id (@idlist) {
1747 my $devs = PVE
::SysFSTools
::lspci
($id);
1748 die "no PCI device found for '$id'\n" if !scalar(@$devs);
1749 push @{$res->{pciid
}}, @$devs;
1754 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1758 my $res = eval { PVE
::JSONSchema
::parse_property_string
($net_fmt, $data) };
1763 if (!defined($res->{macaddr
})) {
1764 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1765 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1770 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1771 sub parse_ipconfig
{
1774 my $res = eval { PVE
::JSONSchema
::parse_property_string
($ipconfig_fmt, $data) };
1780 if ($res->{gw
} && !$res->{ip
}) {
1781 warn 'gateway specified without specifying an IP address';
1784 if ($res->{gw6
} && !$res->{ip6
}) {
1785 warn 'IPv6 gateway specified without specifying an IPv6 address';
1788 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1789 warn 'gateway specified together with DHCP';
1792 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1794 warn "IPv6 gateway specified together with $res->{ip6} address";
1798 if (!$res->{ip
} && !$res->{ip6
}) {
1799 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1808 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1811 sub add_random_macs
{
1812 my ($settings) = @_;
1814 foreach my $opt (keys %$settings) {
1815 next if $opt !~ m/^net(\d+)$/;
1816 my $net = parse_net
($settings->{$opt});
1818 $settings->{$opt} = print_net
($net);
1822 sub vm_is_volid_owner
{
1823 my ($storecfg, $vmid, $volid) = @_;
1825 if ($volid !~ m
|^/|) {
1827 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1828 if ($owner && ($owner == $vmid)) {
1836 sub vmconfig_register_unused_drive
{
1837 my ($storecfg, $vmid, $conf, $drive) = @_;
1839 if (drive_is_cloudinit
($drive)) {
1840 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1842 } elsif (!drive_is_cdrom
($drive)) {
1843 my $volid = $drive->{file
};
1844 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1845 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1850 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1854 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1855 format_description
=> 'UUID',
1856 description
=> "Set SMBIOS1 UUID.",
1861 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1862 format_description
=> 'Base64 encoded string',
1863 description
=> "Set SMBIOS1 version.",
1868 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1869 format_description
=> 'Base64 encoded string',
1870 description
=> "Set SMBIOS1 serial number.",
1875 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1876 format_description
=> 'Base64 encoded string',
1877 description
=> "Set SMBIOS1 manufacturer.",
1882 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1883 format_description
=> 'Base64 encoded string',
1884 description
=> "Set SMBIOS1 product ID.",
1889 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1890 format_description
=> 'Base64 encoded string',
1891 description
=> "Set SMBIOS1 SKU string.",
1896 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1897 format_description
=> 'Base64 encoded string',
1898 description
=> "Set SMBIOS1 family string.",
1903 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
1911 my $res = eval { PVE
::JSONSchema
::parse_property_string
($smbios1_fmt, $data) };
1918 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
1921 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
1923 sub parse_watchdog
{
1926 return undef if !$value;
1928 my $res = eval { PVE
::JSONSchema
::parse_property_string
($watchdog_fmt, $value) };
1933 sub parse_guest_agent
{
1936 return {} if !defined($value->{agent
});
1938 my $res = eval { PVE
::JSONSchema
::parse_property_string
($agent_fmt, $value->{agent
}) };
1941 # if the agent is disabled ignore the other potentially set properties
1942 return {} if !$res->{enabled
};
1949 return {} if !$value;
1950 my $res = eval { PVE
::JSONSchema
::parse_property_string
($vga_fmt, $value) };
1958 return undef if !$value;
1960 my $res = eval { PVE
::JSONSchema
::parse_property_string
($rng_fmt, $value) };
1965 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
1966 sub verify_usb_device
{
1967 my ($value, $noerr) = @_;
1969 return $value if parse_usb_device
($value);
1971 return undef if $noerr;
1973 die "unable to parse usb device\n";
1976 # add JSON properties for create and set function
1977 sub json_config_properties
{
1980 foreach my $opt (keys %$confdesc) {
1981 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1982 $opt eq 'runningmachine' || $opt eq 'runningcpu';
1983 $prop->{$opt} = $confdesc->{$opt};
1989 # return copy of $confdesc_cloudinit to generate documentation
1990 sub cloudinit_config_properties
{
1992 return dclone
($confdesc_cloudinit);
1996 my ($key, $value) = @_;
1998 die "unknown setting '$key'\n" if !$confdesc->{$key};
2000 my $type = $confdesc->{$key}->{type
};
2002 if (!defined($value)) {
2003 die "got undefined value\n";
2006 if ($value =~ m/[\n\r]/) {
2007 die "property contains a line feed\n";
2010 if ($type eq 'boolean') {
2011 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2012 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2013 die "type check ('boolean') failed - got '$value'\n";
2014 } elsif ($type eq 'integer') {
2015 return int($1) if $value =~ m/^(\d+)$/;
2016 die "type check ('integer') failed - got '$value'\n";
2017 } elsif ($type eq 'number') {
2018 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2019 die "type check ('number') failed - got '$value'\n";
2020 } elsif ($type eq 'string') {
2021 if (my $fmt = $confdesc->{$key}->{format
}) {
2022 PVE
::JSONSchema
::check_format
($fmt, $value);
2025 $value =~ s/^\"(.*)\"$/$1/;
2028 die "internal error"
2033 my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2035 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2037 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2039 if ($conf->{template
}) {
2040 # check if any base image is still used by a linked clone
2041 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2042 my ($ds, $drive) = @_;
2043 return if drive_is_cdrom
($drive);
2045 my $volid = $drive->{file
};
2046 return if !$volid || $volid =~ m
|^/|;
2048 die "base volume '$volid' is still in use by linked cloned\n"
2049 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2054 # only remove disks owned by this VM
2055 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2056 my ($ds, $drive) = @_;
2057 return if drive_is_cdrom
($drive, 1);
2059 my $volid = $drive->{file
};
2060 return if !$volid || $volid =~ m
|^/|;
2062 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2063 return if !$path || !$owner || ($owner != $vmid);
2065 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2066 warn "Could not remove disk '$volid', check manually: $@" if $@;
2069 # also remove unused disk
2070 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2071 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2072 my ($volid, $sid, $volname, $d) = @_;
2073 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2077 if (defined $replacement_conf) {
2078 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2080 PVE
::QemuConfig-
>destroy_config($vmid);
2084 sub parse_vm_config
{
2085 my ($filename, $raw) = @_;
2087 return undef if !defined($raw);
2090 digest
=> Digest
::SHA
::sha1_hex
($raw),
2095 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2096 || die "got strange filename '$filename'";
2104 my @lines = split(/\n/, $raw);
2105 foreach my $line (@lines) {
2106 next if $line =~ m/^\s*$/;
2108 if ($line =~ m/^\[PENDING\]\s*$/i) {
2109 $section = 'pending';
2110 if (defined($descr)) {
2112 $conf->{description
} = $descr;
2115 $conf = $res->{$section} = {};
2118 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2120 if (defined($descr)) {
2122 $conf->{description
} = $descr;
2125 $conf = $res->{snapshots
}->{$section} = {};
2129 if ($line =~ m/^\#(.*)\s*$/) {
2130 $descr = '' if !defined($descr);
2131 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2135 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2136 $descr = '' if !defined($descr);
2137 $descr .= PVE
::Tools
::decode_text
($2);
2138 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2139 $conf->{snapstate
} = $1;
2140 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2143 $conf->{$key} = $value;
2144 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2146 if ($section eq 'pending') {
2147 $conf->{delete} = $value; # we parse this later
2149 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2151 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2154 eval { $value = check_type
($key, $value); };
2156 warn "vm $vmid - unable to parse value of '$key' - $@";
2158 $key = 'ide2' if $key eq 'cdrom';
2159 my $fmt = $confdesc->{$key}->{format
};
2160 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2161 my $v = parse_drive
($key, $value);
2162 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2163 $v->{file
} = $volid;
2164 $value = print_drive
($v);
2166 warn "vm $vmid - unable to parse value of '$key'\n";
2171 $conf->{$key} = $value;
2176 if (defined($descr)) {
2178 $conf->{description
} = $descr;
2180 delete $res->{snapstate
}; # just to be sure
2185 sub write_vm_config
{
2186 my ($filename, $conf) = @_;
2188 delete $conf->{snapstate
}; # just to be sure
2190 if ($conf->{cdrom
}) {
2191 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2192 $conf->{ide2
} = $conf->{cdrom
};
2193 delete $conf->{cdrom
};
2196 # we do not use 'smp' any longer
2197 if ($conf->{sockets
}) {
2198 delete $conf->{smp
};
2199 } elsif ($conf->{smp
}) {
2200 $conf->{sockets
} = $conf->{smp
};
2201 delete $conf->{cores
};
2202 delete $conf->{smp
};
2205 my $used_volids = {};
2207 my $cleanup_config = sub {
2208 my ($cref, $pending, $snapname) = @_;
2210 foreach my $key (keys %$cref) {
2211 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2212 $key eq 'snapstate' || $key eq 'pending';
2213 my $value = $cref->{$key};
2214 if ($key eq 'delete') {
2215 die "propertry 'delete' is only allowed in [PENDING]\n"
2217 # fixme: check syntax?
2220 eval { $value = check_type
($key, $value); };
2221 die "unable to parse value of '$key' - $@" if $@;
2223 $cref->{$key} = $value;
2225 if (!$snapname && is_valid_drivename
($key)) {
2226 my $drive = parse_drive
($key, $value);
2227 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2232 &$cleanup_config($conf);
2234 &$cleanup_config($conf->{pending
}, 1);
2236 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2237 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2238 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2241 # remove 'unusedX' settings if we re-add a volume
2242 foreach my $key (keys %$conf) {
2243 my $value = $conf->{$key};
2244 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2245 delete $conf->{$key};
2249 my $generate_raw_config = sub {
2250 my ($conf, $pending) = @_;
2254 # add description as comment to top of file
2255 if (defined(my $descr = $conf->{description
})) {
2257 foreach my $cl (split(/\n/, $descr)) {
2258 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2261 $raw .= "#\n" if $pending;
2265 foreach my $key (sort keys %$conf) {
2266 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2267 $raw .= "$key: $conf->{$key}\n";
2272 my $raw = &$generate_raw_config($conf);
2274 if (scalar(keys %{$conf->{pending
}})){
2275 $raw .= "\n[PENDING]\n";
2276 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2279 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2280 $raw .= "\n[$snapname]\n";
2281 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2291 # we use static defaults from our JSON schema configuration
2292 foreach my $key (keys %$confdesc) {
2293 if (defined(my $default = $confdesc->{$key}->{default})) {
2294 $res->{$key} = $default;
2302 my $vmlist = PVE
::Cluster
::get_vmlist
();
2304 return $res if !$vmlist || !$vmlist->{ids
};
2305 my $ids = $vmlist->{ids
};
2306 my $nodename = nodename
();
2308 foreach my $vmid (keys %$ids) {
2309 my $d = $ids->{$vmid};
2310 next if !$d->{node
} || $d->{node
} ne $nodename;
2311 next if !$d->{type
} || $d->{type
} ne 'qemu';
2312 $res->{$vmid}->{exists} = 1;
2317 # test if VM uses local resources (to prevent migration)
2318 sub check_local_resources
{
2319 my ($conf, $noerr) = @_;
2323 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2324 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2326 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2328 foreach my $k (keys %$conf) {
2329 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2330 # sockets are safe: they will recreated be on the target side post-migrate
2331 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2332 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2335 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2340 # check if used storages are available on all nodes (use by migrate)
2341 sub check_storage_availability
{
2342 my ($storecfg, $conf, $node) = @_;
2344 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2345 my ($ds, $drive) = @_;
2347 my $volid = $drive->{file
};
2350 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2353 # check if storage is available on both nodes
2354 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2355 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2359 # list nodes where all VM images are available (used by has_feature API)
2361 my ($conf, $storecfg) = @_;
2363 my $nodelist = PVE
::Cluster
::get_nodelist
();
2364 my $nodehash = { map { $_ => 1 } @$nodelist };
2365 my $nodename = nodename
();
2367 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2368 my ($ds, $drive) = @_;
2370 my $volid = $drive->{file
};
2373 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2375 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2376 if ($scfg->{disable
}) {
2378 } elsif (my $avail = $scfg->{nodes
}) {
2379 foreach my $node (keys %$nodehash) {
2380 delete $nodehash->{$node} if !$avail->{$node};
2382 } elsif (!$scfg->{shared
}) {
2383 foreach my $node (keys %$nodehash) {
2384 delete $nodehash->{$node} if $node ne $nodename
2393 sub check_local_storage_availability
{
2394 my ($conf, $storecfg) = @_;
2396 my $nodelist = PVE
::Cluster
::get_nodelist
();
2397 my $nodehash = { map { $_ => {} } @$nodelist };
2399 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2400 my ($ds, $drive) = @_;
2402 my $volid = $drive->{file
};
2405 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2407 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2409 if ($scfg->{disable
}) {
2410 foreach my $node (keys %$nodehash) {
2411 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2413 } elsif (my $avail = $scfg->{nodes
}) {
2414 foreach my $node (keys %$nodehash) {
2415 if (!$avail->{$node}) {
2416 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2423 foreach my $node (values %$nodehash) {
2424 if (my $unavail = $node->{unavailable_storages
}) {
2425 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2432 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2434 my ($vmid, $nocheck, $node) = @_;
2436 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2437 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2442 my $vzlist = config_list
();
2444 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2446 while (defined(my $de = $fd->read)) {
2447 next if $de !~ m/^(\d+)\.pid$/;
2449 next if !defined($vzlist->{$vmid});
2450 if (my $pid = check_running
($vmid)) {
2451 $vzlist->{$vmid}->{pid
} = $pid;
2458 our $vmstatus_return_properties = {
2459 vmid
=> get_standard_option
('pve-vmid'),
2461 description
=> "Qemu process status.",
2463 enum
=> ['stopped', 'running'],
2466 description
=> "Maximum memory in bytes.",
2469 renderer
=> 'bytes',
2472 description
=> "Root disk size in bytes.",
2475 renderer
=> 'bytes',
2478 description
=> "VM name.",
2483 description
=> "Qemu QMP agent status.",
2488 description
=> "PID of running qemu process.",
2493 description
=> "Uptime.",
2496 renderer
=> 'duration',
2499 description
=> "Maximum usable CPUs.",
2504 description
=> "The current config lock, if any.",
2509 description
=> "The current configured tags, if any",
2515 my $last_proc_pid_stat;
2517 # get VM status information
2518 # This must be fast and should not block ($full == false)
2519 # We only query KVM using QMP if $full == true (this can be slow)
2521 my ($opt_vmid, $full) = @_;
2525 my $storecfg = PVE
::Storage
::config
();
2527 my $list = vzlist
();
2528 my $defaults = load_defaults
();
2530 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2532 my $cpucount = $cpuinfo->{cpus
} || 1;
2534 foreach my $vmid (keys %$list) {
2535 next if $opt_vmid && ($vmid ne $opt_vmid);
2537 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2539 my $d = { vmid
=> $vmid };
2540 $d->{pid
} = $list->{$vmid}->{pid
};
2542 # fixme: better status?
2543 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2545 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2546 if (defined($size)) {
2547 $d->{disk
} = 0; # no info available
2548 $d->{maxdisk
} = $size;
2554 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2555 * ($conf->{cores
} || $defaults->{cores
});
2556 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2557 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2559 $d->{name
} = $conf->{name
} || "VM $vmid";
2560 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2561 : $defaults->{memory
}*(1024*1024);
2563 if ($conf->{balloon
}) {
2564 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2565 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2566 : $defaults->{shares
};
2577 $d->{diskwrite
} = 0;
2579 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
2581 $d->{serial
} = 1 if conf_has_serial
($conf);
2582 $d->{lock} = $conf->{lock} if $conf->{lock};
2583 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2588 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2589 foreach my $dev (keys %$netdev) {
2590 next if $dev !~ m/^tap([1-9]\d*)i/;
2592 my $d = $res->{$vmid};
2595 $d->{netout
} += $netdev->{$dev}->{receive
};
2596 $d->{netin
} += $netdev->{$dev}->{transmit
};
2599 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
2600 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
2605 my $ctime = gettimeofday
;
2607 foreach my $vmid (keys %$list) {
2609 my $d = $res->{$vmid};
2610 my $pid = $d->{pid
};
2613 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2614 next if !$pstat; # not running
2616 my $used = $pstat->{utime} + $pstat->{stime
};
2618 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2620 if ($pstat->{vsize
}) {
2621 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2624 my $old = $last_proc_pid_stat->{$pid};
2626 $last_proc_pid_stat->{$pid} = {
2634 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2636 if ($dtime > 1000) {
2637 my $dutime = $used - $old->{used
};
2639 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2640 $last_proc_pid_stat->{$pid} = {
2646 $d->{cpu
} = $old->{cpu
};
2650 return $res if !$full;
2652 my $qmpclient = PVE
::QMPClient-
>new();
2654 my $ballooncb = sub {
2655 my ($vmid, $resp) = @_;
2657 my $info = $resp->{'return'};
2658 return if !$info->{max_mem
};
2660 my $d = $res->{$vmid};
2662 # use memory assigned to VM
2663 $d->{maxmem
} = $info->{max_mem
};
2664 $d->{balloon
} = $info->{actual
};
2666 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2667 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2668 $d->{freemem
} = $info->{free_mem
};
2671 $d->{ballooninfo
} = $info;
2674 my $blockstatscb = sub {
2675 my ($vmid, $resp) = @_;
2676 my $data = $resp->{'return'} || [];
2677 my $totalrdbytes = 0;
2678 my $totalwrbytes = 0;
2680 for my $blockstat (@$data) {
2681 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2682 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2684 $blockstat->{device
} =~ s/drive-//;
2685 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2687 $res->{$vmid}->{diskread
} = $totalrdbytes;
2688 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2691 my $statuscb = sub {
2692 my ($vmid, $resp) = @_;
2694 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2695 # this fails if ballon driver is not loaded, so this must be
2696 # the last commnand (following command are aborted if this fails).
2697 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2699 my $status = 'unknown';
2700 if (!defined($status = $resp->{'return'}->{status
})) {
2701 warn "unable to get VM status\n";
2705 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2708 foreach my $vmid (keys %$list) {
2709 next if $opt_vmid && ($vmid ne $opt_vmid);
2710 next if !$res->{$vmid}->{pid
}; # not running
2711 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2714 $qmpclient->queue_execute(undef, 2);
2716 foreach my $vmid (keys %$list) {
2717 next if $opt_vmid && ($vmid ne $opt_vmid);
2718 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2724 sub conf_has_serial
{
2727 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2728 if ($conf->{"serial$i"}) {
2736 sub conf_has_audio
{
2737 my ($conf, $id) = @_;
2740 my $audio = $conf->{"audio$id"};
2741 return undef if !defined($audio);
2743 my $audioproperties = PVE
::JSONSchema
::parse_property_string
($audio_fmt, $audio);
2744 my $audiodriver = $audioproperties->{driver
} // 'spice';
2747 dev
=> $audioproperties->{device
},
2748 dev_id
=> "audiodev$id",
2749 backend
=> $audiodriver,
2750 backend_id
=> "$audiodriver-backend${id}",
2755 my ($audio, $audiopciaddr, $machine_version) = @_;
2759 my $id = $audio->{dev_id
};
2761 if (min_version
($machine_version, 4, 2)) {
2762 $audiodev = ",audiodev=$audio->{backend_id}";
2765 if ($audio->{dev
} eq 'AC97') {
2766 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2767 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
2768 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2769 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2770 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2772 die "unkown audio device '$audio->{dev}', implement me!";
2775 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2780 sub vga_conf_has_spice
{
2783 my $vgaconf = parse_vga
($vga);
2784 my $vgatype = $vgaconf->{type
};
2785 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2792 return get_host_arch
() eq $arch;
2797 return $conf->{arch
} // get_host_arch
();
2800 my $default_machines = {
2805 sub get_vm_machine
{
2806 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2808 my $machine = $forcemachine || $conf->{machine
};
2810 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2812 $machine ||= $default_machines->{$arch};
2813 if ($add_pve_version) {
2814 $kvmversion //= kvm_user_version
();
2815 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
2816 $machine .= "+pve$pvever";
2820 if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2821 # for version-pinned machines that do not include a pve-version (e.g.
2822 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2823 $machine .= '+pve0';
2829 sub get_ovmf_files
($) {
2832 my $ovmf = $OVMF->{$arch}
2833 or die "no OVMF images known for architecture '$arch'\n";
2839 aarch64
=> '/usr/bin/qemu-system-aarch64',
2840 x86_64
=> '/usr/bin/qemu-system-x86_64',
2842 sub get_command_for_arch
($) {
2844 return '/usr/bin/kvm' if is_native
($arch);
2846 my $cmd = $Arch2Qemu->{$arch}
2847 or die "don't know how to emulate architecture '$arch'\n";
2851 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2852 # to use in a QEMU command line (-cpu element), first array_intersect the result
2853 # of query_supported_ with query_understood_. This is necessary because:
2855 # a) query_understood_ returns flags the host cannot use and
2856 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2857 # flags, but CPU settings - with most of them being flags. Those settings
2858 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
2860 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2861 # expensive. If you need the value returned from this, you can get it much
2862 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2863 # $accel being 'kvm' or 'tcg'.
2865 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2866 # changes, automatically populating pmxcfs.
2868 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2869 # since kvm and tcg machines support different flags
2871 sub query_supported_cpu_flags
{
2874 $arch //= get_host_arch
();
2875 my $default_machine = $default_machines->{$arch};
2879 # FIXME: Once this is merged, the code below should work for ARM as well:
2880 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2881 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2884 my $kvm_supported = defined(kvm_version
());
2885 my $qemu_cmd = get_command_for_arch
($arch);
2887 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
2889 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2890 my $query_supported_run_qemu = sub {
2896 '-machine', $default_machine,
2898 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2899 '-mon', 'chardev=qmp,mode=control',
2900 '-pidfile', $pidfile,
2905 push @$cmd, '-accel', 'tcg';
2908 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
2909 die "QEMU flag querying VM exited with code " . $rc if $rc;
2912 my $cmd_result = mon_cmd
(
2914 'query-cpu-model-expansion',
2916 model
=> { name
=> 'host' }
2919 my $props = $cmd_result->{model
}->{props
};
2920 foreach my $prop (keys %$props) {
2921 next if $props->{$prop} ne '1';
2922 # QEMU returns some flags multiple times, with '_', '.' or '-'
2923 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2924 # We only keep those with underscores, to match /proc/cpuinfo
2925 $prop =~ s/\.|-/_/g;
2926 $flags->{$prop} = 1;
2931 # force stop with 10 sec timeout and 'nocheck'
2932 # always stop, even if QMP failed
2933 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
2937 return [ sort keys %$flags ];
2940 # We need to query QEMU twice, since KVM and TCG have different supported flags
2941 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
2942 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
2943 warn "warning: failed querying supported tcg flags: $@\n" if $@;
2945 if ($kvm_supported) {
2946 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
2947 warn "warning: failed querying supported kvm flags: $@\n" if $@;
2954 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2955 my $understood_cpu_flag_dir = "/usr/share/kvm";
2956 sub query_understood_cpu_flags
{
2957 my $arch = get_host_arch
();
2958 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2960 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2963 my $raw = file_get_contents
($filepath);
2964 $raw =~ s/^\s+|\s+$//g;
2965 my @flags = split(/\s+/, $raw);
2970 sub config_to_command
{
2971 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2974 my $globalFlags = [];
2975 my $machineFlags = [];
2980 my $ostype = $conf->{ostype
};
2981 my $winversion = windows_version
($ostype);
2982 my $kvm = $conf->{kvm
};
2983 my $nodename = nodename
();
2985 my $arch = get_vm_arch
($conf);
2986 my $kvm_binary = get_command_for_arch
($arch);
2987 my $kvmver = kvm_user_version
($kvm_binary);
2989 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2990 $kvmver //= "undefined";
2991 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2994 my $add_pve_version = min_version
($kvmver, 4, 1);
2996 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
2997 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, $kvmver);
2998 $kvm //= 1 if is_native
($arch);
3000 $machine_version =~ m/(\d+)\.(\d+)/;
3001 my ($machine_major, $machine_minor) = ($1, $2);
3003 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3004 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3005 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3006 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3007 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3008 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3009 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
3010 ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
3013 # if a specific +pve version is required for a feature, use $version_guard
3014 # instead of min_version to allow machines to be run with the minimum
3016 my $required_pve_version = 0;
3017 my $version_guard = sub {
3018 my ($major, $minor, $pve) = @_;
3019 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3020 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3021 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3022 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3027 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3028 if !defined kvm_version
();
3031 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3032 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3033 my $use_old_bios_files = undef;
3034 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3036 my $cpuunits = defined($conf->{cpuunits
}) ?
3037 $conf->{cpuunits
} : $defaults->{cpuunits
};
3039 push @$cmd, $kvm_binary;
3041 push @$cmd, '-id', $vmid;
3043 my $vmname = $conf->{name
} || "vm$vmid";
3045 push @$cmd, '-name', $vmname;
3049 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3050 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3051 push @$cmd, '-mon', "chardev=qmp,mode=control";
3053 if (min_version
($machine_version, 2, 12)) {
3054 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3055 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3058 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3060 push @$cmd, '-daemonize';
3062 if ($conf->{smbios1
}) {
3063 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3064 if ($smbios_conf->{base64
}) {
3065 # Do not pass base64 flag to qemu
3066 delete $smbios_conf->{base64
};
3067 my $smbios_string = "";
3068 foreach my $key (keys %$smbios_conf) {
3070 if ($key eq "uuid") {
3071 $value = $smbios_conf->{uuid
}
3073 $value = decode_base64
($smbios_conf->{$key});
3075 # qemu accepts any binary data, only commas need escaping by double comma
3077 $smbios_string .= "," . $key . "=" . $value if $value;
3079 push @$cmd, '-smbios', "type=1" . $smbios_string;
3081 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3085 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3086 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3087 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3089 my ($path, $format);
3090 if (my $efidisk = $conf->{efidisk0
}) {
3091 my $d = parse_drive
('efidisk0', $efidisk);
3092 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3093 $format = $d->{format
};
3095 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3096 if (!defined($format)) {
3097 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3098 $format = qemu_img_format
($scfg, $volname);
3102 die "efidisk format must be specified\n"
3103 if !defined($format);
3106 warn "no efidisk configured! Using temporary efivars disk.\n";
3107 $path = "/tmp/$vmid-ovmf.fd";
3108 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3114 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3115 $size_str = ",size=" . (-s
$ovmf_vars);
3118 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3119 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3124 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3125 if (min_version
($machine_version, 4, 0)) {
3126 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3128 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3132 if ($conf->{vmgenid
}) {
3133 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3136 # add usb controllers
3137 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
($conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3138 push @$devices, @usbcontrollers if @usbcontrollers;
3139 my $vga = parse_vga
($conf->{vga
});
3141 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3142 $vga->{type
} = 'qxl' if $qxlnum;
3144 if (!$vga->{type
}) {
3145 if ($arch eq 'aarch64') {
3146 $vga->{type
} = 'virtio';
3147 } elsif (min_version
($machine_version, 2, 9)) {
3148 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3150 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3154 # enable absolute mouse coordinates (needed by vnc)
3156 if (defined($conf->{tablet
})) {
3157 $tablet = $conf->{tablet
};
3159 $tablet = $defaults->{tablet
};
3160 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3161 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3165 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3166 my $kbd = print_keyboarddevice_full
($conf, $arch);
3167 push @$devices, '-device', $kbd if defined($kbd);
3171 my $gpu_passthrough;
3174 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
3175 my $id = "hostpci$i";
3176 my $d = parse_hostpci
($conf->{$id});
3179 if (my $pcie = $d->{pcie
}) {
3180 die "q35 machine model is not enabled" if !$q35;
3181 # win7 wants to have the pcie devices directly on the pcie bus
3182 # instead of in the root port
3183 if ($winversion == 7) {
3184 $pciaddr = print_pcie_addr
("${id}bus0");
3186 # add more root ports if needed, 4 are present by default
3187 # by pve-q35 cfgs, rest added here on demand.
3189 push @$devices, '-device', print_pcie_root_port
($i);
3191 $pciaddr = print_pcie_addr
($id);
3194 $pciaddr = print_pci_addr
($id, $bridges, $arch, $machine_type);
3198 if ($d->{'x-vga'}) {
3199 $xvga = ',x-vga=on' if !($conf->{bios
} && $conf->{bios
} eq 'ovmf');
3201 $vga->{type
} = 'none' if !defined($conf->{vga
});
3202 $gpu_passthrough = 1;
3205 my $pcidevices = $d->{pciid
};
3206 my $multifunction = 1 if @$pcidevices > 1;
3209 if ($d->{mdev
} && scalar(@$pcidevices) == 1) {
3210 my $pci_id = $pcidevices->[0]->{id
};
3211 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $i);
3212 $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3213 } elsif ($d->{mdev
}) {
3214 warn "ignoring mediated device '$id' with multifunction device\n";
3218 foreach my $pcidevice (@$pcidevices) {
3219 my $devicestr = "vfio-pci";
3222 $devicestr .= ",sysfsdev=$sysfspath";
3224 $devicestr .= ",host=$pcidevice->{id}";
3227 my $mf_addr = $multifunction ?
".$j" : '';
3228 $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3231 $devicestr .= ',rombar=0' if defined($d->{rombar
}) && !$d->{rombar
};
3232 $devicestr .= "$xvga";
3233 $devicestr .= ",multifunction=on" if $multifunction;
3234 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile
};
3237 push @$devices, '-device', $devicestr;
3243 my $usb_dev_features = {};
3244 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3246 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
($conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features);
3247 push @$devices, @usbdevices if @usbdevices;
3249 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3250 if (my $path = $conf->{"serial$i"}) {
3251 if ($path eq 'socket') {
3252 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3253 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3254 # On aarch64, serial0 is the UART device. Qemu only allows
3255 # connecting UART devices via the '-serial' command line, as
3256 # the device has a fixed slot on the hardware...
3257 if ($arch eq 'aarch64' && $i == 0) {
3258 push @$devices, '-serial', "chardev:serial$i";
3260 push @$devices, '-device', "isa-serial,chardev=serial$i";
3263 die "no such serial device\n" if ! -c
$path;
3264 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3265 push @$devices, '-device', "isa-serial,chardev=serial$i";
3271 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3272 if (my $path = $conf->{"parallel$i"}) {
3273 die "no such parallel device\n" if ! -c
$path;
3274 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3275 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3276 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3280 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3281 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3282 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3283 push @$devices, @$audio_devs;
3287 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3288 $sockets = $conf->{sockets
} if $conf->{sockets
};
3290 my $cores = $conf->{cores
} || 1;
3292 my $maxcpus = $sockets * $cores;
3294 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3296 my $allowed_vcpus = $cpuinfo->{cpus
};
3298 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3299 if ($allowed_vcpus < $maxcpus);
3301 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3303 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3304 for (my $i = 2; $i <= $vcpus; $i++) {
3305 my $cpustr = print_cpu_device
($conf,$i);
3306 push @$cmd, '-device', $cpustr;
3311 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3313 push @$cmd, '-nodefaults';
3315 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
3317 my $bootindex_hash = {};
3319 foreach my $o (split(//, $bootorder)) {
3320 $bootindex_hash->{$o} = $i*100;
3324 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3326 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3328 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3330 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3331 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3332 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3333 push @$cmd, '-vnc', "unix:$socket,password";
3335 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3336 push @$cmd, '-nographic';
3340 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3341 my $useLocaltime = $conf->{localtime};
3343 if ($winversion >= 5) { # windows
3344 $useLocaltime = 1 if !defined($conf->{localtime});
3346 # use time drift fix when acpi is enabled
3347 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3348 $tdf = 1 if !defined($conf->{tdf
});
3352 if ($winversion >= 6) {
3353 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3354 push @$cmd, '-no-hpet';
3357 push @$rtcFlags, 'driftfix=slew' if $tdf;
3359 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3360 push @$rtcFlags, "base=$conf->{startdate}";
3361 } elsif ($useLocaltime) {
3362 push @$rtcFlags, 'base=localtime';
3366 push @$cmd, '-cpu', $forcecpu;
3368 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3371 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3373 push @$cmd, '-S' if $conf->{freeze
};
3375 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3377 my $guest_agent = parse_guest_agent
($conf);
3379 if ($guest_agent->{enabled
}) {
3380 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3381 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3383 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3384 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3385 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3386 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3387 } elsif ($guest_agent->{type
} eq 'isa') {
3388 push @$devices, '-device', "isa-serial,chardev=qga0";
3392 my $rng = parse_rng
($conf->{rng0
}) if $conf->{rng0
};
3393 if ($rng && &$version_guard(4, 1, 2)) {
3394 check_rng_source
($rng->{source
});
3396 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3397 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3398 my $limiter_str = "";
3400 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3403 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3404 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3405 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3413 for (my $i = 1; $i < $qxlnum; $i++){
3414 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3417 # assume other OS works like Linux
3418 my ($ram, $vram) = ("134217728", "67108864");
3419 if ($vga->{memory
}) {
3420 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3421 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3423 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3424 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3428 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3430 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3431 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3432 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3434 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3435 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3436 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3438 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3439 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3441 my $spice_enhancement = PVE
::JSONSchema
::parse_property_string
($spice_enhancements_fmt, $conf->{spice_enhancements
} // '');
3442 if ($spice_enhancement->{foldersharing
}) {
3443 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3444 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3447 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3448 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming
};
3449 push @$devices, '-spice', "$spice_opts";
3452 # enable balloon by default, unless explicitly disabled
3453 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3454 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3455 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3458 if ($conf->{watchdog
}) {
3459 my $wdopts = parse_watchdog
($conf->{watchdog
});
3460 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3461 my $watchdog = $wdopts->{model
} || 'i6300esb';
3462 push @$devices, '-device', "$watchdog$pciaddr";
3463 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3467 my $scsicontroller = {};
3468 my $ahcicontroller = {};
3469 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3471 # Add iscsi initiator name if available
3472 if (my $initiator = get_initiator_name
()) {
3473 push @$devices, '-iscsi', "initiator-name=$initiator";
3476 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3477 my ($ds, $drive) = @_;
3479 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3480 push @$vollist, $drive->{file
};
3483 # ignore efidisk here, already added in bios/fw handling code above
3484 return if $drive->{interface
} eq 'efidisk';
3486 $use_virtio = 1 if $ds =~ m/^virtio/;
3488 if (drive_is_cdrom
($drive)) {
3489 if ($bootindex_hash->{d
}) {
3490 $drive->{bootindex
} = $bootindex_hash->{d
};
3491 $bootindex_hash->{d
} += 1;
3494 if ($bootindex_hash->{c
}) {
3495 $drive->{bootindex
} = $bootindex_hash->{c
} if $conf->{bootdisk
} && ($conf->{bootdisk
} eq $ds);
3496 $bootindex_hash->{c
} += 1;
3500 if ($drive->{interface
} eq 'virtio'){
3501 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3504 if ($drive->{interface
} eq 'scsi') {
3506 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3508 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3509 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3511 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3512 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3515 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3516 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3517 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3518 } elsif ($drive->{iothread
}) {
3519 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3523 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3524 $queues = ",num_queues=$drive->{queues}";
3527 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3528 $scsicontroller->{$controller}=1;
3531 if ($drive->{interface
} eq 'sata') {
3532 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3533 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3534 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3535 $ahcicontroller->{$controller}=1;
3538 my $drive_cmd = print_drive_commandline_full
($storecfg, $vmid, $drive);
3539 push @$devices, '-drive',$drive_cmd;
3540 push @$devices, '-device', print_drivedevice_full
($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3543 for (my $i = 0; $i < $MAX_NETS; $i++) {
3544 next if !$conf->{"net$i"};
3545 my $d = parse_net
($conf->{"net$i"});
3548 $use_virtio = 1 if $d->{model
} eq 'virtio';
3550 if ($bootindex_hash->{n
}) {
3551 $d->{bootindex
} = $bootindex_hash->{n
};
3552 $bootindex_hash->{n
} += 1;
3555 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, "net$i");
3556 push @$devices, '-netdev', $netdevfull;
3558 my $netdevicefull = print_netdevice_full
($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3559 push @$devices, '-device', $netdevicefull;
3562 if ($conf->{ivshmem
}) {
3563 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3567 $bus = print_pcie_addr
("ivshmem");
3569 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3572 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3573 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3575 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3576 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3579 # pci.4 is nested in pci.1
3580 $bridges->{1} = 1 if $bridges->{4};
3584 if (min_version
($machine_version, 2, 3)) {
3589 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3593 for my $k (sort {$b cmp $a} keys %$bridges) {
3594 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3595 $pciaddr = print_pci_addr
("pci.$k", undef, $arch, $machine_type);
3596 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3598 # add after -readconfig pve-q35.cfg
3599 splice @$devices, 2, 0, '-device', $devstr;
3601 unshift @$devices, '-device', $devstr if $k > 0;
3606 push @$machineFlags, 'accel=tcg';
3609 my $machine_type_min = $machine_type;
3610 if ($add_pve_version) {
3611 $machine_type_min =~ s/\+pve\d+$//;
3612 $machine_type_min .= "+pve$required_pve_version";
3614 push @$machineFlags, "type=${machine_type_min}";
3616 push @$cmd, @$devices;
3617 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3618 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3619 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3621 if (my $vmstate = $conf->{vmstate
}) {
3622 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3623 push @$vollist, $vmstate;
3624 push @$cmd, '-loadstate', $statepath;
3625 print "activating and using '$vmstate' as vmstate\n";
3629 if ($conf->{args
}) {
3630 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3634 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3637 sub check_rng_source
{
3640 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3641 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3644 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3645 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
3646 # Needs to abort, otherwise QEMU crashes on first rng access.
3647 # Note that rng_current cannot be changed to 'none' manually, so
3648 # once the VM is past this point, it is no longer an issue.
3649 die "Cannot start VM with passed-through RNG device: '/dev/hwrng'"
3650 . " exists, but '$rng_current' is set to 'none'. Ensure that"
3651 . " a compatible hardware-RNG is attached to the host.\n";
3658 my $res = mon_cmd
($vmid, 'query-spice');
3660 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3663 sub vm_devices_list
{
3666 my $res = mon_cmd
($vmid, 'query-pci');
3667 my $devices_to_check = [];
3669 foreach my $pcibus (@$res) {
3670 push @$devices_to_check, @{$pcibus->{devices
}},
3673 while (@$devices_to_check) {
3675 for my $d (@$devices_to_check) {
3676 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3677 next if !$d->{'pci_bridge'};
3679 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3680 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3682 $devices_to_check = $to_check;
3685 my $resblock = mon_cmd
($vmid, 'query-block');
3686 foreach my $block (@$resblock) {
3687 if($block->{device
} =~ m/^drive-(\S+)/){
3692 my $resmice = mon_cmd
($vmid, 'query-mice');
3693 foreach my $mice (@$resmice) {
3694 if ($mice->{name
} eq 'QEMU HID Tablet') {
3695 $devices->{tablet
} = 1;
3700 # for usb devices there is no query-usb
3701 # but we can iterate over the entries in
3702 # qom-list path=/machine/peripheral
3703 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3704 foreach my $per (@$resperipheral) {
3705 if ($per->{name
} =~ m/^usb\d+$/) {
3706 $devices->{$per->{name
}} = 1;
3714 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3716 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3718 my $devices_list = vm_devices_list
($vmid);
3719 return 1 if defined($devices_list->{$deviceid});
3721 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3723 if ($deviceid eq 'tablet') {
3725 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
3727 } elsif ($deviceid eq 'keyboard') {
3729 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
3731 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3733 die "usb hotplug currently not reliable\n";
3734 # since we can't reliably hot unplug all added usb devices
3735 # and usb passthrough disables live migration
3736 # we disable usb hotplugging for now
3737 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device));
3739 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3741 qemu_iothread_add
($vmid, $deviceid, $device);
3743 qemu_driveadd
($storecfg, $vmid, $device);
3744 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3746 qemu_deviceadd
($vmid, $devicefull);
3747 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3749 eval { qemu_drivedel
($vmid, $deviceid); };
3754 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3757 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3758 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3759 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3761 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3763 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3764 qemu_iothread_add
($vmid, $deviceid, $device);
3765 $devicefull .= ",iothread=iothread-$deviceid";
3768 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3769 $devicefull .= ",num_queues=$device->{queues}";
3772 qemu_deviceadd
($vmid, $devicefull);
3773 qemu_deviceaddverify
($vmid, $deviceid);
3775 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3777 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3778 qemu_driveadd
($storecfg, $vmid, $device);
3780 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3781 eval { qemu_deviceadd
($vmid, $devicefull); };
3783 eval { qemu_drivedel
($vmid, $deviceid); };
3788 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3790 return undef if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
3792 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
3793 my $use_old_bios_files = undef;
3794 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3796 my $netdevicefull = print_netdevice_full
($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3797 qemu_deviceadd
($vmid, $netdevicefull);
3799 qemu_deviceaddverify
($vmid, $deviceid);
3800 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
3803 eval { qemu_netdevdel
($vmid, $deviceid); };
3808 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3811 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3812 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3814 qemu_deviceadd
($vmid, $devicefull);
3815 qemu_deviceaddverify
($vmid, $deviceid);
3818 die "can't hotplug device '$deviceid'\n";
3824 # fixme: this should raise exceptions on error!
3825 sub vm_deviceunplug
{
3826 my ($vmid, $conf, $deviceid) = @_;
3828 my $devices_list = vm_devices_list
($vmid);
3829 return 1 if !defined($devices_list->{$deviceid});
3831 die "can't unplug bootdisk" if $conf->{bootdisk
} && $conf->{bootdisk
} eq $deviceid;
3833 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3835 qemu_devicedel
($vmid, $deviceid);
3837 } elsif ($deviceid =~ m/^usb\d+$/) {
3839 die "usb hotplug currently not reliable\n";
3840 # when unplugging usb devices this way,
3841 # there may be remaining usb controllers/hubs
3842 # so we disable it for now
3843 qemu_devicedel
($vmid, $deviceid);
3844 qemu_devicedelverify
($vmid, $deviceid);
3846 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3848 qemu_devicedel
($vmid, $deviceid);
3849 qemu_devicedelverify
($vmid, $deviceid);
3850 qemu_drivedel
($vmid, $deviceid);
3851 qemu_iothread_del
($conf, $vmid, $deviceid);
3853 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3855 qemu_devicedel
($vmid, $deviceid);
3856 qemu_devicedelverify
($vmid, $deviceid);
3857 qemu_iothread_del
($conf, $vmid, $deviceid);
3859 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3861 qemu_devicedel
($vmid, $deviceid);
3862 qemu_drivedel
($vmid, $deviceid);
3863 qemu_deletescsihw
($conf, $vmid, $deviceid);
3865 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3867 qemu_devicedel
($vmid, $deviceid);
3868 qemu_devicedelverify
($vmid, $deviceid);
3869 qemu_netdevdel
($vmid, $deviceid);
3872 die "can't unplug device '$deviceid'\n";
3878 sub qemu_deviceadd
{
3879 my ($vmid, $devicefull) = @_;
3881 $devicefull = "driver=".$devicefull;
3882 my %options = split(/[=,]/, $devicefull);
3884 mon_cmd
($vmid, "device_add" , %options);
3887 sub qemu_devicedel
{
3888 my ($vmid, $deviceid) = @_;
3890 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
3893 sub qemu_iothread_add
{
3894 my($vmid, $deviceid, $device) = @_;
3896 if ($device->{iothread
}) {
3897 my $iothreads = vm_iothreads_list
($vmid);
3898 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3902 sub qemu_iothread_del
{
3903 my($conf, $vmid, $deviceid) = @_;
3905 my $confid = $deviceid;
3906 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3907 $confid = 'scsi' . $1;
3909 my $device = parse_drive
($confid, $conf->{$confid});
3910 if ($device->{iothread
}) {
3911 my $iothreads = vm_iothreads_list
($vmid);
3912 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3916 sub qemu_objectadd
{
3917 my($vmid, $objectid, $qomtype) = @_;
3919 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
3924 sub qemu_objectdel
{
3925 my($vmid, $objectid) = @_;
3927 mon_cmd
($vmid, "object-del", id
=> $objectid);
3933 my ($storecfg, $vmid, $device) = @_;
3935 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device);
3936 $drive =~ s/\\/\\\\/g;
3937 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
3939 # If the command succeeds qemu prints: "OK
"
3940 return 1 if $ret =~ m/OK/s;
3942 die "adding drive failed
: $ret\n";
3946 my($vmid, $deviceid) = @_;
3948 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
3951 return 1 if $ret eq "";
3953 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3954 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3956 die "deleting drive
$deviceid failed
: $ret\n";
3959 sub qemu_deviceaddverify {
3960 my ($vmid, $deviceid) = @_;
3962 for (my $i = 0; $i <= 5; $i++) {
3963 my $devices_list = vm_devices_list($vmid);
3964 return 1 if defined($devices_list->{$deviceid});
3968 die "error on hotplug device
'$deviceid'\n";
3972 sub qemu_devicedelverify {
3973 my ($vmid, $deviceid) = @_;
3975 # need to verify that the device is correctly removed as device_del
3976 # is async and empty return is not reliable
3978 for (my $i = 0; $i <= 5; $i++) {
3979 my $devices_list = vm_devices_list($vmid);
3980 return 1 if !defined($devices_list->{$deviceid});
3984 die "error on hot-unplugging device
'$deviceid'\n";
3987 sub qemu_findorcreatescsihw {
3988 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3990 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3992 my $scsihwid="$controller_prefix$controller";
3993 my $devices_list = vm_devices_list($vmid);
3995 if(!defined($devices_list->{$scsihwid})) {
3996 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4002 sub qemu_deletescsihw {
4003 my ($conf, $vmid, $opt) = @_;
4005 my $device = parse_drive($opt, $conf->{$opt});
4007 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4008 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4012 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4014 my $devices_list = vm_devices_list($vmid);
4015 foreach my $opt (keys %{$devices_list}) {
4016 if (is_valid_drivename($opt)) {
4017 my $drive = parse_drive($opt, $conf->{$opt});
4018 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4024 my $scsihwid="scsihw
$controller";
4026 vm_deviceunplug($vmid, $conf, $scsihwid);
4031 sub qemu_add_pci_bridge {
4032 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4038 print_pci_addr($device, $bridges, $arch, $machine_type);
4040 while (my ($k, $v) = each %$bridges) {
4043 return 1 if !defined($bridgeid) || $bridgeid < 1;
4045 my $bridge = "pci
.$bridgeid";
4046 my $devices_list = vm_devices_list($vmid);
4048 if (!defined($devices_list->{$bridge})) {
4049 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4055 sub qemu_set_link_status {
4056 my ($vmid, $device, $up) = @_;
4058 mon_cmd($vmid, "set_link
", name => $device,
4059 up => $up ? JSON::true : JSON::false);
4062 sub qemu_netdevadd {
4063 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4065 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4066 my %options = split(/[=,]/, $netdev);
4068 if (defined(my $vhost = $options{vhost})) {
4069 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4072 if (defined(my $queues = $options{queues})) {
4073 $options{queues} = $queues + 0;
4076 mon_cmd($vmid, "netdev_add
", %options);
4080 sub qemu_netdevdel {
4081 my ($vmid, $deviceid) = @_;
4083 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4086 sub qemu_usb_hotplug {
4087 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4091 # remove the old one first
4092 vm_deviceunplug($vmid, $conf, $deviceid);
4094 # check if xhci controller is necessary and available
4095 if ($device->{usb3}) {
4097 my $devicelist = vm_devices_list($vmid);
4099 if (!$devicelist->{xhci}) {
4100 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4101 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4104 my $d = parse_usb_device($device->{host});
4105 $d->{usb3} = $device->{usb3};
4108 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4111 sub qemu_cpu_hotplug {
4112 my ($vmid, $conf, $vcpus) = @_;
4114 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4117 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4118 $sockets = $conf->{sockets} if $conf->{sockets};
4119 my $cores = $conf->{cores} || 1;
4120 my $maxcpus = $sockets * $cores;
4122 $vcpus = $maxcpus if !$vcpus;
4124 die "you can
't add more vcpus than maxcpus\n"
4125 if $vcpus > $maxcpus;
4127 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4129 if ($vcpus < $currentvcpus) {
4131 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4133 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4134 qemu_devicedel($vmid, "cpu$i");
4136 my $currentrunningvcpus = undef;
4138 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4139 last if scalar(@{$currentrunningvcpus}) == $i-1;
4140 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4144 #update conf after each succesfull cpu unplug
4145 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4146 PVE::QemuConfig->write_config($vmid, $conf);
4149 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4155 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4156 die "vcpus in running vm does not match its configuration\n"
4157 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4159 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4161 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4162 my $cpustr = print_cpu_device($conf, $i);
4163 qemu_deviceadd($vmid, $cpustr);
4166 my $currentrunningvcpus = undef;
4168 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4169 last if scalar(@{$currentrunningvcpus}) == $i;
4170 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4174 #update conf after each succesfull cpu hotplug
4175 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4176 PVE::QemuConfig->write_config($vmid, $conf);
4180 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4181 mon_cmd($vmid, "cpu-add", id => int($i));
4186 sub qemu_block_set_io_throttle {
4187 my ($vmid, $deviceid,
4188 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4189 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4190 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4191 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4193 return if !check_running($vmid) ;
4195 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4197 bps_rd => int($bps_rd),
4198 bps_wr => int($bps_wr),
4200 iops_rd => int($iops_rd),
4201 iops_wr => int($iops_wr),
4202 bps_max => int($bps_max),
4203 bps_rd_max => int($bps_rd_max),
4204 bps_wr_max => int($bps_wr_max),
4205 iops_max => int($iops_max),
4206 iops_rd_max => int($iops_rd_max),
4207 iops_wr_max => int($iops_wr_max),
4208 bps_max_length => int($bps_max_length),
4209 bps_rd_max_length => int($bps_rd_max_length),
4210 bps_wr_max_length => int($bps_wr_max_length),
4211 iops_max_length => int($iops_max_length),
4212 iops_rd_max_length => int($iops_rd_max_length),
4213 iops_wr_max_length => int($iops_wr_max_length),
4218 # old code, only used to shutdown old VM after update
4220 my ($fh, $timeout) = @_;
4222 my $sel = new IO::Select;
4229 while (scalar (@ready = $sel->can_read($timeout))) {
4231 if ($count = $fh->sysread($buf, 8192)) {
4232 if ($buf =~ /^(.*)\(qemu\) $/s) {
4239 if (!defined($count)) {
4246 die "monitor read timeout\n" if !scalar(@ready);
4251 sub qemu_block_resize {
4252 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4254 my $running = check_running($vmid);
4256 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4258 return if !$running;
4260 my $padding = (1024 - $size % 1024) % 1024;
4261 $size = $size + $padding;
4263 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4267 sub qemu_volume_snapshot {
4268 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4270 my $running = check_running($vmid);
4272 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4273 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4275 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4279 sub qemu_volume_snapshot_delete {
4280 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4282 my $running = check_running($vmid);
4287 my $conf = PVE::QemuConfig->load_config($vmid);
4288 PVE::QemuConfig->foreach_volume($conf, sub {
4289 my ($ds, $drive) = @_;
4290 $running = 1 if $drive->{file} eq $volid;
4294 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4295 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4297 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4301 sub set_migration_caps {
4307 "auto-converge" => 1,
4309 "x-rdma-pin-all" => 0,
4314 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4316 for my $supported_capability (@$supported_capabilities) {
4318 capability => $supported_capability->{capability},
4319 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4323 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4327 my ($conf, $func, @param) = @_;
4331 my $test_volid = sub {
4332 my ($key, $drive, $snapname) = @_;
4334 my $volid = $drive->{file};
4337 $volhash->{$volid}->{cdrom} //= 1;
4338 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4340 my $replicate = $drive->{replicate} // 1;
4341 $volhash->{$volid}->{replicate} //= 0;
4342 $volhash->{$volid}->{replicate} = 1 if $replicate;
4344 $volhash->{$volid}->{shared} //= 0;
4345 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4347 $volhash->{$volid}->{referenced_in_config} //= 0;
4348 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4350 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4351 if defined($snapname);
4353 my $size = $drive->{size};
4354 $volhash->{$volid}->{size} //= $size if $size;
4356 $volhash->{$volid}->{is_vmstate} //= 0;
4357 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4359 $volhash->{$volid}->{is_unused} //= 0;
4360 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4363 my $include_opts = {
4364 extra_keys => ['vmstate
'],
4365 include_unused => 1,
4368 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4369 foreach my $snapname (keys %{$conf->{snapshots}}) {
4370 my $snap = $conf->{snapshots}->{$snapname};
4371 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4374 foreach my $volid (keys %$volhash) {
4375 &$func($volid, $volhash->{$volid}, @param);
4379 my $fast_plug_option = {
4387 'vmstatestorage
' => 1,
4392 # hotplug changes in [PENDING]
4393 # $selection hash can be used to only apply specified options, for
4394 # example: { cores => 1 } (only apply changed 'cores
')
4395 # $errors ref is used to return error messages
4396 sub vmconfig_hotplug_pending {
4397 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4399 my $defaults = load_defaults();
4400 my $arch = get_vm_arch($conf);
4401 my $machine_type = get_vm_machine($conf, undef, $arch);
4403 # commit values which do not have any impact on running VM first
4404 # Note: those option cannot raise errors, we we do not care about
4405 # $selection and always apply them.
4407 my $add_error = sub {
4408 my ($opt, $msg) = @_;
4409 $errors->{$opt} = "hotplug problem - $msg";
4413 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4414 if ($fast_plug_option->{$opt}) {
4415 $conf->{$opt} = $conf->{pending}->{$opt};
4416 delete $conf->{pending}->{$opt};
4422 PVE::QemuConfig->write_config($vmid, $conf);
4425 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4427 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4428 foreach my $opt (sort keys %$pending_delete_hash) {
4429 next if $selection && !$selection->{$opt};
4430 my $force = $pending_delete_hash->{$opt}->{force};
4432 if ($opt eq 'hotplug
') {
4433 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4434 } elsif ($opt eq 'tablet
') {
4435 die "skip\n" if !$hotplug_features->{usb};
4436 if ($defaults->{tablet}) {
4437 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4438 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4439 if $arch eq 'aarch64
';
4441 vm_deviceunplug($vmid, $conf, 'tablet
');
4442 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4444 } elsif ($opt =~ m/^usb\d+/) {
4446 # since we cannot reliably hot unplug usb devices
4447 # we are disabling it
4448 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4449 vm_deviceunplug($vmid, $conf, $opt);
4450 } elsif ($opt eq 'vcpus
') {
4451 die "skip\n" if !$hotplug_features->{cpu};
4452 qemu_cpu_hotplug($vmid, $conf, undef);
4453 } elsif ($opt eq 'balloon
') {
4454 # enable balloon device is not hotpluggable
4455 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4456 # here we reset the ballooning value to memory
4457 my $balloon = $conf->{memory} || $defaults->{memory};
4458 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4459 } elsif ($fast_plug_option->{$opt}) {
4461 } elsif ($opt =~ m/^net(\d+)$/) {
4462 die "skip\n" if !$hotplug_features->{network};
4463 vm_deviceunplug($vmid, $conf, $opt);
4464 } elsif (is_valid_drivename($opt)) {
4465 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4466 vm_deviceunplug($vmid, $conf, $opt);
4467 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4468 } elsif ($opt =~ m/^memory$/) {
4469 die "skip\n" if !$hotplug_features->{memory};
4470 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4471 } elsif ($opt eq 'cpuunits
') {
4472 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4473 } elsif ($opt eq 'cpulimit
') {
4474 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4480 &$add_error($opt, $err) if $err ne "skip\n";
4482 delete $conf->{$opt};
4483 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4487 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4488 $apply_pending_cloudinit = sub {
4489 return if $apply_pending_cloudinit_done; # once is enough
4490 $apply_pending_cloudinit_done = 1; # once is enough
4492 my ($key, $value) = @_;
4494 my @cloudinit_opts = keys %$confdesc_cloudinit;
4495 foreach my $opt (keys %{$conf->{pending}}) {
4496 next if !grep { $_ eq $opt } @cloudinit_opts;
4497 $conf->{$opt} = delete $conf->{pending}->{$opt};
4500 my $new_conf = { %$conf };
4501 $new_conf->{$key} = $value;
4502 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4505 foreach my $opt (keys %{$conf->{pending}}) {
4506 next if $selection && !$selection->{$opt};
4507 my $value = $conf->{pending}->{$opt};
4509 if ($opt eq 'hotplug
') {
4510 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4511 } elsif ($opt eq 'tablet
') {
4512 die "skip\n" if !$hotplug_features->{usb};
4514 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4515 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4516 if $arch eq 'aarch64
';
4517 } elsif ($value == 0) {
4518 vm_deviceunplug($vmid, $conf, 'tablet
');
4519 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4521 } elsif ($opt =~ m/^usb\d+$/) {
4523 # since we cannot reliably hot unplug usb devices
4524 # we are disabling it
4525 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4526 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4527 die "skip\n" if !$d;
4528 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4529 } elsif ($opt eq 'vcpus
') {
4530 die "skip\n" if !$hotplug_features->{cpu};
4531 qemu_cpu_hotplug($vmid, $conf, $value);
4532 } elsif ($opt eq 'balloon
') {
4533 # enable/disable balloning device is not hotpluggable
4534 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4535 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4536 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4538 # allow manual ballooning if shares is set to zero
4539 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4540 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4541 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4543 } elsif ($opt =~ m/^net(\d+)$/) {
4544 # some changes can be done without hotplug
4545 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4546 $vmid, $opt, $value, $arch, $machine_type);
4547 } elsif (is_valid_drivename($opt)) {
4548 die "skip\n" if $opt eq 'efidisk0
';
4549 # some changes can be done without hotplug
4550 my $drive = parse_drive($opt, $value);
4551 if (drive_is_cloudinit($drive)) {
4552 &$apply_pending_cloudinit($opt, $value);
4554 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4555 $vmid, $opt, $value, $arch, $machine_type);
4556 } elsif ($opt =~ m/^memory$/) { #dimms
4557 die "skip\n" if !$hotplug_features->{memory};
4558 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4559 } elsif ($opt eq 'cpuunits
') {
4560 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4561 } elsif ($opt eq 'cpulimit
') {
4562 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4563 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4565 die "skip\n"; # skip non-hot-pluggable options
4569 &$add_error($opt, $err) if $err ne "skip\n";
4571 $conf->{$opt} = $value;
4572 delete $conf->{pending}->{$opt};
4576 PVE::QemuConfig->write_config($vmid, $conf);
4579 sub try_deallocate_drive {
4580 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4582 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4583 my $volid = $drive->{file};
4584 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4585 my $sid = PVE::Storage::parse_volume_id($volid);
4586 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4588 # check if the disk is really unused
4589 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4590 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4591 PVE::Storage::vdisk_free($storecfg, $volid);
4594 # If vm is not owner of this disk remove from config
4602 sub vmconfig_delete_or_detach_drive {
4603 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4605 my $drive = parse_drive($opt, $conf->{$opt});
4607 my $rpcenv = PVE::RPCEnvironment::get();
4608 my $authuser = $rpcenv->get_user();
4611 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4612 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4614 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4620 sub vmconfig_apply_pending {
4621 my ($vmid, $conf, $storecfg, $errors) = @_;
4623 my $add_apply_error = sub {
4624 my ($opt, $msg) = @_;
4625 my $err_msg = "unable to apply pending change $opt : $msg";
4626 $errors->{$opt} = $err_msg;
4632 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4633 foreach my $opt (sort keys %$pending_delete_hash) {
4634 my $force = $pending_delete_hash->{$opt}->{force};
4636 if ($opt =~ m/^unused/) {
4637 die "internal error";
4638 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4639 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4643 $add_apply_error->($opt, $err);
4645 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4646 delete $conf->{$opt};
4650 PVE::QemuConfig->cleanup_pending($conf);
4652 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4653 next if $opt eq 'delete'; # just to be sure
4655 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4656 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4660 $add_apply_error->($opt, $err);
4662 $conf->{$opt} = delete $conf->{pending}->{$opt};
4666 # write all changes at once to avoid unnecessary i/o
4667 PVE::QemuConfig->write_config($vmid, $conf);
4670 sub vmconfig_update_net {
4671 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4673 my $newnet = parse_net($value);
4675 if ($conf->{$opt}) {
4676 my $oldnet = parse_net($conf->{$opt});
4678 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4679 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4680 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4681 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4683 # for non online change, we try to hot-unplug
4684 die "skip\n" if !$hotplug;
4685 vm_deviceunplug($vmid, $conf, $opt);
4688 die "internal error" if $opt !~ m/net(\d+)/;
4689 my $iface = "tap${vmid}i$1";
4691 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4692 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4693 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4694 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4695 PVE::Network::tap_unplug($iface);
4698 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4700 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4702 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4703 # Rate can be applied on its own but any change above needs to
4704 # include the rate in tap_plug since OVS resets everything.
4705 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4708 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4709 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4717 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4723 sub vmconfig_update_disk {
4724 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4726 my $drive = parse_drive($opt, $value);
4728 if ($conf->{$opt}) {
4730 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
4732 my $media = $drive->{media} || 'disk
';
4733 my $oldmedia = $old_drive->{media} || 'disk
';
4734 die "unable to change media type\n" if $media ne $oldmedia;
4736 if (!drive_is_cdrom($old_drive)) {
4738 if ($drive->{file} ne $old_drive->{file}) {
4740 die "skip\n" if !$hotplug;
4742 # unplug and register as unused
4743 vm_deviceunplug($vmid, $conf, $opt);
4744 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4747 # update existing disk
4749 # skip non hotpluggable value
4750 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4751 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4752 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4753 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4754 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4759 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4760 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4761 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4762 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4763 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4764 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4765 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4766 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4767 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4768 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4769 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4770 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4771 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4772 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4773 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4774 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4775 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4776 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4778 qemu_block_set_io_throttle($vmid,"drive-$opt",
4779 ($drive->{mbps} || 0)*1024*1024,
4780 ($drive->{mbps_rd} || 0)*1024*1024,
4781 ($drive->{mbps_wr} || 0)*1024*1024,
4782 $drive->{iops} || 0,
4783 $drive->{iops_rd} || 0,
4784 $drive->{iops_wr} || 0,
4785 ($drive->{mbps_max} || 0)*1024*1024,
4786 ($drive->{mbps_rd_max} || 0)*1024*1024,
4787 ($drive->{mbps_wr_max} || 0)*1024*1024,
4788 $drive->{iops_max} || 0,
4789 $drive->{iops_rd_max} || 0,
4790 $drive->{iops_wr_max} || 0,
4791 $drive->{bps_max_length} || 1,
4792 $drive->{bps_rd_max_length} || 1,
4793 $drive->{bps_wr_max_length} || 1,
4794 $drive->{iops_max_length} || 1,
4795 $drive->{iops_rd_max_length} || 1,
4796 $drive->{iops_wr_max_length} || 1);
4805 if ($drive->{file} eq 'none
') {
4806 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4807 if (drive_is_cloudinit($old_drive)) {
4808 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4811 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4813 # force eject if locked
4814 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4817 mon_cmd($vmid, "blockdev-change-medium",
4818 id => "$opt", filename => "$path");
4827 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4829 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4830 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4833 # called in locked context by incoming migration
4834 sub vm_migrate_get_nbd_disks {
4835 my ($storecfg, $conf, $replicated_volumes) = @_;
4837 my $local_volumes = {};
4838 PVE::QemuConfig->foreach_volume($conf, sub {
4839 my ($ds, $drive) = @_;
4841 return if drive_is_cdrom($drive);
4843 my $volid = $drive->{file};
4847 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4849 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4850 return if $scfg->{shared};
4852 # replicated disks re-use existing state via bitmap
4853 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4854 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4856 return $local_volumes;
4859 # called in locked context by incoming migration
4860 sub vm_migrate_alloc_nbd_disks {
4861 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4866 foreach my $opt (sort keys %$source_volumes) {
4867 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4869 if ($use_existing) {
4870 $nbd->{$opt}->{drivestr} = print_drive($drive);
4871 $nbd->{$opt}->{volid} = $volid;
4872 $nbd->{$opt}->{replicated} = 1;
4876 # If a remote storage is specified and the format of the original
4877 # volume is not available there, fall back to the default format.
4878 # Otherwise use the same format as the original.
4879 if (!$storagemap->{identity}) {
4880 $storeid = map_storage($storagemap, $storeid);
4881 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4882 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4883 my $fileFormat = qemu_img_format($scfg, $volname);
4884 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4886 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4887 $format = qemu_img_format($scfg, $volname);
4890 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4891 my $newdrive = $drive;
4892 $newdrive->{format} = $format;
4893 $newdrive->{file} = $newvolid;
4894 my $drivestr = print_drive($newdrive);
4895 $nbd->{$opt}->{drivestr} = $drivestr;
4896 $nbd->{$opt}->{volid} = $newvolid;
4902 # see vm_start_nolock for parameters, additionally:
4904 # storagemap = parsed storage map for allocating NBD disks
4906 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4908 return PVE::QemuConfig->lock_config($vmid, sub {
4909 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4911 die "you can't start a vm
if it
's a template\n" if PVE::QemuConfig->is_template($conf);
4913 $params->{resume} = PVE::QemuConfig->has_lock($conf, 'suspended
');
4915 PVE::QemuConfig->check_lock($conf)
4916 if !($params->{skiplock} || $params->{resume});
4918 die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4920 if (my $storagemap = $migrate_opts->{storagemap}) {
4921 my $replicated = $migrate_opts->{replicated_volumes};
4922 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4923 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4925 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4926 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4930 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4936 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
4937 # skiplock => 0/1, skip checking for config lock
4938 # forcemachine => to force Qemu machine (rollback/migration)
4939 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
4940 # timeout => in seconds
4941 # paused => start VM in paused state (backup)
4942 # resume => resume from hibernation
4944 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4945 # migratedfrom => source node
4946 # spice_ticket => used for spice migration, passed via tunnel/stdin
4947 # network => CIDR of migration network
4948 # type => secure/insecure - tunnel over encrypted connection or plain-text
4949 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
4950 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4951 sub vm_start_nolock {
4952 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4954 my $statefile = $params->{statefile};
4955 my $resume = $params->{resume};
4957 my $migratedfrom = $migrate_opts->{migratedfrom};
4958 my $migration_type = $migrate_opts->{type};
4962 # clean up leftover reboot request files
4963 eval { clear_reboot_request($vmid); };
4966 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4967 vmconfig_apply_pending($vmid, $conf, $storecfg);
4968 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4971 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4973 my $defaults = load_defaults();
4975 # set environment variable useful inside network script
4976 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4978 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
4980 my $forcemachine = $params->{forcemachine};
4981 my $forcecpu = $params->{forcecpu};
4983 # enforce machine and CPU type on suspended vm to ensure HW compatibility
4984 $forcemachine = $conf->{runningmachine};
4985 $forcecpu = $conf->{runningcpu};
4986 print "Resuming suspended VM\n";
4989 my ($cmd, $vollist, $spice_port) =
4990 config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4993 my $get_migration_ip = sub {
4994 my ($nodename) = @_;
4996 return $migration_ip if defined($migration_ip);
4998 my $cidr = $migrate_opts->{network};
5000 if (!defined($cidr)) {
5001 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5002 $cidr = $dc_conf->{migration}->{network};
5005 if (defined($cidr)) {
5006 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5008 die "could not get IP: no address configured on local " .
5009 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5011 die "could not get IP: multiple addresses configured on local " .
5012 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5014 $migration_ip = @$ips[0];
5017 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5018 if !defined($migration_ip);
5020 return $migration_ip;
5025 if ($statefile eq 'tcp
') {
5026 my $localip = "localhost";
5027 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5028 my $nodename = nodename();
5030 if (!defined($migration_type)) {
5031 if (defined($datacenterconf->{migration}->{type})) {
5032 $migration_type = $datacenterconf->{migration}->{type};
5034 $migration_type = 'secure
';
5038 if ($migration_type eq 'insecure
') {
5039 $localip = $get_migration_ip->($nodename);
5040 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5043 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5044 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5045 $migrate_uri = "tcp:${localip}:${migrate_port}";
5046 push @$cmd, '-incoming
', $migrate_uri;
5049 } elsif ($statefile eq 'unix
') {
5050 # should be default for secure migrations as a ssh TCP forward
5051 # tunnel is not deterministic reliable ready and fails regurarly
5052 # to set up in time, so use UNIX socket forwards
5053 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5054 unlink $socket_addr;
5056 $migrate_uri = "unix:$socket_addr";
5058 push @$cmd, '-incoming
', $migrate_uri;
5061 } elsif (-e $statefile) {
5062 push @$cmd, '-loadstate
', $statefile;
5064 my $statepath = PVE::Storage::path($storecfg, $statefile);
5065 push @$vollist, $statefile;
5066 push @$cmd, '-loadstate
', $statepath;
5068 } elsif ($params->{paused}) {
5073 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
5074 my $d = parse_hostpci($conf->{"hostpci$i"});
5076 my $pcidevices = $d->{pciid};
5077 foreach my $pcidevice (@$pcidevices) {
5078 my $pciid = $pcidevice->{id};
5080 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5081 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5082 die "no pci device info for device '$pciid'\n" if !$info;
5085 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5086 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5088 die "can't unbind
/bind pci group to vfio
'$pciid'\n"
5089 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5090 die "can
't reset pci device '$pciid'\n"
5091 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5096 PVE::Storage::activate_volumes($storecfg, $vollist);
5099 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5100 outfunc => sub {}, errfunc => sub {});
5102 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5103 # timeout should be more than enough here...
5104 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5106 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5107 : $defaults->{cpuunits};
5109 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5111 timeout => $statefile ? undef : $start_timeout,
5116 # when migrating, prefix QEMU output so other side can pick up any
5117 # errors that might occur and show the user
5118 if ($migratedfrom) {
5119 $run_params{quiet} = 1;
5120 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5124 Slice => 'qemu
.slice
',
5126 CPUShares => $cpuunits
5129 if (my $cpulimit = $conf->{cpulimit}) {
5130 $properties{CPUQuota} = int($cpulimit * 100);
5132 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5134 my $run_qemu = sub {
5135 PVE::Tools::run_fork sub {
5136 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5138 my $exitcode = run_command($cmd, %run_params);
5139 die "QEMU exited with code $exitcode\n" if $exitcode;
5143 if ($conf->{hugepages}) {
5146 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5147 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5149 PVE::QemuServer::Memory::hugepages_mount();
5150 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5152 eval { $run_qemu->() };
5154 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5158 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5160 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5163 eval { $run_qemu->() };
5167 # deactivate volumes if start fails
5168 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5169 die "start failed: $err";
5172 print "migration listens on $migrate_uri\n" if $migrate_uri;
5173 $res->{migrate_uri} = $migrate_uri;
5175 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5176 eval { mon_cmd($vmid, "cont"); };
5180 #start nbd server for storage migration
5181 if (my $nbd = $migrate_opts->{nbd}) {
5182 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5184 my $migrate_storage_uri;
5185 # nbd_protocol_version > 0 for unix socket support
5186 if ($nbd_protocol_version > 0 && $migration_type eq 'secure
') {
5187 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5188 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix
', data => { path => $socket_path } } );
5189 $migrate_storage_uri = "nbd:unix:$socket_path";
5191 my $nodename = nodename();
5192 my $localip = $get_migration_ip->($nodename);
5193 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5194 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5196 mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet
', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5197 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5198 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5201 $res->{migrate_storage_uri} = $migrate_storage_uri;
5203 foreach my $opt (sort keys %$nbd) {
5204 my $drivestr = $nbd->{$opt}->{drivestr};
5205 my $volid = $nbd->{$opt}->{volid};
5206 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5207 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5208 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5209 print "re-using replicated volume: $opt - $volid\n"
5210 if $nbd->{$opt}->{replicated};
5212 $res->{drives}->{$opt} = $nbd->{$opt};
5213 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5217 if ($migratedfrom) {
5219 set_migration_caps($vmid);
5224 print "spice listens on port $spice_port\n";
5225 $res->{spice_port} = $spice_port;
5226 if ($migrate_opts->{spice_ticket}) {
5227 mon_cmd($vmid, "set_password", protocol => 'spice
', password => $migrate_opts->{spice_ticket});
5228 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5233 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5234 if !$statefile && $conf->{balloon};
5236 foreach my $opt (keys %$conf) {
5237 next if $opt !~ m/^net\d+$/;
5238 my $nicconf = parse_net($conf->{$opt});
5239 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5243 mon_cmd($vmid, 'qom-set
',
5244 path => "machine/peripheral/balloon0",
5245 property => "guest-stats-polling-interval",
5246 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5249 print "Resumed VM, removing state\n";
5250 if (my $vmstate = $conf->{vmstate}) {
5251 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5252 PVE::Storage::vdisk_free($storecfg, $vmstate);
5254 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5255 PVE
::QemuConfig-
>write_config($vmid, $conf);
5258 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5263 sub vm_commandline
{
5264 my ($storecfg, $vmid, $snapname) = @_;
5266 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5271 my $snapshot = $conf->{snapshots
}->{$snapname};
5272 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5274 # check for machine or CPU overrides in snapshot
5275 $forcemachine = $snapshot->{runningmachine
};
5276 $forcecpu = $snapshot->{runningcpu
};
5278 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5283 my $defaults = load_defaults
();
5285 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults,
5286 $forcemachine, $forcecpu);
5288 return PVE
::Tools
::cmd2string
($cmd);
5292 my ($vmid, $skiplock) = @_;
5294 PVE
::QemuConfig-
>lock_config($vmid, sub {
5296 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5298 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5300 mon_cmd
($vmid, "system_reset");
5304 sub get_vm_volumes
{
5308 foreach_volid
($conf, sub {
5309 my ($volid, $attr) = @_;
5311 return if $volid =~ m
|^/|;
5313 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5316 push @$vollist, $volid;
5322 sub vm_stop_cleanup
{
5323 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5328 my $vollist = get_vm_volumes
($conf);
5329 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5332 foreach my $ext (qw(mon qmp pid vnc qga)) {
5333 unlink "/var/run/qemu-server/${vmid}.$ext";
5336 if ($conf->{ivshmem
}) {
5337 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5338 # just delete it for now, VMs which have this already open do not
5339 # are affected, but new VMs will get a separated one. If this
5340 # becomes an issue we either add some sort of ref-counting or just
5341 # add a "don't delete on stop" flag to the ivshmem format.
5342 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5345 foreach my $key (keys %$conf) {
5346 next if $key !~ m/^hostpci(\d+)$/;
5347 my $hostpciindex = $1;
5348 my $d = parse_hostpci
($conf->{$key});
5349 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5351 foreach my $pci (@{$d->{pciid
}}) {
5352 my $pciid = $pci->{id
};
5353 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5357 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5359 warn $@ if $@; # avoid errors - just warn
5362 # call only in locked context
5364 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5366 my $pid = check_running
($vmid, $nocheck);
5371 $conf = PVE
::QemuConfig-
>load_config($vmid);
5372 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5373 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5374 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5375 $timeout = $opts->{down
} if $opts->{down
};
5377 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5382 if (defined($conf) && parse_guest_agent
($conf)->{enabled
}) {
5383 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5385 mon_cmd
($vmid, "system_powerdown");
5388 mon_cmd
($vmid, "quit");
5394 $timeout = 60 if !defined($timeout);
5397 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5402 if ($count >= $timeout) {
5404 warn "VM still running - terminating now with SIGTERM\n";
5407 die "VM quit/powerdown failed - got timeout\n";
5410 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5414 if (!check_running
($vmid, $nocheck)) {
5415 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5419 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5422 die "VM quit/powerdown failed\n";
5430 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5435 if ($count >= $timeout) {
5436 warn "VM still running - terminating now with SIGKILL\n";
5441 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5444 # Note: use $nocheck to skip tests if VM configuration file exists.
5445 # We need that when migration VMs to other nodes (files already moved)
5446 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5448 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5450 $force = 1 if !defined($force) && !$shutdown;
5453 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5454 kill 15, $pid if $pid;
5455 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5456 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5460 PVE
::QemuConfig-
>lock_config($vmid, sub {
5461 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5466 my ($vmid, $timeout) = @_;
5468 PVE
::QemuConfig-
>lock_config($vmid, sub {
5471 # only reboot if running, as qmeventd starts it again on a stop event
5472 return if !check_running
($vmid);
5474 create_reboot_request
($vmid);
5476 my $storecfg = PVE
::Storage
::config
();
5477 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5481 # avoid that the next normal shutdown will be confused for a reboot
5482 clear_reboot_request
($vmid);
5488 # note: if using the statestorage parameter, the caller has to check privileges
5490 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5497 PVE
::QemuConfig-
>lock_config($vmid, sub {
5499 $conf = PVE
::QemuConfig-
>load_config($vmid);
5501 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5502 PVE
::QemuConfig-
>check_lock($conf)
5503 if !($skiplock || $is_backing_up);
5505 die "cannot suspend to disk during backup\n"
5506 if $is_backing_up && $includestate;
5508 if ($includestate) {
5509 $conf->{lock} = 'suspending';
5510 my $date = strftime
("%Y-%m-%d", localtime(time()));
5511 $storecfg = PVE
::Storage
::config
();
5512 if (!$statestorage) {
5513 $statestorage = find_vmstate_storage
($conf, $storecfg);
5514 # check permissions for the storage
5515 my $rpcenv = PVE
::RPCEnvironment
::get
();
5516 if ($rpcenv->{type
} ne 'cli') {
5517 my $authuser = $rpcenv->get_user();
5518 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5523 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5524 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5525 PVE
::QemuConfig-
>write_config($vmid, $conf);
5527 mon_cmd
($vmid, "stop");
5531 if ($includestate) {
5533 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5536 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5538 my $state = mon_cmd
($vmid, "query-savevm");
5539 if (!$state->{status
}) {
5540 die "savevm not active\n";
5541 } elsif ($state->{status
} eq 'active') {
5544 } elsif ($state->{status
} eq 'completed') {
5545 print "State saved, quitting\n";
5547 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5548 die "query-savevm failed with error '$state->{error}'\n"
5550 die "query-savevm returned status '$state->{status}'\n";
5556 PVE
::QemuConfig-
>lock_config($vmid, sub {
5557 $conf = PVE
::QemuConfig-
>load_config($vmid);
5559 # cleanup, but leave suspending lock, to indicate something went wrong
5561 mon_cmd
($vmid, "savevm-end");
5562 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5563 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5564 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5565 PVE
::QemuConfig-
>write_config($vmid, $conf);
5571 die "lock changed unexpectedly\n"
5572 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5574 mon_cmd
($vmid, "quit");
5575 $conf->{lock} = 'suspended';
5576 PVE
::QemuConfig-
>write_config($vmid, $conf);
5582 my ($vmid, $skiplock, $nocheck) = @_;
5584 PVE
::QemuConfig-
>lock_config($vmid, sub {
5585 my $res = mon_cmd
($vmid, 'query-status');
5586 my $resume_cmd = 'cont';
5588 if ($res->{status
} && $res->{status
} eq 'suspended') {
5589 $resume_cmd = 'system_wakeup';
5594 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5596 PVE
::QemuConfig-
>check_lock($conf)
5597 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5600 mon_cmd
($vmid, $resume_cmd);
5605 my ($vmid, $skiplock, $key) = @_;
5607 PVE
::QemuConfig-
>lock_config($vmid, sub {
5609 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5611 # there is no qmp command, so we use the human monitor command
5612 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5613 die $res if $res ne '';
5617 # vzdump restore implementaion
5619 sub tar_archive_read_firstfile
{
5620 my $archive = shift;
5622 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5624 # try to detect archive type first
5625 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5626 die "unable to open file '$archive'\n";
5627 my $firstfile = <$fh>;
5631 die "ERROR: archive contaions no data\n" if !$firstfile;
5637 sub tar_restore_cleanup
{
5638 my ($storecfg, $statfile) = @_;
5640 print STDERR
"starting cleanup\n";
5642 if (my $fd = IO
::File-
>new($statfile, "r")) {
5643 while (defined(my $line = <$fd>)) {
5644 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5647 if ($volid =~ m
|^/|) {
5648 unlink $volid || die 'unlink failed\n';
5650 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5652 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5654 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5656 print STDERR
"unable to parse line in statfile - $line";
5663 sub restore_file_archive
{
5664 my ($archive, $vmid, $user, $opts) = @_;
5666 return restore_vma_archive
($archive, $vmid, $user, $opts)
5669 my $info = PVE
::Storage
::archive_info
($archive);
5670 my $format = $opts->{format
} // $info->{format
};
5671 my $comp = $info->{compression
};
5673 # try to detect archive format
5674 if ($format eq 'tar') {
5675 return restore_tar_archive
($archive, $vmid, $user, $opts);
5677 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5681 # hepler to remove disks that will not be used after restore
5682 my $restore_cleanup_oldconf = sub {
5683 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5685 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
5686 my ($ds, $drive) = @_;
5688 return if drive_is_cdrom
($drive, 1);
5690 my $volid = $drive->{file
};
5691 return if !$volid || $volid =~ m
|^/|;
5693 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
5694 return if !$path || !$owner || ($owner != $vmid);
5696 # Note: only delete disk we want to restore
5697 # other volumes will become unused
5698 if ($virtdev_hash->{$ds}) {
5699 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
5706 # delete vmstate files, after the restore we have no snapshots anymore
5707 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5708 my $snap = $oldconf->{snapshots
}->{$snapname};
5709 if ($snap->{vmstate
}) {
5710 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
5718 # Helper to parse vzdump backup device hints
5720 # $rpcenv: Environment, used to ckeck storage permissions
5721 # $user: User ID, to check storage permissions
5722 # $storecfg: Storage configuration
5723 # $fh: the file handle for reading the configuration
5724 # $devinfo: should contain device sizes for all backu-up'ed devices
5725 # $options: backup options (pool, default storage)
5727 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5728 my $parse_backup_hints = sub {
5729 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5731 my $virtdev_hash = {};
5733 while (defined(my $line = <$fh>)) {
5734 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5735 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5736 die "archive does not contain data for drive '$virtdev'\n"
5737 if !$devinfo->{$devname};
5739 if (defined($options->{storage
})) {
5740 $storeid = $options->{storage
} || 'local';
5741 } elsif (!$storeid) {
5744 $format = 'raw' if !$format;
5745 $devinfo->{$devname}->{devname
} = $devname;
5746 $devinfo->{$devname}->{virtdev
} = $virtdev;
5747 $devinfo->{$devname}->{format
} = $format;
5748 $devinfo->{$devname}->{storeid
} = $storeid;
5750 # check permission on storage
5751 my $pool = $options->{pool
}; # todo: do we need that?
5752 if ($user ne 'root@pam') {
5753 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5756 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5757 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5759 my $drive = parse_drive
($virtdev, $2);
5760 if (drive_is_cloudinit
($drive)) {
5761 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
5762 $storeid = $options->{storage
} if defined ($options->{storage
});
5763 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5764 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
5766 $virtdev_hash->{$virtdev} = {
5768 storeid
=> $storeid,
5769 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
5776 return $virtdev_hash;
5779 # Helper to allocate and activate all volumes required for a restore
5781 # $storecfg: Storage configuration
5782 # $virtdev_hash: as returned by parse_backup_hints()
5784 # Returns: { $virtdev => $volid }
5785 my $restore_allocate_devices = sub {
5786 my ($storecfg, $virtdev_hash, $vmid) = @_;
5789 foreach my $virtdev (sort keys %$virtdev_hash) {
5790 my $d = $virtdev_hash->{$virtdev};
5791 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5792 my $storeid = $d->{storeid
};
5793 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5795 # test if requested format is supported
5796 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5797 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
5798 $d->{format
} = $defFormat if !$supported;
5801 if ($d->{is_cloudinit
}) {
5802 $name = "vm-$vmid-cloudinit";
5803 $name .= ".$d->{format}" if $d->{format
} ne 'raw';
5806 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
5808 print STDERR
"new volume ID is '$volid'\n";
5809 $d->{volid
} = $volid;
5811 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
5813 $map->{$virtdev} = $volid;
5819 my $restore_update_config_line = sub {
5820 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5822 return if $line =~ m/^\#qmdump\#/;
5823 return if $line =~ m/^\#vzdump\#/;
5824 return if $line =~ m/^lock:/;
5825 return if $line =~ m/^unused\d+:/;
5826 return if $line =~ m/^parent:/;
5828 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5829 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5830 # try to convert old 1.X settings
5831 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5832 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5833 my ($model, $macaddr) = split(/\=/, $devconfig);
5834 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5837 bridge
=> "vmbr$ind",
5838 macaddr
=> $macaddr,
5840 my $netstr = print_net
($net);
5842 print $outfd "net$cookie->{netcount}: $netstr\n";
5843 $cookie->{netcount
}++;
5845 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5846 my ($id, $netstr) = ($1, $2);
5847 my $net = parse_net
($netstr);
5848 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5849 $netstr = print_net
($net);
5850 print $outfd "$id: $netstr\n";
5851 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5854 my $di = parse_drive
($virtdev, $value);
5855 if (defined($di->{backup
}) && !$di->{backup
}) {
5856 print $outfd "#$line";
5857 } elsif ($map->{$virtdev}) {
5858 delete $di->{format
}; # format can change on restore
5859 $di->{file
} = $map->{$virtdev};
5860 $value = print_drive
($di);
5861 print $outfd "$virtdev: $value\n";
5865 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5867 if ($vmgenid ne '0') {
5868 # always generate a new vmgenid if there was a valid one setup
5869 $vmgenid = generate_uuid
();
5871 print $outfd "vmgenid: $vmgenid\n";
5872 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5873 my ($uuid, $uuid_str);
5874 UUID
::generate
($uuid);
5875 UUID
::unparse
($uuid, $uuid_str);
5876 my $smbios1 = parse_smbios1
($2);
5877 $smbios1->{uuid
} = $uuid_str;
5878 print $outfd $1.print_smbios1
($smbios1)."\n";
5884 my $restore_deactivate_volumes = sub {
5885 my ($storecfg, $devinfo) = @_;
5888 foreach my $devname (keys %$devinfo) {
5889 my $volid = $devinfo->{$devname}->{volid
};
5890 push @$vollist, $volid if $volid;
5893 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5896 my $restore_destroy_volumes = sub {
5897 my ($storecfg, $devinfo) = @_;
5899 foreach my $devname (keys %$devinfo) {
5900 my $volid = $devinfo->{$devname}->{volid
};
5903 if ($volid =~ m
|^/|) {
5904 unlink $volid || die 'unlink failed\n';
5906 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5908 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5910 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5915 my ($cfg, $vmid) = @_;
5917 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5919 my $volid_hash = {};
5920 foreach my $storeid (keys %$info) {
5921 foreach my $item (@{$info->{$storeid}}) {
5922 next if !($item->{volid
} && $item->{size
});
5923 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5924 $volid_hash->{$item->{volid
}} = $item;
5931 sub update_disk_config
{
5932 my ($vmid, $conf, $volid_hash) = @_;
5935 my $prefix = "VM $vmid:";
5937 # used and unused disks
5938 my $referenced = {};
5940 # Note: it is allowed to define multiple storages with same path (alias), so
5941 # we need to check both 'volid' and real 'path' (two different volid can point
5942 # to the same path).
5944 my $referencedpath = {};
5947 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5948 my ($opt, $drive) = @_;
5950 my $volid = $drive->{file
};
5953 # mark volid as "in-use" for next step
5954 $referenced->{$volid} = 1;
5955 if ($volid_hash->{$volid} &&
5956 (my $path = $volid_hash->{$volid}->{path
})) {
5957 $referencedpath->{$path} = 1;
5960 return if drive_is_cdrom
($drive);
5961 return if !$volid_hash->{$volid};
5963 my ($updated, $old_size, $new_size) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volid_hash);
5964 if (defined($updated)) {
5966 $conf->{$opt} = print_drive
($updated);
5967 print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5971 # remove 'unusedX' entry if volume is used
5972 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
5973 my ($opt, $drive) = @_;
5975 my $volid = $drive->{file
};
5978 my $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
5979 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5980 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5982 delete $conf->{$opt};
5985 $referenced->{$volid} = 1;
5986 $referencedpath->{$path} = 1 if $path;
5989 foreach my $volid (sort keys %$volid_hash) {
5990 next if $volid =~ m/vm-$vmid-state-/;
5991 next if $referenced->{$volid};
5992 my $path = $volid_hash->{$volid}->{path
};
5993 next if !$path; # just to be sure
5994 next if $referencedpath->{$path};
5996 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
5997 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5998 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6005 my ($vmid, $nolock, $dryrun) = @_;
6007 my $cfg = PVE
::Storage
::config
();
6009 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6010 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6011 foreach my $stor (keys %{$cfg->{ids
}}) {
6012 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
6015 print "rescan volumes...\n";
6016 my $volid_hash = scan_volids
($cfg, $vmid);
6018 my $updatefn = sub {
6021 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6023 PVE
::QemuConfig-
>check_lock($conf);
6026 foreach my $volid (keys %$volid_hash) {
6027 my $info = $volid_hash->{$volid};
6028 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6031 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6033 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6036 if (defined($vmid)) {
6040 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6043 my $vmlist = config_list
();
6044 foreach my $vmid (keys %$vmlist) {
6048 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6054 sub restore_proxmox_backup_archive
{
6055 my ($archive, $vmid, $user, $options) = @_;
6057 my $storecfg = PVE
::Storage
::config
();
6059 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6060 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6062 my $server = $scfg->{server
};
6063 my $datastore = $scfg->{datastore
};
6064 my $username = $scfg->{username
} // 'root@pam';
6065 my $fingerprint = $scfg->{fingerprint
};
6067 my $repo = "$username\@$server:$datastore";
6068 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6069 local $ENV{PBS_PASSWORD
} = $password;
6070 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6072 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6073 PVE
::Storage
::parse_volname
($storecfg, $archive);
6075 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6077 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6079 my $tmpdir = "/var/tmp/vzdumptmp$$";
6083 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6084 my $tmpfn = "$conffile.$$.tmp";
6085 # disable interrupts (always do cleanups)
6089 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6091 # Note: $oldconf is undef if VM does not exists
6092 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6093 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6095 my $rpcenv = PVE
::RPCEnvironment
::get
();
6104 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6106 my $cfgfn = "$tmpdir/qemu-server.conf";
6107 my $firewall_config_fn = "$tmpdir/fw.conf";
6108 my $index_fn = "$tmpdir/index.json";
6110 my $cmd = "restore";
6112 my $param = [$pbs_backup_name, "index.json", $index_fn];
6113 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6114 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6115 $index = decode_json
($index);
6117 # print Dumper($index);
6118 foreach my $info (@{$index->{files
}}) {
6119 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6121 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6122 $devinfo->{$devname}->{size
} = $1;
6124 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6129 my $is_qemu_server_backup = scalar(grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}});
6130 if (!$is_qemu_server_backup) {
6131 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6133 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6135 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6136 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6138 if ($has_firewall_config) {
6139 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6140 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6142 my $pve_firewall_dir = '/etc/pve/firewall';
6143 mkdir $pve_firewall_dir; # make sure the dir exists
6144 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6147 my $fh = IO
::File-
>new($cfgfn, "r") ||
6148 "unable to read qemu-server.conf - $!\n";
6150 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6152 # fixme: rate limit?
6154 # create empty/temp config
6155 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6157 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6160 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6162 foreach my $virtdev (sort keys %$virtdev_hash) {
6163 my $d = $virtdev_hash->{$virtdev};
6164 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6166 my $volid = $d->{volid
};
6168 my $path = PVE
::Storage
::path
($storecfg, $volid);
6170 my $pbs_restore_cmd = [
6171 '/usr/bin/pbs-restore',
6172 '--repository', $repo,
6174 "$d->{devname}.img.fidx",
6179 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6180 push @$pbs_restore_cmd, '--skip-zero';
6183 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6184 print "restore proxmox backup image: $dbg_cmdstring\n";
6185 run_command
($pbs_restore_cmd);
6188 $fh->seek(0, 0) || die "seek failed - $!\n";
6190 my $outfd = new IO
::File
($tmpfn, "w") ||
6191 die "unable to write config for VM $vmid\n";
6193 my $cookie = { netcount
=> 0 };
6194 while (defined(my $line = <$fh>)) {
6195 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique
});
6203 $restore_deactivate_volumes->($storecfg, $devinfo);
6209 $restore_destroy_volumes->($storecfg, $devinfo);
6213 rename($tmpfn, $conffile) ||
6214 die "unable to commit configuration file '$conffile'\n";
6216 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6218 eval { rescan
($vmid, 1); };
6222 sub restore_vma_archive
{
6223 my ($archive, $vmid, $user, $opts, $comp) = @_;
6225 my $readfrom = $archive;
6227 my $cfg = PVE
::Storage
::config
();
6229 my $bwlimit = $opts->{bwlimit
};
6231 my $dbg_cmdstring = '';
6232 my $add_pipe = sub {
6234 push @$commands, $cmd;
6235 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6236 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6241 if ($archive eq '-') {
6244 # If we use a backup from a PVE defined storage we also consider that
6245 # storage's rate limit:
6246 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6247 if (defined($volid)) {
6248 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6249 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6251 print STDERR
"applying read rate limit: $readlimit\n";
6252 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6253 $add_pipe->($cstream);
6259 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6260 my $cmd = $info->{decompressor
};
6261 push @$cmd, $readfrom;
6265 my $tmpdir = "/var/tmp/vzdumptmp$$";
6268 # disable interrupts (always do cleanups)
6272 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6274 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6275 POSIX
::mkfifo
($mapfifo, 0600);
6278 my $openfifo = sub {
6279 open($fifofh, '>', $mapfifo) || die $!;
6282 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6289 my $rpcenv = PVE
::RPCEnvironment
::get
();
6291 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6292 my $tmpfn = "$conffile.$$.tmp";
6294 # Note: $oldconf is undef if VM does not exist
6295 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6296 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6300 my $print_devmap = sub {
6301 my $cfgfn = "$tmpdir/qemu-server.conf";
6303 # we can read the config - that is already extracted
6304 my $fh = IO
::File-
>new($cfgfn, "r") ||
6305 "unable to read qemu-server.conf - $!\n";
6307 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6309 my $pve_firewall_dir = '/etc/pve/firewall';
6310 mkdir $pve_firewall_dir; # make sure the dir exists
6311 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6314 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6316 foreach my $key (keys %storage_limits) {
6317 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6319 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6320 $storage_limits{$key} = $limit * 1024;
6323 foreach my $devname (keys %$devinfo) {
6324 die "found no device mapping information for device '$devname'\n"
6325 if !$devinfo->{$devname}->{virtdev
};
6328 # create empty/temp config
6330 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6331 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6335 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6337 # print restore information to $fifofh
6338 foreach my $virtdev (sort keys %$virtdev_hash) {
6339 my $d = $virtdev_hash->{$virtdev};
6340 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6342 my $storeid = $d->{storeid
};
6343 my $volid = $d->{volid
};
6346 if (my $limit = $storage_limits{$storeid}) {
6347 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6350 my $write_zeros = 1;
6351 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6355 my $path = PVE
::Storage
::path
($cfg, $volid);
6357 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6359 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6362 $fh->seek(0, 0) || die "seek failed - $!\n";
6364 my $outfd = new IO
::File
($tmpfn, "w") ||
6365 die "unable to write config for VM $vmid\n";
6367 my $cookie = { netcount
=> 0 };
6368 while (defined(my $line = <$fh>)) {
6369 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6382 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6383 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6385 $oldtimeout = alarm($timeout);
6392 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6393 my ($dev_id, $size, $devname) = ($1, $2, $3);
6394 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6395 } elsif ($line =~ m/^CTIME: /) {
6396 # we correctly received the vma config, so we can disable
6397 # the timeout now for disk allocation (set to 10 minutes, so
6398 # that we always timeout if something goes wrong)
6401 print $fifofh "done\n";
6402 my $tmp = $oldtimeout || 0;
6403 $oldtimeout = undef;
6409 print "restore vma archive: $dbg_cmdstring\n";
6410 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6414 alarm($oldtimeout) if $oldtimeout;
6416 $restore_deactivate_volumes->($cfg, $devinfo);
6423 $restore_destroy_volumes->($cfg, $devinfo);
6427 rename($tmpfn, $conffile) ||
6428 die "unable to commit configuration file '$conffile'\n";
6430 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6432 eval { rescan
($vmid, 1); };
6436 sub restore_tar_archive
{
6437 my ($archive, $vmid, $user, $opts) = @_;
6439 if ($archive ne '-') {
6440 my $firstfile = tar_archive_read_firstfile
($archive);
6441 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6442 if $firstfile ne 'qemu-server.conf';
6445 my $storecfg = PVE
::Storage
::config
();
6447 # avoid zombie disks when restoring over an existing VM -> cleanup first
6448 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6449 # skiplock=1 because qmrestore has set the 'create' lock itself already
6450 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6451 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6453 my $tocmd = "/usr/lib/qemu-server/qmextract";
6455 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6456 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6457 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6458 $tocmd .= ' --info' if $opts->{info
};
6460 # tar option "xf" does not autodetect compression when read from STDIN,
6461 # so we pipe to zcat
6462 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6463 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6465 my $tmpdir = "/var/tmp/vzdumptmp$$";
6468 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6469 local $ENV{VZDUMP_VMID
} = $vmid;
6470 local $ENV{VZDUMP_USER
} = $user;
6472 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6473 my $tmpfn = "$conffile.$$.tmp";
6475 # disable interrupts (always do cleanups)
6479 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6487 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6489 if ($archive eq '-') {
6490 print "extracting archive from STDIN\n";
6491 run_command
($cmd, input
=> "<&STDIN");
6493 print "extracting archive '$archive'\n";
6497 return if $opts->{info
};
6501 my $statfile = "$tmpdir/qmrestore.stat";
6502 if (my $fd = IO
::File-
>new($statfile, "r")) {
6503 while (defined (my $line = <$fd>)) {
6504 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6505 $map->{$1} = $2 if $1;
6507 print STDERR
"unable to parse line in statfile - $line\n";
6513 my $confsrc = "$tmpdir/qemu-server.conf";
6515 my $srcfd = new IO
::File
($confsrc, "r") ||
6516 die "unable to open file '$confsrc'\n";
6518 my $outfd = new IO
::File
($tmpfn, "w") ||
6519 die "unable to write config for VM $vmid\n";
6521 my $cookie = { netcount
=> 0 };
6522 while (defined (my $line = <$srcfd>)) {
6523 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6531 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6537 rename $tmpfn, $conffile ||
6538 die "unable to commit configuration file '$conffile'\n";
6540 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6542 eval { rescan
($vmid, 1); };
6546 sub foreach_storage_used_by_vm
{
6547 my ($conf, $func) = @_;
6551 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6552 my ($ds, $drive) = @_;
6553 return if drive_is_cdrom
($drive);
6555 my $volid = $drive->{file
};
6557 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6558 $sidhash->{$sid} = $sid if $sid;
6561 foreach my $sid (sort keys %$sidhash) {
6566 my $qemu_snap_storage = {
6569 sub do_snapshots_with_qemu
{
6570 my ($storecfg, $volid) = @_;
6572 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6573 my $scfg = $storecfg->{ids
}->{$storage_name};
6575 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6579 if ($volid =~ m/\.(qcow2|qed)$/){
6586 sub qga_check_running
{
6587 my ($vmid, $nowarn) = @_;
6589 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6591 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6597 sub template_create
{
6598 my ($vmid, $conf, $disk) = @_;
6600 my $storecfg = PVE
::Storage
::config
();
6602 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6603 my ($ds, $drive) = @_;
6605 return if drive_is_cdrom
($drive);
6606 return if $disk && $ds ne $disk;
6608 my $volid = $drive->{file
};
6609 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6611 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6612 $drive->{file
} = $voliddst;
6613 $conf->{$ds} = print_drive
($drive);
6614 PVE
::QemuConfig-
>write_config($vmid, $conf);
6618 sub convert_iscsi_path
{
6621 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6626 my $initiator_name = get_initiator_name
();
6628 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6629 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6632 die "cannot convert iscsi path '$path', unkown format\n";
6635 sub qemu_img_convert
{
6636 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6638 my $storecfg = PVE
::Storage
::config
();
6639 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6640 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6642 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6646 my $src_is_iscsi = 0;
6650 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6651 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6652 $src_format = qemu_img_format
($src_scfg, $src_volname);
6653 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6654 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6655 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6656 } elsif (-f
$src_volid) {
6657 $src_path = $src_volid;
6658 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6663 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6665 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6666 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6667 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6668 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6671 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6672 push @$cmd, '-l', "snapshot.name=$snapname"
6673 if $snapname && $src_format && $src_format eq "qcow2";
6674 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6675 push @$cmd, '-T', $cachemode if defined($cachemode);
6677 if ($src_is_iscsi) {
6678 push @$cmd, '--image-opts';
6679 $src_path = convert_iscsi_path
($src_path);
6680 } elsif ($src_format) {
6681 push @$cmd, '-f', $src_format;
6684 if ($dst_is_iscsi) {
6685 push @$cmd, '--target-image-opts';
6686 $dst_path = convert_iscsi_path
($dst_path);
6688 push @$cmd, '-O', $dst_format;
6691 push @$cmd, $src_path;
6693 if (!$dst_is_iscsi && $is_zero_initialized) {
6694 push @$cmd, "zeroinit:$dst_path";
6696 push @$cmd, $dst_path;
6701 if($line =~ m/\((\S+)\/100\
%\)/){
6703 my $transferred = int($size * $percent / 100);
6704 my $remaining = $size - $transferred;
6706 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6711 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6713 die "copy failed: $err" if $err;
6716 sub qemu_img_format
{
6717 my ($scfg, $volname) = @_;
6719 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6726 sub qemu_drive_mirror
{
6727 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6729 $jobs = {} if !$jobs;
6733 $jobs->{"drive-$drive"} = {};
6735 if ($dst_volid =~ /^nbd:/) {
6736 $qemu_target = $dst_volid;
6739 my $storecfg = PVE
::Storage
::config
();
6740 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6742 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6744 $format = qemu_img_format
($dst_scfg, $dst_volname);
6746 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6748 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6751 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6752 $opts->{format
} = $format if $format;
6754 if (defined($src_bitmap)) {
6755 $opts->{sync
} = 'incremental';
6756 $opts->{bitmap
} = $src_bitmap;
6757 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6760 if (defined($bwlimit)) {
6761 $opts->{speed
} = $bwlimit * 1024;
6762 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6764 print "drive mirror is starting for drive-$drive\n";
6767 # if a job already runs for this device we get an error, catch it for cleanup
6768 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
6770 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6772 die "mirroring error: $err\n";
6775 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
6778 # $completion can be either
6779 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6780 # 'cancel': wait until all jobs are ready, block-job-cancel them
6781 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6782 sub qemu_drive_mirror_monitor
{
6783 my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6785 $completion //= 'complete';
6788 my $err_complete = 0;
6791 die "storage migration timed out\n" if $err_complete > 300;
6793 my $stats = mon_cmd
($vmid, "query-block-jobs");
6795 my $running_mirror_jobs = {};
6796 foreach my $stat (@$stats) {
6797 next if $stat->{type
} ne 'mirror';
6798 $running_mirror_jobs->{$stat->{device
}} = $stat;
6801 my $readycounter = 0;
6803 foreach my $job (keys %$jobs) {
6805 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6806 print "$job : finished\n";
6807 delete $jobs->{$job};
6811 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6813 my $busy = $running_mirror_jobs->{$job}->{busy
};
6814 my $ready = $running_mirror_jobs->{$job}->{ready
};
6815 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6816 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6817 my $remaining = $total - $transferred;
6818 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6820 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6823 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6826 last if scalar(keys %$jobs) == 0;
6828 if ($readycounter == scalar(keys %$jobs)) {
6829 print "all mirroring jobs are ready \n";
6830 last if $completion eq 'skip'; #do the complete later
6832 if ($vmiddst && $vmiddst != $vmid) {
6833 my $agent_running = $qga && qga_check_running
($vmid);
6834 if ($agent_running) {
6835 print "freeze filesystem\n";
6836 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6838 print "suspend vm\n";
6839 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6842 # if we clone a disk for a new target vm, we don't switch the disk
6843 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6845 if ($agent_running) {
6846 print "unfreeze filesystem\n";
6847 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6849 print "resume vm\n";
6850 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6856 foreach my $job (keys %$jobs) {
6857 # try to switch the disk if source and destination are on the same guest
6858 print "$job: Completing block job...\n";
6861 if ($completion eq 'complete') {
6862 $op = 'block-job-complete';
6863 } elsif ($completion eq 'cancel') {
6864 $op = 'block-job-cancel';
6866 die "invalid completion value: $completion\n";
6868 eval { mon_cmd
($vmid, $op, device
=> $job) };
6869 if ($@ =~ m/cannot be completed/) {
6870 print "$job: Block job cannot be completed, try again.\n";
6873 print "$job: Completed successfully.\n";
6874 $jobs->{$job}->{complete
} = 1;
6885 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6886 die "mirroring error: $err";
6891 sub qemu_blockjobs_cancel
{
6892 my ($vmid, $jobs) = @_;
6894 foreach my $job (keys %$jobs) {
6895 print "$job: Cancelling block job\n";
6896 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6897 $jobs->{$job}->{cancel
} = 1;
6901 my $stats = mon_cmd
($vmid, "query-block-jobs");
6903 my $running_jobs = {};
6904 foreach my $stat (@$stats) {
6905 $running_jobs->{$stat->{device
}} = $stat;
6908 foreach my $job (keys %$jobs) {
6910 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6911 print "$job: Done.\n";
6912 delete $jobs->{$job};
6916 last if scalar(keys %$jobs) == 0;
6923 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6924 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6929 print "create linked clone of drive $drivename ($drive->{file})\n";
6930 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6931 push @$newvollist, $newvolid;
6934 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6935 $storeid = $storage if $storage;
6937 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6938 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
6940 print "create full clone of drive $drivename ($drive->{file})\n";
6942 if (drive_is_cloudinit
($drive)) {
6943 $name = "vm-$newvmid-cloudinit";
6944 $name .= ".$dst_format" if $dst_format ne 'raw';
6946 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
6947 } elsif ($drivename eq 'efidisk0') {
6948 $size = get_efivars_size
($conf);
6950 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6951 push @$newvollist, $newvolid;
6953 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
6955 if (drive_is_cloudinit
($drive)) {
6959 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6960 if (!$running || $snapname) {
6961 # TODO: handle bwlimits
6962 if ($drivename eq 'efidisk0') {
6963 # the relevant data on the efidisk may be smaller than the source
6964 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6965 # that is given by the OVMF_VARS.fd
6966 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
6967 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
6968 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6970 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
6974 my $kvmver = get_running_qemu_version
($vmid);
6975 if (!min_version
($kvmver, 2, 7)) {
6976 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6977 if $drive->{iothread
};
6980 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6985 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
6988 $disk->{format
} = undef;
6989 $disk->{file
} = $newvolid;
6990 $disk->{size
} = $size;
6995 sub get_running_qemu_version
{
6997 my $res = mon_cmd
($vmid, "query-version");
6998 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7001 sub qemu_use_old_bios_files
{
7002 my ($machine_type) = @_;
7004 return if !$machine_type;
7006 my $use_old_bios_files = undef;
7008 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7010 $use_old_bios_files = 1;
7012 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, kvm_user_version
());
7013 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7014 # load new efi bios files on migration. So this hack is required to allow
7015 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7016 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7017 $use_old_bios_files = !min_version
($version, 2, 4);
7020 return ($use_old_bios_files, $machine_type);
7023 sub get_efivars_size
{
7025 my $arch = get_vm_arch
($conf);
7026 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7027 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
7028 return -s
$ovmf_vars;
7031 sub update_efidisk_size
{
7034 return if !defined($conf->{efidisk0
});
7036 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
7037 $disk->{size
} = get_efivars_size
($conf);
7038 $conf->{efidisk0
} = print_drive
($disk);
7043 sub create_efidisk
($$$$$) {
7044 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7046 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7047 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7049 my $vars_size_b = -s
$ovmf_vars;
7050 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7051 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7052 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7054 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7055 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7057 return ($volid, $size/1024);
7060 sub vm_iothreads_list
{
7063 my $res = mon_cmd
($vmid, 'query-iothreads');
7066 foreach my $iothread (@$res) {
7067 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7074 my ($conf, $drive) = @_;
7078 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7080 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7086 my $controller = int($drive->{index} / $maxdev);
7087 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single') ?
"virtioscsi" : "scsihw";
7089 return ($maxdev, $controller, $controller_prefix);
7092 sub windows_version
{
7095 return 0 if !$ostype;
7099 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7101 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7103 } elsif ($ostype =~ m/^win(\d+)$/) {
7110 sub resolve_dst_disk_format
{
7111 my ($storecfg, $storeid, $src_volname, $format) = @_;
7112 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7115 # if no target format is specified, use the source disk format as hint
7117 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7118 $format = qemu_img_format
($scfg, $src_volname);
7124 # test if requested format is supported - else use default
7125 my $supported = grep { $_ eq $format } @$validFormats;
7126 $format = $defFormat if !$supported;
7130 # NOTE: if this logic changes, please update docs & possibly gui logic
7131 sub find_vmstate_storage
{
7132 my ($conf, $storecfg) = @_;
7134 # first, return storage from conf if set
7135 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7137 my ($target, $shared, $local);
7139 foreach_storage_used_by_vm
($conf, sub {
7141 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7142 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7143 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7146 # second, use shared storage where VM has at least one disk
7147 # third, use local storage where VM has at least one disk
7148 # fall back to local storage
7149 $target = $shared // $local // 'local';
7155 my ($uuid, $uuid_str);
7156 UUID
::generate
($uuid);
7157 UUID
::unparse
($uuid, $uuid_str);
7161 sub generate_smbios1_uuid
{
7162 return "uuid=".generate_uuid
();
7168 mon_cmd
($vmid, 'nbd-server-stop');
7171 sub create_reboot_request
{
7173 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7174 or die "failed to create reboot trigger file: $!\n";
7178 sub clear_reboot_request
{
7180 my $path = "/run/qemu-server/$vmid.reboot";
7183 $res = unlink($path);
7184 die "could not remove reboot request for $vmid: $!"
7185 if !$res && $! != POSIX
::ENOENT
;
7190 # bash completion helper
7192 sub complete_backup_archives
{
7193 my ($cmdname, $pname, $cvalue) = @_;
7195 my $cfg = PVE
::Storage
::config
();
7199 if ($cvalue =~ m/^([^:]+):/) {
7203 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7206 foreach my $id (keys %$data) {
7207 foreach my $item (@{$data->{$id}}) {
7208 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7209 push @$res, $item->{volid
} if defined($item->{volid
});
7216 my $complete_vmid_full = sub {
7219 my $idlist = vmstatus
();
7223 foreach my $id (keys %$idlist) {
7224 my $d = $idlist->{$id};
7225 if (defined($running)) {
7226 next if $d->{template
};
7227 next if $running && $d->{status
} ne 'running';
7228 next if !$running && $d->{status
} eq 'running';
7237 return &$complete_vmid_full();
7240 sub complete_vmid_stopped
{
7241 return &$complete_vmid_full(0);
7244 sub complete_vmid_running
{
7245 return &$complete_vmid_full(1);
7248 sub complete_storage
{
7250 my $cfg = PVE
::Storage
::config
();
7251 my $ids = $cfg->{ids
};
7254 foreach my $sid (keys %$ids) {
7255 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7256 next if !$ids->{$sid}->{content
}->{images
};
7263 sub complete_migration_storage
{
7264 my ($cmd, $param, $current_value, $all_args) = @_;
7266 my $targetnode = @$all_args[1];
7268 my $cfg = PVE
::Storage
::config
();
7269 my $ids = $cfg->{ids
};
7272 foreach my $sid (keys %$ids) {
7273 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7274 next if !$ids->{$sid}->{content
}->{images
};