1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
30 use PVE
::DataCenterConfig
;
31 use PVE
::Exception
qw(raise raise_param_exc);
32 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
34 use PVE
::JSONSchema
qw(get_standard_option);
36 use PVE
::RPCEnvironment
;
40 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
44 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
45 use PVE
::QemuServer
::Cloudinit
;
46 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
47 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
48 use PVE
::QemuServer
::Machine
;
49 use PVE
::QemuServer
::Memory
;
50 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
51 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
52 use PVE
::QemuServer
::USB
qw(parse_usb_device);
56 require PVE
::Network
::SDN
::Zones
;
60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
63 "$EDK2_FW_BASE/OVMF_CODE.fd",
64 "$EDK2_FW_BASE/OVMF_VARS.fd"
67 "$EDK2_FW_BASE/AAVMF_CODE.fd",
68 "$EDK2_FW_BASE/AAVMF_VARS.fd"
72 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
74 # Note about locking: we use flock on the config file protect
75 # against concurent actions.
76 # Aditionaly, we have a 'lock' setting in the config file. This
77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
78 # allowed when such lock is set. But you can ignore this kind of
79 # lock with the --skiplock flag.
81 cfs_register_file
('/qemu-server/',
85 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
86 description
=> "Some command save/restore state from this location.",
92 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
93 description
=> "Specifies the Qemu machine type.",
95 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
102 my ($map, $source) = @_;
104 return $source if !defined($map);
106 return $map->{entries
}->{$source}
107 if $map->{entries
} && defined($map->{entries
}->{$source});
109 return $map->{default} if $map->{default};
111 # identity (fallback)
115 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
116 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
118 format
=> 'storagepair-list',
122 #no warnings 'redefine';
125 my ($controller, $vmid, $option, $value) = @_;
127 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
128 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
134 $nodename_cache //= PVE
::INotify
::nodename
();
135 return $nodename_cache;
142 enum
=> [qw(i6300esb ib700)],
143 description
=> "Watchdog type to emulate.",
144 default => 'i6300esb',
149 enum
=> [qw(reset shutdown poweroff pause debug none)],
150 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
154 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
158 description
=> "Enable/disable Qemu GuestAgent.",
163 fstrim_cloned_disks
=> {
164 description
=> "Run fstrim after cloning/moving a disk.",
170 description
=> "Select the agent type",
174 enum
=> [qw(virtio isa)],
180 description
=> "Select the VGA type.",
185 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
188 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
200 description
=> "The size of the file in MB.",
204 pattern
=> '[a-zA-Z0-9\-]+',
206 format_description
=> 'string',
207 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
214 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
215 description
=> "Configure an audio device."
222 description
=> "Driver backend for the audio device."
226 my $spice_enhancements_fmt = {
231 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
235 enum
=> ['off', 'all', 'filter'],
238 description
=> "Enable video streaming. Uses compression for detected video streams."
245 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
247 description
=> "The file on the host to gather entropy from. In most"
248 . " cases /dev/urandom should be preferred over /dev/random"
249 . " to avoid entropy-starvation issues on the host. Using"
250 . " urandom does *not* decrease security in any meaningful"
251 . " way, as it's still seeded from real entropy, and the"
252 . " bytes provided will most likely be mixed with real"
253 . " entropy on the guest as well. /dev/hwrng can be used"
254 . " to pass through a hardware RNG from the host.",
258 description
=> "Maximum bytes of entropy injected into the guest every"
259 . " 'period' milliseconds. Prefer a lower value when using"
260 . " /dev/random as source. Use 0 to disable limiting"
261 . " (potentially dangerous!).",
264 # default is 1 KiB/s, provides enough entropy to the guest to avoid
265 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
266 # of overwhelming the host, provided we're reading from /dev/urandom
271 description
=> "Every 'period' milliseconds the entropy-injection quota"
272 . " is reset, allowing the guest to retrieve another"
273 . " 'max_bytes' of entropy.",
283 description
=> "Specifies whether a VM will be started during system bootup.",
289 description
=> "Automatic restart after crash (currently ignored).",
294 type
=> 'string', format
=> 'pve-hotplug-features',
295 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
296 default => 'network,disk,usb',
301 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
307 description
=> "Lock/unlock the VM.",
308 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
313 description
=> "Limit of CPU usage.",
314 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
322 description
=> "CPU weight for a VM.",
323 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
331 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
338 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
344 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
352 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
353 "It should not be necessary to set it.",
354 enum
=> PVE
::Tools
::kvmkeymaplist
(),
359 type
=> 'string', format
=> 'dns-name',
360 description
=> "Set a name for the VM. Only used on the configuration web interface.",
365 description
=> "SCSI controller model",
366 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
372 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
377 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
378 description
=> "Specify guest operating system.",
379 verbose_description
=> <<EODESC,
380 Specify guest operating system. This is used to enable special
381 optimization/features for specific operating systems:
384 other;; unspecified OS
385 wxp;; Microsoft Windows XP
386 w2k;; Microsoft Windows 2000
387 w2k3;; Microsoft Windows 2003
388 w2k8;; Microsoft Windows 2008
389 wvista;; Microsoft Windows Vista
390 win7;; Microsoft Windows 7
391 win8;; Microsoft Windows 8/2012/2012r2
392 win10;; Microsoft Windows 10/2016
393 l24;; Linux 2.4 Kernel
394 l26;; Linux 2.6 - 5.X Kernel
395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
401 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
402 pattern
=> '[acdn]{1,4}',
407 type
=> 'string', format
=> 'pve-qm-bootdisk',
408 description
=> "Enable booting from specified disk.",
409 pattern
=> '(ide|sata|scsi|virtio)\d+',
414 description
=> "The number of CPUs. Please use option -sockets instead.",
421 description
=> "The number of CPU sockets.",
428 description
=> "The number of cores per socket.",
435 description
=> "Enable/disable NUMA.",
441 description
=> "Enable/disable hugepages memory.",
442 enum
=> [qw(any 2 1024)],
447 description
=> "Number of hotplugged vcpus.",
454 description
=> "Enable/disable ACPI.",
459 description
=> "Enable/disable Qemu GuestAgent and its properties.",
461 format
=> $agent_fmt,
466 description
=> "Enable/disable KVM hardware virtualization.",
472 description
=> "Enable/disable time drift fix.",
478 description
=> "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
483 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
487 type
=> 'string', format
=> $vga_fmt,
488 description
=> "Configure the VGA hardware.",
489 verbose_description
=> "Configure the VGA Hardware. If you want to use ".
490 "high resolution modes (>= 1280x1024x16) you may need to increase " .
491 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
492 "is 'std' for all OS types besides some Windows versions (XP and " .
493 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
494 "display server. For win* OS you can select how many independent " .
495 "displays you want, Linux guests can add displays them self.\n".
496 "You can also run without any graphic card, using a serial device as terminal.",
500 type
=> 'string', format
=> 'pve-qm-watchdog',
501 description
=> "Create a virtual hardware watchdog device.",
502 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled" .
503 " (by a guest action), the watchdog must be periodically polled " .
504 "by an agent inside the guest or else the watchdog will reset " .
505 "the guest (or execute the respective action specified)",
510 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
511 description
=> "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
512 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
515 startup
=> get_standard_option
('pve-startup-order'),
519 description
=> "Enable/disable Template.",
525 description
=> "Arbitrary arguments passed to kvm.",
526 verbose_description
=> <<EODESCR,
527 Arbitrary arguments passed to kvm, for example:
529 args: -no-reboot -no-hpet
531 NOTE: this option is for experts only.
538 description
=> "Enable/disable the USB tablet device.",
539 verbose_description
=> "Enable/disable the USB tablet device. This device is " .
540 "usually needed to allow absolute mouse positioning with VNC. " .
541 "Else the mouse runs out of sync with normal VNC clients. " .
542 "If you're running lots of console-only guests on one host, " .
543 "you may consider disabling this to save some context switches. " .
544 "This is turned off by default if you use spice (-vga=qxl).",
549 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
553 migrate_downtime
=> {
556 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
562 type
=> 'string', format
=> 'pve-qm-ide',
563 typetext
=> '<volume>',
564 description
=> "This is an alias for option -ide2",
568 description
=> "Emulated CPU type.",
570 format
=> 'pve-vm-cpu-conf',
572 parent
=> get_standard_option
('pve-snapshot-name', {
574 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
578 description
=> "Timestamp for snapshots.",
584 type
=> 'string', format
=> 'pve-volume-id',
585 description
=> "Reference to a volume which stores the VM state. This is used internally for snapshots.",
587 vmstatestorage
=> get_standard_option
('pve-storage-id', {
588 description
=> "Default storage for VM state volumes/files.",
591 runningmachine
=> get_standard_option
('pve-qemu-machine', {
592 description
=> "Specifies the QEMU machine type of the running vm. This is used internally for snapshots.",
595 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used internally for snapshots.",
598 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
599 format_description
=> 'QEMU -cpu parameter'
601 machine
=> get_standard_option
('pve-qemu-machine'),
603 description
=> "Virtual processor architecture. Defaults to the host.",
606 enum
=> [qw(x86_64 aarch64)],
609 description
=> "Specify SMBIOS type 1 fields.",
610 type
=> 'string', format
=> 'pve-qm-smbios1',
617 description
=> "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
623 enum
=> [ qw(seabios ovmf) ],
624 description
=> "Select BIOS implementation.",
625 default => 'seabios',
629 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
630 format_description
=> 'UUID',
631 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
632 verbose_description
=> "The VM generation ID (vmgenid) device exposes a".
633 " 128-bit integer value identifier to the guest OS. This allows to".
634 " notify the guest operating system when the virtual machine is".
635 " executed with a different configuration (e.g. snapshot execution".
636 " or creation from a template). The guest operating system notices".
637 " the change, and is then able to react as appropriate by marking".
638 " its copies of distributed databases as dirty, re-initializing its".
639 " random number generator, etc.\n".
640 "Note that auto-creation only works when done throug API/CLI create".
641 " or update methods, but not when manually editing the config file.",
642 default => "1 (autogenerated)",
647 format
=> 'pve-volume-id',
649 description
=> "Script that will be executed during various steps in the vms lifetime.",
653 format
=> $ivshmem_fmt,
654 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
659 format
=> $audio_fmt,
660 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
663 spice_enhancements
=> {
665 format
=> $spice_enhancements_fmt,
666 description
=> "Configure additional enhancements for SPICE.",
670 type
=> 'string', format
=> 'pve-tag-list',
671 description
=> 'Tags of the VM. This is only meta information.',
677 description
=> "Configure a VirtIO-based Random Number Generator.",
686 description
=> 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
687 format
=> 'pve-volume-id',
688 format_description
=> 'volume',
693 description
=> 'Specify a custom file containing all network data passed to the VM via cloud-init.',
694 format
=> 'pve-volume-id',
695 format_description
=> 'volume',
700 description
=> 'Specify a custom file containing all user data passed to the VM via cloud-init.',
701 format
=> 'pve-volume-id',
702 format_description
=> 'volume',
705 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
707 my $confdesc_cloudinit = {
711 description
=> 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
712 enum
=> ['configdrive2', 'nocloud'],
717 description
=> "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
722 description
=> 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
727 description
=> 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
728 format
=> 'pve-qm-cicustom',
733 description
=> "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
737 type
=> 'string', format
=> 'address-list',
738 description
=> "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
743 format
=> 'urlencoded',
744 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
748 # what about other qemu settings ?
750 #machine => 'string',
763 ##soundhw => 'string',
765 while (my ($k, $v) = each %$confdesc) {
766 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
769 my $MAX_USB_DEVICES = 5;
771 my $MAX_SERIAL_PORTS = 4;
772 my $MAX_PARALLEL_PORTS = 3;
778 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
779 description
=> "CPUs accessing this NUMA node.",
780 format_description
=> "id[-id];...",
784 description
=> "Amount of memory this NUMA node provides.",
789 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
790 description
=> "Host NUMA nodes to use.",
791 format_description
=> "id[-id];...",
796 enum
=> [qw(preferred bind interleave)],
797 description
=> "NUMA allocation policy.",
801 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
804 type
=> 'string', format
=> $numa_fmt,
805 description
=> "NUMA topology.",
807 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
809 for (my $i = 0; $i < $MAX_NUMA; $i++) {
810 $confdesc->{"numa$i"} = $numadesc;
813 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
814 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
815 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
816 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
818 my $net_fmt_bridge_descr = <<__EOD__;
819 Bridge to attach the network device to. The Proxmox VE standard bridge
822 If you do not specify a bridge, we create a kvm user (NATed) network
823 device, which provides DHCP and DNS services. The following addresses
830 The DHCP server assign addresses to the guest starting from 10.0.2.15.
834 macaddr
=> get_standard_option
('mac-addr', {
835 description
=> "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
839 description
=> "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
840 enum
=> $nic_model_list,
843 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
846 description
=> $net_fmt_bridge_descr,
847 format_description
=> 'bridge',
848 pattern
=> '[-_.\w\d]+',
853 minimum
=> 0, maximum
=> 16,
854 description
=> 'Number of packet queues to be used on the device.',
860 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
865 minimum
=> 1, maximum
=> 4094,
866 description
=> 'VLAN tag to apply to packets on this interface.',
871 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
872 description
=> 'VLAN trunks to pass through this interface.',
873 format_description
=> 'vlanid[;vlanid...]',
878 description
=> 'Whether this interface should be protected by the firewall.',
883 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
888 minimum
=> 1, maximum
=> 65520,
889 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
896 type
=> 'string', format
=> $net_fmt,
897 description
=> "Specify network devices.",
900 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
905 format
=> 'pve-ipv4-config',
906 format_description
=> 'IPv4Format/CIDR',
907 description
=> 'IPv4 address in CIDR format.',
914 format_description
=> 'GatewayIPv4',
915 description
=> 'Default gateway for IPv4 traffic.',
921 format
=> 'pve-ipv6-config',
922 format_description
=> 'IPv6Format/CIDR',
923 description
=> 'IPv6 address in CIDR format.',
930 format_description
=> 'GatewayIPv6',
931 description
=> 'Default gateway for IPv6 traffic.',
936 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
939 type
=> 'string', format
=> 'pve-qm-ipconfig',
940 description
=> <<'EODESCR',
941 cloud-init: Specify IP addresses and gateways for the corresponding interface.
943 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
945 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
946 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
948 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
951 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
953 for (my $i = 0; $i < $MAX_NETS; $i++) {
954 $confdesc->{"net$i"} = $netdesc;
955 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
958 foreach my $key (keys %$confdesc_cloudinit) {
959 $confdesc->{$key} = $confdesc_cloudinit->{$key};
962 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
963 sub verify_volume_id_or_qm_path
{
964 my ($volid, $noerr) = @_;
966 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
970 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
971 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
973 return undef if $noerr;
982 type
=> 'string', format
=> 'pve-qm-usb-device',
983 format_description
=> 'HOSTUSBDEVICE|spice',
984 description
=> <<EODESCR,
985 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
987 'bus-port(.port)*' (decimal numbers) or
988 'vendor_id:product_id' (hexadeciaml numbers) or
991 You can use the 'lsusb -t' command to list existing usb devices.
993 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
995 The value 'spice' can be used to add a usb redirection devices for spice.
1001 description
=> "Specifies whether if given host option is a USB3 device or port.",
1008 type
=> 'string', format
=> $usb_fmt,
1009 description
=> "Configure an USB device (n is 0 to 4).",
1011 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1016 pattern
=> '(/dev/.+|socket)',
1017 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1018 verbose_description
=> <<EODESCR,
1019 Create a serial device inside the VM (n is 0 to 3), and pass through a
1020 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1021 host side (use 'qm terminal' to open a terminal connection).
1023 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1025 CAUTION: Experimental! User reported problems with this option.
1032 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1033 description
=> "Map host parallel devices (n is 0 to 2).",
1034 verbose_description
=> <<EODESCR,
1035 Map host parallel devices (n is 0 to 2).
1037 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1039 CAUTION: Experimental! User reported problems with this option.
1043 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1044 $confdesc->{"parallel$i"} = $paralleldesc;
1047 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1048 $confdesc->{"serial$i"} = $serialdesc;
1051 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1052 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1055 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1056 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1059 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1060 $confdesc->{"usb$i"} = $usbdesc;
1063 my $kvm_api_version = 0;
1066 return $kvm_api_version if $kvm_api_version;
1068 open my $fh, '<', '/dev/kvm'
1071 # 0xae00 => KVM_GET_API_VERSION
1072 $kvm_api_version = ioctl($fh, 0xae00, 0);
1074 return $kvm_api_version;
1077 my $kvm_user_version = {};
1080 sub kvm_user_version
{
1083 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1084 my $st = stat($binary);
1086 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1087 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1088 $cachedmtime == $st->mtime;
1090 $kvm_user_version->{$binary} = 'unknown';
1091 $kvm_mtime->{$binary} = $st->mtime;
1095 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1096 $kvm_user_version->{$binary} = $2;
1100 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1103 return $kvm_user_version->{$binary};
1107 sub kernel_has_vhost_net
{
1108 return -c
'/dev/vhost-net';
1113 return defined($confdesc->{$key});
1117 sub get_cdrom_path
{
1119 return $cdrom_path if $cdrom_path;
1121 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1122 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1123 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1127 my ($storecfg, $vmid, $cdrom) = @_;
1129 if ($cdrom eq 'cdrom') {
1130 return get_cdrom_path
();
1131 } elsif ($cdrom eq 'none') {
1133 } elsif ($cdrom =~ m
|^/|) {
1136 return PVE
::Storage
::path
($storecfg, $cdrom);
1140 # try to convert old style file names to volume IDs
1141 sub filename_to_volume_id
{
1142 my ($vmid, $file, $media) = @_;
1144 if (!($file eq 'none' || $file eq 'cdrom' ||
1145 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1147 return undef if $file =~ m
|/|;
1149 if ($media && $media eq 'cdrom') {
1150 $file = "local:iso/$file";
1152 $file = "local:$vmid/$file";
1159 sub verify_media_type
{
1160 my ($opt, $vtype, $media) = @_;
1165 if ($media eq 'disk') {
1167 } elsif ($media eq 'cdrom') {
1170 die "internal error";
1173 return if ($vtype eq $etype);
1175 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1178 sub cleanup_drive_path
{
1179 my ($opt, $storecfg, $drive) = @_;
1181 # try to convert filesystem paths to volume IDs
1183 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1184 ($drive->{file
} !~ m
|^/dev/.+|) &&
1185 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1186 ($drive->{file
} !~ m/^\d+$/)) {
1187 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1188 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1189 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1190 verify_media_type
($opt, $vtype, $drive->{media
});
1191 $drive->{file
} = $volid;
1194 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1197 sub parse_hotplug_features
{
1202 return $res if $data eq '0';
1204 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1206 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1207 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1210 die "invalid hotplug feature '$feature'\n";
1216 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1217 sub pve_verify_hotplug_features
{
1218 my ($value, $noerr) = @_;
1220 return $value if parse_hotplug_features
($value);
1222 return undef if $noerr;
1224 die "unable to parse hotplug option\n";
1228 my($fh, $noerr) = @_;
1231 my $SG_GET_VERSION_NUM = 0x2282;
1233 my $versionbuf = "\x00" x
8;
1234 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1236 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1239 my $version = unpack("I", $versionbuf);
1240 if ($version < 30000) {
1241 die "scsi generic interface too old\n" if !$noerr;
1245 my $buf = "\x00" x
36;
1246 my $sensebuf = "\x00" x
8;
1247 my $cmd = pack("C x3 C x1", 0x12, 36);
1249 # see /usr/include/scsi/sg.h
1250 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1252 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1253 length($sensebuf), 0, length($buf), $buf,
1254 $cmd, $sensebuf, 6000);
1256 $ret = ioctl($fh, $SG_IO, $packet);
1258 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1262 my @res = unpack($sg_io_hdr_t, $packet);
1263 if ($res[17] || $res[18]) {
1264 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1269 (my $byte0, my $byte1, $res->{vendor
},
1270 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1272 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1273 $res->{type
} = $byte0 & 31;
1281 my $fh = IO
::File-
>new("+<$path") || return undef;
1282 my $res = scsi_inquiry
($fh, 1);
1288 sub print_tabletdevice_full
{
1289 my ($conf, $arch) = @_;
1291 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1293 # we use uhci for old VMs because tablet driver was buggy in older qemu
1295 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1301 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1304 sub print_keyboarddevice_full
{
1305 my ($conf, $arch, $machine) = @_;
1307 return undef if $arch ne 'aarch64';
1309 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1312 sub print_drivedevice_full
{
1313 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1318 if ($drive->{interface
} eq 'virtio') {
1319 my $pciaddr = print_pci_addr
("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1320 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1321 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread
};
1322 } elsif ($drive->{interface
} eq 'scsi') {
1324 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1325 my $unit = $drive->{index} % $maxdev;
1326 my $devicetype = 'hd';
1328 if (drive_is_cdrom
($drive)) {
1331 if ($drive->{file
} =~ m
|^/|) {
1332 $path = $drive->{file
};
1333 if (my $info = path_is_scsi
($path)) {
1334 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1335 $devicetype = 'block';
1336 } elsif ($info->{type
} == 1) { # tape
1337 $devicetype = 'generic';
1341 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1344 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1345 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, kvm_user_version
());
1346 if ($path =~ m/^iscsi\:\/\
// &&
1347 !min_version
($version, 4, 1)) {
1348 $devicetype = 'generic';
1352 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1353 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1355 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1358 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1359 $device .= ",rotation_rate=1";
1361 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1363 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1364 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1365 my $controller = int($drive->{index} / $maxdev);
1366 my $unit = $drive->{index} % $maxdev;
1367 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1369 $device = "ide-$devicetype";
1370 if ($drive->{interface
} eq 'ide') {
1371 $device .= ",bus=ide.$controller,unit=$unit";
1373 $device .= ",bus=ahci$controller.$unit";
1375 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1377 if ($devicetype eq 'hd') {
1378 if (my $model = $drive->{model
}) {
1379 $model = URI
::Escape
::uri_unescape
($model);
1380 $device .= ",model=$model";
1382 if ($drive->{ssd
}) {
1383 $device .= ",rotation_rate=1";
1386 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1387 } elsif ($drive->{interface
} eq 'usb') {
1389 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1391 die "unsupported interface type";
1394 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1396 if (my $serial = $drive->{serial
}) {
1397 $serial = URI
::Escape
::uri_unescape
($serial);
1398 $device .= ",serial=$serial";
1405 sub get_initiator_name
{
1408 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return undef;
1409 while (defined(my $line = <$fh>)) {
1410 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1419 sub print_drive_commandline_full
{
1420 my ($storecfg, $vmid, $drive) = @_;
1423 my $volid = $drive->{file
};
1426 if (drive_is_cdrom
($drive)) {
1427 $path = get_iso_path
($storecfg, $vmid, $volid);
1429 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1431 $path = PVE
::Storage
::path
($storecfg, $volid);
1432 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1433 $format = qemu_img_format
($scfg, $volname);
1441 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1442 foreach my $o (@qemu_drive_options) {
1443 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1446 # snapshot only accepts on|off
1447 if (defined($drive->{snapshot
})) {
1448 my $v = $drive->{snapshot
} ?
'on' : 'off';
1449 $opts .= ",snapshot=$v";
1452 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1453 my ($dir, $qmpname) = @$type;
1454 if (my $v = $drive->{"mbps$dir"}) {
1455 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1457 if (my $v = $drive->{"mbps${dir}_max"}) {
1458 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1460 if (my $v = $drive->{"bps${dir}_max_length"}) {
1461 $opts .= ",throttling.bps$qmpname-max-length=$v";
1463 if (my $v = $drive->{"iops${dir}"}) {
1464 $opts .= ",throttling.iops$qmpname=$v";
1466 if (my $v = $drive->{"iops${dir}_max"}) {
1467 $opts .= ",throttling.iops$qmpname-max=$v";
1469 if (my $v = $drive->{"iops${dir}_max_length"}) {
1470 $opts .= ",throttling.iops$qmpname-max-length=$v";
1474 $opts .= ",format=$format" if $format && !$drive->{format
};
1476 my $cache_direct = 0;
1478 if (my $cache = $drive->{cache
}) {
1479 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1480 } elsif (!drive_is_cdrom
($drive)) {
1481 $opts .= ",cache=none";
1485 # aio native works only with O_DIRECT
1486 if (!$drive->{aio
}) {
1488 $opts .= ",aio=native";
1490 $opts .= ",aio=threads";
1494 if (!drive_is_cdrom
($drive)) {
1496 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1497 $detectzeroes = 'off';
1498 } elsif ($drive->{discard
}) {
1499 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1501 # This used to be our default with discard not being specified:
1502 $detectzeroes = 'on';
1504 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1507 my $pathinfo = $path ?
"file=$path," : '';
1509 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1512 sub print_netdevice_full
{
1513 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1515 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
1517 my $device = $net->{model
};
1518 if ($net->{model
} eq 'virtio') {
1519 $device = 'virtio-net-pci';
1522 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1523 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1524 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1525 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1526 my $vectors = $net->{queues
} * 2 + 2;
1527 $tmpstr .= ",vectors=$vectors,mq=on";
1529 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1531 if (my $mtu = $net->{mtu
}) {
1532 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1533 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1536 } elsif ($mtu < 576) {
1537 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1538 } elsif ($mtu > $bridge_mtu) {
1539 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1541 $tmpstr .= ",host_mtu=$mtu";
1543 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1547 if ($use_old_bios_files) {
1549 if ($device eq 'virtio-net-pci') {
1550 $romfile = 'pxe-virtio.rom';
1551 } elsif ($device eq 'e1000') {
1552 $romfile = 'pxe-e1000.rom';
1553 } elsif ($device eq 'ne2k') {
1554 $romfile = 'pxe-ne2k_pci.rom';
1555 } elsif ($device eq 'pcnet') {
1556 $romfile = 'pxe-pcnet.rom';
1557 } elsif ($device eq 'rtl8139') {
1558 $romfile = 'pxe-rtl8139.rom';
1560 $tmpstr .= ",romfile=$romfile" if $romfile;
1566 sub print_netdev_full
{
1567 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1570 if ($netid =~ m/^net(\d+)$/) {
1574 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1576 my $ifname = "tap${vmid}i$i";
1578 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1579 die "interface name '$ifname' is too long (max 15 character)\n"
1580 if length($ifname) >= 16;
1582 my $vhostparam = '';
1583 if (is_native
($arch)) {
1584 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1587 my $vmname = $conf->{name
} || "vm$vmid";
1590 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1592 if ($net->{bridge
}) {
1593 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1595 $netdev = "type=user,id=$netid,hostname=$vmname";
1598 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1604 'cirrus' => 'cirrus-vga',
1606 'vmware' => 'vmware-svga',
1607 'virtio' => 'virtio-vga',
1610 sub print_vga_device
{
1611 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1613 my $type = $vga_map->{$vga->{type
}};
1614 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1615 $type = 'virtio-gpu';
1617 my $vgamem_mb = $vga->{memory
};
1619 my $max_outputs = '';
1621 $type = $id ?
'qxl' : 'qxl-vga';
1623 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1624 # set max outputs so linux can have up to 4 qxl displays with one device
1625 if (min_version
($machine_version, 4, 1)) {
1626 $max_outputs = ",max_outputs=4";
1631 die "no devicetype for $vga->{type}\n" if !$type;
1635 if ($vga->{type
} eq 'virtio') {
1636 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1637 $memory = ",max_hostmem=$bytes";
1639 # from https://www.spice-space.org/multiple-monitors.html
1640 $memory = ",vgamem_mb=$vga->{memory}";
1641 my $ram = $vgamem_mb * 4;
1642 my $vram = $vgamem_mb * 2;
1643 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1645 $memory = ",vgamem_mb=$vga->{memory}";
1647 } elsif ($qxlnum && $id) {
1648 $memory = ",ram_size=67108864,vram_size=33554432";
1652 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1653 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1656 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1657 my $vgaid = "vga" . ($id // '');
1660 if ($q35 && $vgaid eq 'vga') {
1661 # the first display uses pcie.0 bus on q35 machines
1662 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1664 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1667 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1670 sub parse_number_sets
{
1673 foreach my $part (split(/;/, $set)) {
1674 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1675 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1676 push @$res, [ $1, $2 ];
1678 die "invalid range: $part\n";
1687 my $res = PVE
::JSONSchema
::parse_property_string
($numa_fmt, $data);
1688 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1689 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1693 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1697 my $res = eval { PVE
::JSONSchema
::parse_property_string
($net_fmt, $data) };
1702 if (!defined($res->{macaddr
})) {
1703 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1704 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1709 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1710 sub parse_ipconfig
{
1713 my $res = eval { PVE
::JSONSchema
::parse_property_string
($ipconfig_fmt, $data) };
1719 if ($res->{gw
} && !$res->{ip
}) {
1720 warn 'gateway specified without specifying an IP address';
1723 if ($res->{gw6
} && !$res->{ip6
}) {
1724 warn 'IPv6 gateway specified without specifying an IPv6 address';
1727 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1728 warn 'gateway specified together with DHCP';
1731 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1733 warn "IPv6 gateway specified together with $res->{ip6} address";
1737 if (!$res->{ip
} && !$res->{ip6
}) {
1738 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1747 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1750 sub add_random_macs
{
1751 my ($settings) = @_;
1753 foreach my $opt (keys %$settings) {
1754 next if $opt !~ m/^net(\d+)$/;
1755 my $net = parse_net
($settings->{$opt});
1757 $settings->{$opt} = print_net
($net);
1761 sub vm_is_volid_owner
{
1762 my ($storecfg, $vmid, $volid) = @_;
1764 if ($volid !~ m
|^/|) {
1766 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1767 if ($owner && ($owner == $vmid)) {
1775 sub vmconfig_register_unused_drive
{
1776 my ($storecfg, $vmid, $conf, $drive) = @_;
1778 if (drive_is_cloudinit
($drive)) {
1779 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1781 } elsif (!drive_is_cdrom
($drive)) {
1782 my $volid = $drive->{file
};
1783 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1784 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1789 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1793 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1794 format_description
=> 'UUID',
1795 description
=> "Set SMBIOS1 UUID.",
1800 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1801 format_description
=> 'Base64 encoded string',
1802 description
=> "Set SMBIOS1 version.",
1807 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1808 format_description
=> 'Base64 encoded string',
1809 description
=> "Set SMBIOS1 serial number.",
1814 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1815 format_description
=> 'Base64 encoded string',
1816 description
=> "Set SMBIOS1 manufacturer.",
1821 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1822 format_description
=> 'Base64 encoded string',
1823 description
=> "Set SMBIOS1 product ID.",
1828 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1829 format_description
=> 'Base64 encoded string',
1830 description
=> "Set SMBIOS1 SKU string.",
1835 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1836 format_description
=> 'Base64 encoded string',
1837 description
=> "Set SMBIOS1 family string.",
1842 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
1850 my $res = eval { PVE
::JSONSchema
::parse_property_string
($smbios1_fmt, $data) };
1857 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
1860 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
1862 sub parse_watchdog
{
1865 return undef if !$value;
1867 my $res = eval { PVE
::JSONSchema
::parse_property_string
($watchdog_fmt, $value) };
1872 sub parse_guest_agent
{
1875 return {} if !defined($value->{agent
});
1877 my $res = eval { PVE
::JSONSchema
::parse_property_string
($agent_fmt, $value->{agent
}) };
1880 # if the agent is disabled ignore the other potentially set properties
1881 return {} if !$res->{enabled
};
1888 return {} if !$value;
1889 my $res = eval { PVE
::JSONSchema
::parse_property_string
($vga_fmt, $value) };
1897 return undef if !$value;
1899 my $res = eval { PVE
::JSONSchema
::parse_property_string
($rng_fmt, $value) };
1904 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
1905 sub verify_usb_device
{
1906 my ($value, $noerr) = @_;
1908 return $value if parse_usb_device
($value);
1910 return undef if $noerr;
1912 die "unable to parse usb device\n";
1915 # add JSON properties for create and set function
1916 sub json_config_properties
{
1919 foreach my $opt (keys %$confdesc) {
1920 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
1921 $opt eq 'runningmachine' || $opt eq 'runningcpu';
1922 $prop->{$opt} = $confdesc->{$opt};
1928 # return copy of $confdesc_cloudinit to generate documentation
1929 sub cloudinit_config_properties
{
1931 return dclone
($confdesc_cloudinit);
1935 my ($key, $value) = @_;
1937 die "unknown setting '$key'\n" if !$confdesc->{$key};
1939 my $type = $confdesc->{$key}->{type
};
1941 if (!defined($value)) {
1942 die "got undefined value\n";
1945 if ($value =~ m/[\n\r]/) {
1946 die "property contains a line feed\n";
1949 if ($type eq 'boolean') {
1950 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1951 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1952 die "type check ('boolean') failed - got '$value'\n";
1953 } elsif ($type eq 'integer') {
1954 return int($1) if $value =~ m/^(\d+)$/;
1955 die "type check ('integer') failed - got '$value'\n";
1956 } elsif ($type eq 'number') {
1957 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1958 die "type check ('number') failed - got '$value'\n";
1959 } elsif ($type eq 'string') {
1960 if (my $fmt = $confdesc->{$key}->{format
}) {
1961 PVE
::JSONSchema
::check_format
($fmt, $value);
1964 $value =~ s/^\"(.*)\"$/$1/;
1967 die "internal error"
1972 my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
1974 my $conf = PVE
::QemuConfig-
>load_config($vmid);
1976 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
1978 if ($conf->{template
}) {
1979 # check if any base image is still used by a linked clone
1980 PVE
::QemuConfig-
>foreach_volume($conf, sub {
1981 my ($ds, $drive) = @_;
1982 return if drive_is_cdrom
($drive);
1984 my $volid = $drive->{file
};
1985 return if !$volid || $volid =~ m
|^/|;
1987 die "base volume '$volid' is still in use by linked cloned\n"
1988 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
1993 # only remove disks owned by this VM
1994 PVE
::QemuConfig-
>foreach_volume($conf, sub {
1995 my ($ds, $drive) = @_;
1996 return if drive_is_cdrom
($drive, 1);
1998 my $volid = $drive->{file
};
1999 return if !$volid || $volid =~ m
|^/|;
2001 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2002 return if !$path || !$owner || ($owner != $vmid);
2004 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2005 warn "Could not remove disk '$volid', check manually: $@" if $@;
2008 # also remove unused disk
2009 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2010 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2011 my ($volid, $sid, $volname, $d) = @_;
2012 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2016 if (defined $replacement_conf) {
2017 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2019 PVE
::QemuConfig-
>destroy_config($vmid);
2023 sub parse_vm_config
{
2024 my ($filename, $raw) = @_;
2026 return undef if !defined($raw);
2029 digest
=> Digest
::SHA
::sha1_hex
($raw),
2034 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2035 || die "got strange filename '$filename'";
2043 my @lines = split(/\n/, $raw);
2044 foreach my $line (@lines) {
2045 next if $line =~ m/^\s*$/;
2047 if ($line =~ m/^\[PENDING\]\s*$/i) {
2048 $section = 'pending';
2049 if (defined($descr)) {
2051 $conf->{description
} = $descr;
2054 $conf = $res->{$section} = {};
2057 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2059 if (defined($descr)) {
2061 $conf->{description
} = $descr;
2064 $conf = $res->{snapshots
}->{$section} = {};
2068 if ($line =~ m/^\#(.*)\s*$/) {
2069 $descr = '' if !defined($descr);
2070 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2074 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2075 $descr = '' if !defined($descr);
2076 $descr .= PVE
::Tools
::decode_text
($2);
2077 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2078 $conf->{snapstate
} = $1;
2079 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2082 $conf->{$key} = $value;
2083 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2085 if ($section eq 'pending') {
2086 $conf->{delete} = $value; # we parse this later
2088 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2090 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2093 eval { $value = check_type
($key, $value); };
2095 warn "vm $vmid - unable to parse value of '$key' - $@";
2097 $key = 'ide2' if $key eq 'cdrom';
2098 my $fmt = $confdesc->{$key}->{format
};
2099 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2100 my $v = parse_drive
($key, $value);
2101 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2102 $v->{file
} = $volid;
2103 $value = print_drive
($v);
2105 warn "vm $vmid - unable to parse value of '$key'\n";
2110 $conf->{$key} = $value;
2115 if (defined($descr)) {
2117 $conf->{description
} = $descr;
2119 delete $res->{snapstate
}; # just to be sure
2124 sub write_vm_config
{
2125 my ($filename, $conf) = @_;
2127 delete $conf->{snapstate
}; # just to be sure
2129 if ($conf->{cdrom
}) {
2130 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2131 $conf->{ide2
} = $conf->{cdrom
};
2132 delete $conf->{cdrom
};
2135 # we do not use 'smp' any longer
2136 if ($conf->{sockets
}) {
2137 delete $conf->{smp
};
2138 } elsif ($conf->{smp
}) {
2139 $conf->{sockets
} = $conf->{smp
};
2140 delete $conf->{cores
};
2141 delete $conf->{smp
};
2144 my $used_volids = {};
2146 my $cleanup_config = sub {
2147 my ($cref, $pending, $snapname) = @_;
2149 foreach my $key (keys %$cref) {
2150 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2151 $key eq 'snapstate' || $key eq 'pending';
2152 my $value = $cref->{$key};
2153 if ($key eq 'delete') {
2154 die "propertry 'delete' is only allowed in [PENDING]\n"
2156 # fixme: check syntax?
2159 eval { $value = check_type
($key, $value); };
2160 die "unable to parse value of '$key' - $@" if $@;
2162 $cref->{$key} = $value;
2164 if (!$snapname && is_valid_drivename
($key)) {
2165 my $drive = parse_drive
($key, $value);
2166 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2171 &$cleanup_config($conf);
2173 &$cleanup_config($conf->{pending
}, 1);
2175 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2176 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2177 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2180 # remove 'unusedX' settings if we re-add a volume
2181 foreach my $key (keys %$conf) {
2182 my $value = $conf->{$key};
2183 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2184 delete $conf->{$key};
2188 my $generate_raw_config = sub {
2189 my ($conf, $pending) = @_;
2193 # add description as comment to top of file
2194 if (defined(my $descr = $conf->{description
})) {
2196 foreach my $cl (split(/\n/, $descr)) {
2197 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2200 $raw .= "#\n" if $pending;
2204 foreach my $key (sort keys %$conf) {
2205 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2206 $raw .= "$key: $conf->{$key}\n";
2211 my $raw = &$generate_raw_config($conf);
2213 if (scalar(keys %{$conf->{pending
}})){
2214 $raw .= "\n[PENDING]\n";
2215 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2218 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2219 $raw .= "\n[$snapname]\n";
2220 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2230 # we use static defaults from our JSON schema configuration
2231 foreach my $key (keys %$confdesc) {
2232 if (defined(my $default = $confdesc->{$key}->{default})) {
2233 $res->{$key} = $default;
2241 my $vmlist = PVE
::Cluster
::get_vmlist
();
2243 return $res if !$vmlist || !$vmlist->{ids
};
2244 my $ids = $vmlist->{ids
};
2245 my $nodename = nodename
();
2247 foreach my $vmid (keys %$ids) {
2248 my $d = $ids->{$vmid};
2249 next if !$d->{node
} || $d->{node
} ne $nodename;
2250 next if !$d->{type
} || $d->{type
} ne 'qemu';
2251 $res->{$vmid}->{exists} = 1;
2256 # test if VM uses local resources (to prevent migration)
2257 sub check_local_resources
{
2258 my ($conf, $noerr) = @_;
2262 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2263 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2265 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2267 foreach my $k (keys %$conf) {
2268 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2269 # sockets are safe: they will recreated be on the target side post-migrate
2270 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2271 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2274 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2279 # check if used storages are available on all nodes (use by migrate)
2280 sub check_storage_availability
{
2281 my ($storecfg, $conf, $node) = @_;
2283 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2284 my ($ds, $drive) = @_;
2286 my $volid = $drive->{file
};
2289 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2292 # check if storage is available on both nodes
2293 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2294 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2298 # list nodes where all VM images are available (used by has_feature API)
2300 my ($conf, $storecfg) = @_;
2302 my $nodelist = PVE
::Cluster
::get_nodelist
();
2303 my $nodehash = { map { $_ => 1 } @$nodelist };
2304 my $nodename = nodename
();
2306 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2307 my ($ds, $drive) = @_;
2309 my $volid = $drive->{file
};
2312 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2314 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2315 if ($scfg->{disable
}) {
2317 } elsif (my $avail = $scfg->{nodes
}) {
2318 foreach my $node (keys %$nodehash) {
2319 delete $nodehash->{$node} if !$avail->{$node};
2321 } elsif (!$scfg->{shared
}) {
2322 foreach my $node (keys %$nodehash) {
2323 delete $nodehash->{$node} if $node ne $nodename
2332 sub check_local_storage_availability
{
2333 my ($conf, $storecfg) = @_;
2335 my $nodelist = PVE
::Cluster
::get_nodelist
();
2336 my $nodehash = { map { $_ => {} } @$nodelist };
2338 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2339 my ($ds, $drive) = @_;
2341 my $volid = $drive->{file
};
2344 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2346 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2348 if ($scfg->{disable
}) {
2349 foreach my $node (keys %$nodehash) {
2350 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2352 } elsif (my $avail = $scfg->{nodes
}) {
2353 foreach my $node (keys %$nodehash) {
2354 if (!$avail->{$node}) {
2355 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2362 foreach my $node (values %$nodehash) {
2363 if (my $unavail = $node->{unavailable_storages
}) {
2364 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2371 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2373 my ($vmid, $nocheck, $node) = @_;
2375 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2376 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2381 my $vzlist = config_list
();
2383 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2385 while (defined(my $de = $fd->read)) {
2386 next if $de !~ m/^(\d+)\.pid$/;
2388 next if !defined($vzlist->{$vmid});
2389 if (my $pid = check_running
($vmid)) {
2390 $vzlist->{$vmid}->{pid
} = $pid;
2397 our $vmstatus_return_properties = {
2398 vmid
=> get_standard_option
('pve-vmid'),
2400 description
=> "Qemu process status.",
2402 enum
=> ['stopped', 'running'],
2405 description
=> "Maximum memory in bytes.",
2408 renderer
=> 'bytes',
2411 description
=> "Root disk size in bytes.",
2414 renderer
=> 'bytes',
2417 description
=> "VM name.",
2422 description
=> "Qemu QMP agent status.",
2427 description
=> "PID of running qemu process.",
2432 description
=> "Uptime.",
2435 renderer
=> 'duration',
2438 description
=> "Maximum usable CPUs.",
2443 description
=> "The current config lock, if any.",
2448 description
=> "The current configured tags, if any",
2454 my $last_proc_pid_stat;
2456 # get VM status information
2457 # This must be fast and should not block ($full == false)
2458 # We only query KVM using QMP if $full == true (this can be slow)
2460 my ($opt_vmid, $full) = @_;
2464 my $storecfg = PVE
::Storage
::config
();
2466 my $list = vzlist
();
2467 my $defaults = load_defaults
();
2469 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2471 my $cpucount = $cpuinfo->{cpus
} || 1;
2473 foreach my $vmid (keys %$list) {
2474 next if $opt_vmid && ($vmid ne $opt_vmid);
2476 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2478 my $d = { vmid
=> $vmid };
2479 $d->{pid
} = $list->{$vmid}->{pid
};
2481 # fixme: better status?
2482 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2484 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2485 if (defined($size)) {
2486 $d->{disk
} = 0; # no info available
2487 $d->{maxdisk
} = $size;
2493 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2494 * ($conf->{cores
} || $defaults->{cores
});
2495 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2496 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2498 $d->{name
} = $conf->{name
} || "VM $vmid";
2499 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2500 : $defaults->{memory
}*(1024*1024);
2502 if ($conf->{balloon
}) {
2503 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2504 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2505 : $defaults->{shares
};
2516 $d->{diskwrite
} = 0;
2518 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
2520 $d->{serial
} = 1 if conf_has_serial
($conf);
2521 $d->{lock} = $conf->{lock} if $conf->{lock};
2522 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2527 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2528 foreach my $dev (keys %$netdev) {
2529 next if $dev !~ m/^tap([1-9]\d*)i/;
2531 my $d = $res->{$vmid};
2534 $d->{netout
} += $netdev->{$dev}->{receive
};
2535 $d->{netin
} += $netdev->{$dev}->{transmit
};
2538 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
2539 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
2544 my $ctime = gettimeofday
;
2546 foreach my $vmid (keys %$list) {
2548 my $d = $res->{$vmid};
2549 my $pid = $d->{pid
};
2552 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2553 next if !$pstat; # not running
2555 my $used = $pstat->{utime} + $pstat->{stime
};
2557 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2559 if ($pstat->{vsize
}) {
2560 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2563 my $old = $last_proc_pid_stat->{$pid};
2565 $last_proc_pid_stat->{$pid} = {
2573 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2575 if ($dtime > 1000) {
2576 my $dutime = $used - $old->{used
};
2578 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2579 $last_proc_pid_stat->{$pid} = {
2585 $d->{cpu
} = $old->{cpu
};
2589 return $res if !$full;
2591 my $qmpclient = PVE
::QMPClient-
>new();
2593 my $ballooncb = sub {
2594 my ($vmid, $resp) = @_;
2596 my $info = $resp->{'return'};
2597 return if !$info->{max_mem
};
2599 my $d = $res->{$vmid};
2601 # use memory assigned to VM
2602 $d->{maxmem
} = $info->{max_mem
};
2603 $d->{balloon
} = $info->{actual
};
2605 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2606 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2607 $d->{freemem
} = $info->{free_mem
};
2610 $d->{ballooninfo
} = $info;
2613 my $blockstatscb = sub {
2614 my ($vmid, $resp) = @_;
2615 my $data = $resp->{'return'} || [];
2616 my $totalrdbytes = 0;
2617 my $totalwrbytes = 0;
2619 for my $blockstat (@$data) {
2620 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2621 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2623 $blockstat->{device
} =~ s/drive-//;
2624 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2626 $res->{$vmid}->{diskread
} = $totalrdbytes;
2627 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2630 my $statuscb = sub {
2631 my ($vmid, $resp) = @_;
2633 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2634 # this fails if ballon driver is not loaded, so this must be
2635 # the last commnand (following command are aborted if this fails).
2636 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2638 my $status = 'unknown';
2639 if (!defined($status = $resp->{'return'}->{status
})) {
2640 warn "unable to get VM status\n";
2644 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2647 foreach my $vmid (keys %$list) {
2648 next if $opt_vmid && ($vmid ne $opt_vmid);
2649 next if !$res->{$vmid}->{pid
}; # not running
2650 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2653 $qmpclient->queue_execute(undef, 2);
2655 foreach my $vmid (keys %$list) {
2656 next if $opt_vmid && ($vmid ne $opt_vmid);
2657 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2663 sub conf_has_serial
{
2666 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2667 if ($conf->{"serial$i"}) {
2675 sub conf_has_audio
{
2676 my ($conf, $id) = @_;
2679 my $audio = $conf->{"audio$id"};
2680 return undef if !defined($audio);
2682 my $audioproperties = PVE
::JSONSchema
::parse_property_string
($audio_fmt, $audio);
2683 my $audiodriver = $audioproperties->{driver
} // 'spice';
2686 dev
=> $audioproperties->{device
},
2687 dev_id
=> "audiodev$id",
2688 backend
=> $audiodriver,
2689 backend_id
=> "$audiodriver-backend${id}",
2694 my ($audio, $audiopciaddr, $machine_version) = @_;
2698 my $id = $audio->{dev_id
};
2700 if (min_version
($machine_version, 4, 2)) {
2701 $audiodev = ",audiodev=$audio->{backend_id}";
2704 if ($audio->{dev
} eq 'AC97') {
2705 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2706 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
2707 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2708 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2709 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2711 die "unkown audio device '$audio->{dev}', implement me!";
2714 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2719 sub vga_conf_has_spice
{
2722 my $vgaconf = parse_vga
($vga);
2723 my $vgatype = $vgaconf->{type
};
2724 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2731 return get_host_arch
() eq $arch;
2736 return $conf->{arch
} // get_host_arch
();
2739 my $default_machines = {
2744 sub get_vm_machine
{
2745 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2747 my $machine = $forcemachine || $conf->{machine
};
2749 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2751 $machine ||= $default_machines->{$arch};
2752 if ($add_pve_version) {
2753 $kvmversion //= kvm_user_version
();
2754 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
2755 $machine .= "+pve$pvever";
2759 if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2760 # for version-pinned machines that do not include a pve-version (e.g.
2761 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2762 $machine .= '+pve0';
2768 sub get_ovmf_files
($) {
2771 my $ovmf = $OVMF->{$arch}
2772 or die "no OVMF images known for architecture '$arch'\n";
2778 aarch64
=> '/usr/bin/qemu-system-aarch64',
2779 x86_64
=> '/usr/bin/qemu-system-x86_64',
2781 sub get_command_for_arch
($) {
2783 return '/usr/bin/kvm' if is_native
($arch);
2785 my $cmd = $Arch2Qemu->{$arch}
2786 or die "don't know how to emulate architecture '$arch'\n";
2790 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2791 # to use in a QEMU command line (-cpu element), first array_intersect the result
2792 # of query_supported_ with query_understood_. This is necessary because:
2794 # a) query_understood_ returns flags the host cannot use and
2795 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2796 # flags, but CPU settings - with most of them being flags. Those settings
2797 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
2799 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2800 # expensive. If you need the value returned from this, you can get it much
2801 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2802 # $accel being 'kvm' or 'tcg'.
2804 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2805 # changes, automatically populating pmxcfs.
2807 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2808 # since kvm and tcg machines support different flags
2810 sub query_supported_cpu_flags
{
2813 $arch //= get_host_arch
();
2814 my $default_machine = $default_machines->{$arch};
2818 # FIXME: Once this is merged, the code below should work for ARM as well:
2819 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2820 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2823 my $kvm_supported = defined(kvm_version
());
2824 my $qemu_cmd = get_command_for_arch
($arch);
2826 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
2828 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2829 my $query_supported_run_qemu = sub {
2835 '-machine', $default_machine,
2837 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2838 '-mon', 'chardev=qmp,mode=control',
2839 '-pidfile', $pidfile,
2844 push @$cmd, '-accel', 'tcg';
2847 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
2848 die "QEMU flag querying VM exited with code " . $rc if $rc;
2851 my $cmd_result = mon_cmd
(
2853 'query-cpu-model-expansion',
2855 model
=> { name
=> 'host' }
2858 my $props = $cmd_result->{model
}->{props
};
2859 foreach my $prop (keys %$props) {
2860 next if $props->{$prop} ne '1';
2861 # QEMU returns some flags multiple times, with '_', '.' or '-'
2862 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2863 # We only keep those with underscores, to match /proc/cpuinfo
2864 $prop =~ s/\.|-/_/g;
2865 $flags->{$prop} = 1;
2870 # force stop with 10 sec timeout and 'nocheck'
2871 # always stop, even if QMP failed
2872 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
2876 return [ sort keys %$flags ];
2879 # We need to query QEMU twice, since KVM and TCG have different supported flags
2880 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
2881 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
2882 warn "warning: failed querying supported tcg flags: $@\n" if $@;
2884 if ($kvm_supported) {
2885 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
2886 warn "warning: failed querying supported kvm flags: $@\n" if $@;
2893 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2894 my $understood_cpu_flag_dir = "/usr/share/kvm";
2895 sub query_understood_cpu_flags
{
2896 my $arch = get_host_arch
();
2897 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2899 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2902 my $raw = file_get_contents
($filepath);
2903 $raw =~ s/^\s+|\s+$//g;
2904 my @flags = split(/\s+/, $raw);
2909 sub config_to_command
{
2910 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
2913 my $globalFlags = [];
2914 my $machineFlags = [];
2919 my $ostype = $conf->{ostype
};
2920 my $winversion = windows_version
($ostype);
2921 my $kvm = $conf->{kvm
};
2922 my $nodename = nodename
();
2924 my $arch = get_vm_arch
($conf);
2925 my $kvm_binary = get_command_for_arch
($arch);
2926 my $kvmver = kvm_user_version
($kvm_binary);
2928 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2929 $kvmver //= "undefined";
2930 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2933 my $add_pve_version = min_version
($kvmver, 4, 1);
2935 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
2936 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, $kvmver);
2937 $kvm //= 1 if is_native
($arch);
2939 $machine_version =~ m/(\d+)\.(\d+)/;
2940 my ($machine_major, $machine_minor) = ($1, $2);
2942 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
2943 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
2944 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
2945 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2946 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
2947 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
2948 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2949 ." is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2952 # if a specific +pve version is required for a feature, use $version_guard
2953 # instead of min_version to allow machines to be run with the minimum
2955 my $required_pve_version = 0;
2956 my $version_guard = sub {
2957 my ($major, $minor, $pve) = @_;
2958 return 0 if !min_version
($machine_version, $major, $minor, $pve);
2959 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
2960 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
2961 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2966 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2967 if !defined kvm_version
();
2970 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2971 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
2972 my $use_old_bios_files = undef;
2973 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
2975 my $cpuunits = defined($conf->{cpuunits
}) ?
2976 $conf->{cpuunits
} : $defaults->{cpuunits
};
2978 push @$cmd, $kvm_binary;
2980 push @$cmd, '-id', $vmid;
2982 my $vmname = $conf->{name
} || "vm$vmid";
2984 push @$cmd, '-name', $vmname;
2988 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
2989 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2990 push @$cmd, '-mon', "chardev=qmp,mode=control";
2992 if (min_version
($machine_version, 2, 12)) {
2993 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
2994 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
2997 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
2999 push @$cmd, '-daemonize';
3001 if ($conf->{smbios1
}) {
3002 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3003 if ($smbios_conf->{base64
}) {
3004 # Do not pass base64 flag to qemu
3005 delete $smbios_conf->{base64
};
3006 my $smbios_string = "";
3007 foreach my $key (keys %$smbios_conf) {
3009 if ($key eq "uuid") {
3010 $value = $smbios_conf->{uuid
}
3012 $value = decode_base64
($smbios_conf->{$key});
3014 # qemu accepts any binary data, only commas need escaping by double comma
3016 $smbios_string .= "," . $key . "=" . $value if $value;
3018 push @$cmd, '-smbios', "type=1" . $smbios_string;
3020 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3024 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3025 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3026 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3028 my ($path, $format);
3029 if (my $efidisk = $conf->{efidisk0
}) {
3030 my $d = parse_drive
('efidisk0', $efidisk);
3031 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3032 $format = $d->{format
};
3034 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3035 if (!defined($format)) {
3036 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3037 $format = qemu_img_format
($scfg, $volname);
3041 die "efidisk format must be specified\n"
3042 if !defined($format);
3045 warn "no efidisk configured! Using temporary efivars disk.\n";
3046 $path = "/tmp/$vmid-ovmf.fd";
3047 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3053 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3054 $size_str = ",size=" . (-s
$ovmf_vars);
3057 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3058 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3063 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3064 if (min_version
($machine_version, 4, 0)) {
3065 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3067 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3071 if ($conf->{vmgenid
}) {
3072 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3075 # add usb controllers
3076 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
($conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3077 push @$devices, @usbcontrollers if @usbcontrollers;
3078 my $vga = parse_vga
($conf->{vga
});
3080 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3081 $vga->{type
} = 'qxl' if $qxlnum;
3083 if (!$vga->{type
}) {
3084 if ($arch eq 'aarch64') {
3085 $vga->{type
} = 'virtio';
3086 } elsif (min_version
($machine_version, 2, 9)) {
3087 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3089 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3093 # enable absolute mouse coordinates (needed by vnc)
3095 if (defined($conf->{tablet
})) {
3096 $tablet = $conf->{tablet
};
3098 $tablet = $defaults->{tablet
};
3099 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3100 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3104 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3105 my $kbd = print_keyboarddevice_full
($conf, $arch);
3106 push @$devices, '-device', $kbd if defined($kbd);
3109 # host pci device passthrough
3110 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3111 $vmid, $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type);
3114 my $usb_dev_features = {};
3115 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3117 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
($conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features);
3118 push @$devices, @usbdevices if @usbdevices;
3120 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3121 if (my $path = $conf->{"serial$i"}) {
3122 if ($path eq 'socket') {
3123 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3124 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3125 # On aarch64, serial0 is the UART device. Qemu only allows
3126 # connecting UART devices via the '-serial' command line, as
3127 # the device has a fixed slot on the hardware...
3128 if ($arch eq 'aarch64' && $i == 0) {
3129 push @$devices, '-serial', "chardev:serial$i";
3131 push @$devices, '-device', "isa-serial,chardev=serial$i";
3134 die "no such serial device\n" if ! -c
$path;
3135 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3136 push @$devices, '-device', "isa-serial,chardev=serial$i";
3142 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3143 if (my $path = $conf->{"parallel$i"}) {
3144 die "no such parallel device\n" if ! -c
$path;
3145 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3146 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3147 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3151 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3152 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3153 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3154 push @$devices, @$audio_devs;
3158 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3159 $sockets = $conf->{sockets
} if $conf->{sockets
};
3161 my $cores = $conf->{cores
} || 1;
3163 my $maxcpus = $sockets * $cores;
3165 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3167 my $allowed_vcpus = $cpuinfo->{cpus
};
3169 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3170 if ($allowed_vcpus < $maxcpus);
3172 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3174 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3175 for (my $i = 2; $i <= $vcpus; $i++) {
3176 my $cpustr = print_cpu_device
($conf,$i);
3177 push @$cmd, '-device', $cpustr;
3182 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3184 push @$cmd, '-nodefaults';
3186 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
3188 my $bootindex_hash = {};
3190 foreach my $o (split(//, $bootorder)) {
3191 $bootindex_hash->{$o} = $i*100;
3195 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3197 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3199 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3201 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3202 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3203 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3204 push @$cmd, '-vnc', "unix:$socket,password";
3206 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3207 push @$cmd, '-nographic';
3211 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3212 my $useLocaltime = $conf->{localtime};
3214 if ($winversion >= 5) { # windows
3215 $useLocaltime = 1 if !defined($conf->{localtime});
3217 # use time drift fix when acpi is enabled
3218 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3219 $tdf = 1 if !defined($conf->{tdf
});
3223 if ($winversion >= 6) {
3224 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3225 push @$cmd, '-no-hpet';
3228 push @$rtcFlags, 'driftfix=slew' if $tdf;
3230 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3231 push @$rtcFlags, "base=$conf->{startdate}";
3232 } elsif ($useLocaltime) {
3233 push @$rtcFlags, 'base=localtime';
3237 push @$cmd, '-cpu', $forcecpu;
3239 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3242 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3244 push @$cmd, '-S' if $conf->{freeze
};
3246 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3248 my $guest_agent = parse_guest_agent
($conf);
3250 if ($guest_agent->{enabled
}) {
3251 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3252 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3254 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3255 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3256 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3257 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3258 } elsif ($guest_agent->{type
} eq 'isa') {
3259 push @$devices, '-device', "isa-serial,chardev=qga0";
3263 my $rng = parse_rng
($conf->{rng0
}) if $conf->{rng0
};
3264 if ($rng && &$version_guard(4, 1, 2)) {
3265 check_rng_source
($rng->{source
});
3267 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3268 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3269 my $limiter_str = "";
3271 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3274 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3275 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3276 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3284 for (my $i = 1; $i < $qxlnum; $i++){
3285 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3288 # assume other OS works like Linux
3289 my ($ram, $vram) = ("134217728", "67108864");
3290 if ($vga->{memory
}) {
3291 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3292 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3294 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3295 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3299 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3301 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3302 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3303 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3305 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3306 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3307 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3309 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3310 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3312 my $spice_enhancement = PVE
::JSONSchema
::parse_property_string
($spice_enhancements_fmt, $conf->{spice_enhancements
} // '');
3313 if ($spice_enhancement->{foldersharing
}) {
3314 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3315 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3318 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3319 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming
};
3320 push @$devices, '-spice', "$spice_opts";
3323 # enable balloon by default, unless explicitly disabled
3324 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3325 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3326 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3329 if ($conf->{watchdog
}) {
3330 my $wdopts = parse_watchdog
($conf->{watchdog
});
3331 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3332 my $watchdog = $wdopts->{model
} || 'i6300esb';
3333 push @$devices, '-device', "$watchdog$pciaddr";
3334 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3338 my $scsicontroller = {};
3339 my $ahcicontroller = {};
3340 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3342 # Add iscsi initiator name if available
3343 if (my $initiator = get_initiator_name
()) {
3344 push @$devices, '-iscsi', "initiator-name=$initiator";
3347 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3348 my ($ds, $drive) = @_;
3350 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3351 push @$vollist, $drive->{file
};
3354 # ignore efidisk here, already added in bios/fw handling code above
3355 return if $drive->{interface
} eq 'efidisk';
3357 $use_virtio = 1 if $ds =~ m/^virtio/;
3359 if (drive_is_cdrom
($drive)) {
3360 if ($bootindex_hash->{d
}) {
3361 $drive->{bootindex
} = $bootindex_hash->{d
};
3362 $bootindex_hash->{d
} += 1;
3365 if ($bootindex_hash->{c
}) {
3366 $drive->{bootindex
} = $bootindex_hash->{c
} if $conf->{bootdisk
} && ($conf->{bootdisk
} eq $ds);
3367 $bootindex_hash->{c
} += 1;
3371 if ($drive->{interface
} eq 'virtio'){
3372 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3375 if ($drive->{interface
} eq 'scsi') {
3377 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3379 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3380 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3382 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3383 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3386 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3387 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3388 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3389 } elsif ($drive->{iothread
}) {
3390 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3394 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3395 $queues = ",num_queues=$drive->{queues}";
3398 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3399 $scsicontroller->{$controller}=1;
3402 if ($drive->{interface
} eq 'sata') {
3403 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3404 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3405 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3406 $ahcicontroller->{$controller}=1;
3409 my $drive_cmd = print_drive_commandline_full
($storecfg, $vmid, $drive);
3410 $drive_cmd .= ',readonly' if PVE
::QemuConfig-
>is_template($conf);
3412 push @$devices, '-drive',$drive_cmd;
3413 push @$devices, '-device', print_drivedevice_full
($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3416 for (my $i = 0; $i < $MAX_NETS; $i++) {
3417 next if !$conf->{"net$i"};
3418 my $d = parse_net
($conf->{"net$i"});
3421 $use_virtio = 1 if $d->{model
} eq 'virtio';
3423 if ($bootindex_hash->{n
}) {
3424 $d->{bootindex
} = $bootindex_hash->{n
};
3425 $bootindex_hash->{n
} += 1;
3428 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, "net$i");
3429 push @$devices, '-netdev', $netdevfull;
3431 my $netdevicefull = print_netdevice_full
($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3432 push @$devices, '-device', $netdevicefull;
3435 if ($conf->{ivshmem
}) {
3436 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3440 $bus = print_pcie_addr
("ivshmem");
3442 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3445 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3446 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3448 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3449 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3452 # pci.4 is nested in pci.1
3453 $bridges->{1} = 1 if $bridges->{4};
3457 if (min_version
($machine_version, 2, 3)) {
3462 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3466 for my $k (sort {$b cmp $a} keys %$bridges) {
3467 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3470 if ($k == 2 && $legacy_igd) {
3473 $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
3475 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3477 # add after -readconfig pve-q35.cfg
3478 splice @$devices, 2, 0, '-device', $devstr;
3480 unshift @$devices, '-device', $devstr if $k > 0;
3485 push @$machineFlags, 'accel=tcg';
3488 my $machine_type_min = $machine_type;
3489 if ($add_pve_version) {
3490 $machine_type_min =~ s/\+pve\d+$//;
3491 $machine_type_min .= "+pve$required_pve_version";
3493 push @$machineFlags, "type=${machine_type_min}";
3495 push @$cmd, @$devices;
3496 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3497 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3498 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3500 if (my $vmstate = $conf->{vmstate
}) {
3501 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3502 push @$vollist, $vmstate;
3503 push @$cmd, '-loadstate', $statepath;
3504 print "activating and using '$vmstate' as vmstate\n";
3508 if ($conf->{args
}) {
3509 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3513 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3516 sub check_rng_source
{
3519 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3520 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3523 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3524 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
3525 # Needs to abort, otherwise QEMU crashes on first rng access.
3526 # Note that rng_current cannot be changed to 'none' manually, so
3527 # once the VM is past this point, it is no longer an issue.
3528 die "Cannot start VM with passed-through RNG device: '/dev/hwrng'"
3529 . " exists, but '$rng_current' is set to 'none'. Ensure that"
3530 . " a compatible hardware-RNG is attached to the host.\n";
3537 my $res = mon_cmd
($vmid, 'query-spice');
3539 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3542 sub vm_devices_list
{
3545 my $res = mon_cmd
($vmid, 'query-pci');
3546 my $devices_to_check = [];
3548 foreach my $pcibus (@$res) {
3549 push @$devices_to_check, @{$pcibus->{devices
}},
3552 while (@$devices_to_check) {
3554 for my $d (@$devices_to_check) {
3555 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3556 next if !$d->{'pci_bridge'};
3558 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3559 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3561 $devices_to_check = $to_check;
3564 my $resblock = mon_cmd
($vmid, 'query-block');
3565 foreach my $block (@$resblock) {
3566 if($block->{device
} =~ m/^drive-(\S+)/){
3571 my $resmice = mon_cmd
($vmid, 'query-mice');
3572 foreach my $mice (@$resmice) {
3573 if ($mice->{name
} eq 'QEMU HID Tablet') {
3574 $devices->{tablet
} = 1;
3579 # for usb devices there is no query-usb
3580 # but we can iterate over the entries in
3581 # qom-list path=/machine/peripheral
3582 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3583 foreach my $per (@$resperipheral) {
3584 if ($per->{name
} =~ m/^usb\d+$/) {
3585 $devices->{$per->{name
}} = 1;
3593 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3595 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3597 my $devices_list = vm_devices_list
($vmid);
3598 return 1 if defined($devices_list->{$deviceid});
3600 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3602 if ($deviceid eq 'tablet') {
3604 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
3606 } elsif ($deviceid eq 'keyboard') {
3608 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
3610 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3612 die "usb hotplug currently not reliable\n";
3613 # since we can't reliably hot unplug all added usb devices
3614 # and usb passthrough disables live migration
3615 # we disable usb hotplugging for now
3616 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device));
3618 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3620 qemu_iothread_add
($vmid, $deviceid, $device);
3622 qemu_driveadd
($storecfg, $vmid, $device);
3623 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3625 qemu_deviceadd
($vmid, $devicefull);
3626 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3628 eval { qemu_drivedel
($vmid, $deviceid); };
3633 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3636 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3637 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3638 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3640 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3642 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3643 qemu_iothread_add
($vmid, $deviceid, $device);
3644 $devicefull .= ",iothread=iothread-$deviceid";
3647 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3648 $devicefull .= ",num_queues=$device->{queues}";
3651 qemu_deviceadd
($vmid, $devicefull);
3652 qemu_deviceaddverify
($vmid, $deviceid);
3654 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3656 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3657 qemu_driveadd
($storecfg, $vmid, $device);
3659 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3660 eval { qemu_deviceadd
($vmid, $devicefull); };
3662 eval { qemu_drivedel
($vmid, $deviceid); };
3667 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3669 return undef if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
3671 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
3672 my $use_old_bios_files = undef;
3673 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3675 my $netdevicefull = print_netdevice_full
($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3676 qemu_deviceadd
($vmid, $netdevicefull);
3678 qemu_deviceaddverify
($vmid, $deviceid);
3679 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
3682 eval { qemu_netdevdel
($vmid, $deviceid); };
3687 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3690 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3691 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3693 qemu_deviceadd
($vmid, $devicefull);
3694 qemu_deviceaddverify
($vmid, $deviceid);
3697 die "can't hotplug device '$deviceid'\n";
3703 # fixme: this should raise exceptions on error!
3704 sub vm_deviceunplug
{
3705 my ($vmid, $conf, $deviceid) = @_;
3707 my $devices_list = vm_devices_list
($vmid);
3708 return 1 if !defined($devices_list->{$deviceid});
3710 die "can't unplug bootdisk" if $conf->{bootdisk
} && $conf->{bootdisk
} eq $deviceid;
3712 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3714 qemu_devicedel
($vmid, $deviceid);
3716 } elsif ($deviceid =~ m/^usb\d+$/) {
3718 die "usb hotplug currently not reliable\n";
3719 # when unplugging usb devices this way,
3720 # there may be remaining usb controllers/hubs
3721 # so we disable it for now
3722 qemu_devicedel
($vmid, $deviceid);
3723 qemu_devicedelverify
($vmid, $deviceid);
3725 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3727 qemu_devicedel
($vmid, $deviceid);
3728 qemu_devicedelverify
($vmid, $deviceid);
3729 qemu_drivedel
($vmid, $deviceid);
3730 qemu_iothread_del
($conf, $vmid, $deviceid);
3732 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3734 qemu_devicedel
($vmid, $deviceid);
3735 qemu_devicedelverify
($vmid, $deviceid);
3736 qemu_iothread_del
($conf, $vmid, $deviceid);
3738 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3740 qemu_devicedel
($vmid, $deviceid);
3741 qemu_drivedel
($vmid, $deviceid);
3742 qemu_deletescsihw
($conf, $vmid, $deviceid);
3744 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3746 qemu_devicedel
($vmid, $deviceid);
3747 qemu_devicedelverify
($vmid, $deviceid);
3748 qemu_netdevdel
($vmid, $deviceid);
3751 die "can't unplug device '$deviceid'\n";
3757 sub qemu_deviceadd
{
3758 my ($vmid, $devicefull) = @_;
3760 $devicefull = "driver=".$devicefull;
3761 my %options = split(/[=,]/, $devicefull);
3763 mon_cmd
($vmid, "device_add" , %options);
3766 sub qemu_devicedel
{
3767 my ($vmid, $deviceid) = @_;
3769 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
3772 sub qemu_iothread_add
{
3773 my($vmid, $deviceid, $device) = @_;
3775 if ($device->{iothread
}) {
3776 my $iothreads = vm_iothreads_list
($vmid);
3777 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3781 sub qemu_iothread_del
{
3782 my($conf, $vmid, $deviceid) = @_;
3784 my $confid = $deviceid;
3785 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3786 $confid = 'scsi' . $1;
3788 my $device = parse_drive
($confid, $conf->{$confid});
3789 if ($device->{iothread
}) {
3790 my $iothreads = vm_iothreads_list
($vmid);
3791 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3795 sub qemu_objectadd
{
3796 my($vmid, $objectid, $qomtype) = @_;
3798 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
3803 sub qemu_objectdel
{
3804 my($vmid, $objectid) = @_;
3806 mon_cmd
($vmid, "object-del", id
=> $objectid);
3812 my ($storecfg, $vmid, $device) = @_;
3814 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device);
3815 $drive =~ s/\\/\\\\/g;
3816 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
3818 # If the command succeeds qemu prints: "OK
"
3819 return 1 if $ret =~ m/OK/s;
3821 die "adding drive failed
: $ret\n";
3825 my($vmid, $deviceid) = @_;
3827 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
3830 return 1 if $ret eq "";
3832 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3833 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3835 die "deleting drive
$deviceid failed
: $ret\n";
3838 sub qemu_deviceaddverify {
3839 my ($vmid, $deviceid) = @_;
3841 for (my $i = 0; $i <= 5; $i++) {
3842 my $devices_list = vm_devices_list($vmid);
3843 return 1 if defined($devices_list->{$deviceid});
3847 die "error on hotplug device
'$deviceid'\n";
3851 sub qemu_devicedelverify {
3852 my ($vmid, $deviceid) = @_;
3854 # need to verify that the device is correctly removed as device_del
3855 # is async and empty return is not reliable
3857 for (my $i = 0; $i <= 5; $i++) {
3858 my $devices_list = vm_devices_list($vmid);
3859 return 1 if !defined($devices_list->{$deviceid});
3863 die "error on hot-unplugging device
'$deviceid'\n";
3866 sub qemu_findorcreatescsihw {
3867 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3869 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3871 my $scsihwid="$controller_prefix$controller";
3872 my $devices_list = vm_devices_list($vmid);
3874 if(!defined($devices_list->{$scsihwid})) {
3875 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3881 sub qemu_deletescsihw {
3882 my ($conf, $vmid, $opt) = @_;
3884 my $device = parse_drive($opt, $conf->{$opt});
3886 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3887 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
3891 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3893 my $devices_list = vm_devices_list($vmid);
3894 foreach my $opt (keys %{$devices_list}) {
3895 if (is_valid_drivename($opt)) {
3896 my $drive = parse_drive($opt, $conf->{$opt});
3897 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3903 my $scsihwid="scsihw
$controller";
3905 vm_deviceunplug($vmid, $conf, $scsihwid);
3910 sub qemu_add_pci_bridge {
3911 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3917 print_pci_addr($device, $bridges, $arch, $machine_type);
3919 while (my ($k, $v) = each %$bridges) {
3922 return 1 if !defined($bridgeid) || $bridgeid < 1;
3924 my $bridge = "pci
.$bridgeid";
3925 my $devices_list = vm_devices_list($vmid);
3927 if (!defined($devices_list->{$bridge})) {
3928 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3934 sub qemu_set_link_status {
3935 my ($vmid, $device, $up) = @_;
3937 mon_cmd($vmid, "set_link
", name => $device,
3938 up => $up ? JSON::true : JSON::false);
3941 sub qemu_netdevadd {
3942 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
3944 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
3945 my %options = split(/[=,]/, $netdev);
3947 if (defined(my $vhost = $options{vhost})) {
3948 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
3951 if (defined(my $queues = $options{queues})) {
3952 $options{queues} = $queues + 0;
3955 mon_cmd($vmid, "netdev_add
", %options);
3959 sub qemu_netdevdel {
3960 my ($vmid, $deviceid) = @_;
3962 mon_cmd($vmid, "netdev_del
", id => $deviceid);
3965 sub qemu_usb_hotplug {
3966 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3970 # remove the old one first
3971 vm_deviceunplug($vmid, $conf, $deviceid);
3973 # check if xhci controller is necessary and available
3974 if ($device->{usb3}) {
3976 my $devicelist = vm_devices_list($vmid);
3978 if (!$devicelist->{xhci}) {
3979 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
3980 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
3983 my $d = parse_usb_device($device->{host});
3984 $d->{usb3} = $device->{usb3};
3987 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
3990 sub qemu_cpu_hotplug {
3991 my ($vmid, $conf, $vcpus) = @_;
3993 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
3996 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3997 $sockets = $conf->{sockets} if $conf->{sockets};
3998 my $cores = $conf->{cores} || 1;
3999 my $maxcpus = $sockets * $cores;
4001 $vcpus = $maxcpus if !$vcpus;
4003 die "you can
't add more vcpus than maxcpus\n"
4004 if $vcpus > $maxcpus;
4006 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4008 if ($vcpus < $currentvcpus) {
4010 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4012 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4013 qemu_devicedel($vmid, "cpu$i");
4015 my $currentrunningvcpus = undef;
4017 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4018 last if scalar(@{$currentrunningvcpus}) == $i-1;
4019 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4023 #update conf after each succesfull cpu unplug
4024 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4025 PVE::QemuConfig->write_config($vmid, $conf);
4028 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4034 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4035 die "vcpus in running vm does not match its configuration\n"
4036 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4038 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4040 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4041 my $cpustr = print_cpu_device($conf, $i);
4042 qemu_deviceadd($vmid, $cpustr);
4045 my $currentrunningvcpus = undef;
4047 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4048 last if scalar(@{$currentrunningvcpus}) == $i;
4049 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4053 #update conf after each succesfull cpu hotplug
4054 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4055 PVE::QemuConfig->write_config($vmid, $conf);
4059 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4060 mon_cmd($vmid, "cpu-add", id => int($i));
4065 sub qemu_block_set_io_throttle {
4066 my ($vmid, $deviceid,
4067 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4068 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4069 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4070 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4072 return if !check_running($vmid) ;
4074 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4076 bps_rd => int($bps_rd),
4077 bps_wr => int($bps_wr),
4079 iops_rd => int($iops_rd),
4080 iops_wr => int($iops_wr),
4081 bps_max => int($bps_max),
4082 bps_rd_max => int($bps_rd_max),
4083 bps_wr_max => int($bps_wr_max),
4084 iops_max => int($iops_max),
4085 iops_rd_max => int($iops_rd_max),
4086 iops_wr_max => int($iops_wr_max),
4087 bps_max_length => int($bps_max_length),
4088 bps_rd_max_length => int($bps_rd_max_length),
4089 bps_wr_max_length => int($bps_wr_max_length),
4090 iops_max_length => int($iops_max_length),
4091 iops_rd_max_length => int($iops_rd_max_length),
4092 iops_wr_max_length => int($iops_wr_max_length),
4097 # old code, only used to shutdown old VM after update
4099 my ($fh, $timeout) = @_;
4101 my $sel = new IO::Select;
4108 while (scalar (@ready = $sel->can_read($timeout))) {
4110 if ($count = $fh->sysread($buf, 8192)) {
4111 if ($buf =~ /^(.*)\(qemu\) $/s) {
4118 if (!defined($count)) {
4125 die "monitor read timeout\n" if !scalar(@ready);
4130 sub qemu_block_resize {
4131 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4133 my $running = check_running($vmid);
4135 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4137 return if !$running;
4139 my $padding = (1024 - $size % 1024) % 1024;
4140 $size = $size + $padding;
4142 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4146 sub qemu_volume_snapshot {
4147 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4149 my $running = check_running($vmid);
4151 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4152 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4154 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4158 sub qemu_volume_snapshot_delete {
4159 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4161 my $running = check_running($vmid);
4166 my $conf = PVE::QemuConfig->load_config($vmid);
4167 PVE::QemuConfig->foreach_volume($conf, sub {
4168 my ($ds, $drive) = @_;
4169 $running = 1 if $drive->{file} eq $volid;
4173 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4174 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4176 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4180 sub set_migration_caps {
4186 "auto-converge" => 1,
4188 "x-rdma-pin-all" => 0,
4193 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4195 for my $supported_capability (@$supported_capabilities) {
4197 capability => $supported_capability->{capability},
4198 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4202 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4206 my ($conf, $func, @param) = @_;
4210 my $test_volid = sub {
4211 my ($key, $drive, $snapname) = @_;
4213 my $volid = $drive->{file};
4216 $volhash->{$volid}->{cdrom} //= 1;
4217 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4219 my $replicate = $drive->{replicate} // 1;
4220 $volhash->{$volid}->{replicate} //= 0;
4221 $volhash->{$volid}->{replicate} = 1 if $replicate;
4223 $volhash->{$volid}->{shared} //= 0;
4224 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4226 $volhash->{$volid}->{referenced_in_config} //= 0;
4227 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4229 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4230 if defined($snapname);
4232 my $size = $drive->{size};
4233 $volhash->{$volid}->{size} //= $size if $size;
4235 $volhash->{$volid}->{is_vmstate} //= 0;
4236 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4238 $volhash->{$volid}->{is_unused} //= 0;
4239 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4242 my $include_opts = {
4243 extra_keys => ['vmstate
'],
4244 include_unused => 1,
4247 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4248 foreach my $snapname (keys %{$conf->{snapshots}}) {
4249 my $snap = $conf->{snapshots}->{$snapname};
4250 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4253 foreach my $volid (keys %$volhash) {
4254 &$func($volid, $volhash->{$volid}, @param);
4258 my $fast_plug_option = {
4266 'vmstatestorage
' => 1,
4271 # hotplug changes in [PENDING]
4272 # $selection hash can be used to only apply specified options, for
4273 # example: { cores => 1 } (only apply changed 'cores
')
4274 # $errors ref is used to return error messages
4275 sub vmconfig_hotplug_pending {
4276 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4278 my $defaults = load_defaults();
4279 my $arch = get_vm_arch($conf);
4280 my $machine_type = get_vm_machine($conf, undef, $arch);
4282 # commit values which do not have any impact on running VM first
4283 # Note: those option cannot raise errors, we we do not care about
4284 # $selection and always apply them.
4286 my $add_error = sub {
4287 my ($opt, $msg) = @_;
4288 $errors->{$opt} = "hotplug problem - $msg";
4292 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4293 if ($fast_plug_option->{$opt}) {
4294 $conf->{$opt} = $conf->{pending}->{$opt};
4295 delete $conf->{pending}->{$opt};
4301 PVE::QemuConfig->write_config($vmid, $conf);
4304 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4306 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4307 foreach my $opt (sort keys %$pending_delete_hash) {
4308 next if $selection && !$selection->{$opt};
4309 my $force = $pending_delete_hash->{$opt}->{force};
4311 if ($opt eq 'hotplug
') {
4312 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4313 } elsif ($opt eq 'tablet
') {
4314 die "skip\n" if !$hotplug_features->{usb};
4315 if ($defaults->{tablet}) {
4316 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4317 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4318 if $arch eq 'aarch64
';
4320 vm_deviceunplug($vmid, $conf, 'tablet
');
4321 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4323 } elsif ($opt =~ m/^usb\d+/) {
4325 # since we cannot reliably hot unplug usb devices
4326 # we are disabling it
4327 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4328 vm_deviceunplug($vmid, $conf, $opt);
4329 } elsif ($opt eq 'vcpus
') {
4330 die "skip\n" if !$hotplug_features->{cpu};
4331 qemu_cpu_hotplug($vmid, $conf, undef);
4332 } elsif ($opt eq 'balloon
') {
4333 # enable balloon device is not hotpluggable
4334 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4335 # here we reset the ballooning value to memory
4336 my $balloon = $conf->{memory} || $defaults->{memory};
4337 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4338 } elsif ($fast_plug_option->{$opt}) {
4340 } elsif ($opt =~ m/^net(\d+)$/) {
4341 die "skip\n" if !$hotplug_features->{network};
4342 vm_deviceunplug($vmid, $conf, $opt);
4343 } elsif (is_valid_drivename($opt)) {
4344 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4345 vm_deviceunplug($vmid, $conf, $opt);
4346 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4347 } elsif ($opt =~ m/^memory$/) {
4348 die "skip\n" if !$hotplug_features->{memory};
4349 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4350 } elsif ($opt eq 'cpuunits
') {
4351 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4352 } elsif ($opt eq 'cpulimit
') {
4353 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4359 &$add_error($opt, $err) if $err ne "skip\n";
4361 delete $conf->{$opt};
4362 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4366 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4367 $apply_pending_cloudinit = sub {
4368 return if $apply_pending_cloudinit_done; # once is enough
4369 $apply_pending_cloudinit_done = 1; # once is enough
4371 my ($key, $value) = @_;
4373 my @cloudinit_opts = keys %$confdesc_cloudinit;
4374 foreach my $opt (keys %{$conf->{pending}}) {
4375 next if !grep { $_ eq $opt } @cloudinit_opts;
4376 $conf->{$opt} = delete $conf->{pending}->{$opt};
4379 my $new_conf = { %$conf };
4380 $new_conf->{$key} = $value;
4381 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4384 foreach my $opt (keys %{$conf->{pending}}) {
4385 next if $selection && !$selection->{$opt};
4386 my $value = $conf->{pending}->{$opt};
4388 if ($opt eq 'hotplug
') {
4389 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4390 } elsif ($opt eq 'tablet
') {
4391 die "skip\n" if !$hotplug_features->{usb};
4393 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4394 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4395 if $arch eq 'aarch64
';
4396 } elsif ($value == 0) {
4397 vm_deviceunplug($vmid, $conf, 'tablet
');
4398 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4400 } elsif ($opt =~ m/^usb\d+$/) {
4402 # since we cannot reliably hot unplug usb devices
4403 # we are disabling it
4404 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4405 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4406 die "skip\n" if !$d;
4407 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4408 } elsif ($opt eq 'vcpus
') {
4409 die "skip\n" if !$hotplug_features->{cpu};
4410 qemu_cpu_hotplug($vmid, $conf, $value);
4411 } elsif ($opt eq 'balloon
') {
4412 # enable/disable balloning device is not hotpluggable
4413 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4414 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4415 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4417 # allow manual ballooning if shares is set to zero
4418 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4419 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4420 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4422 } elsif ($opt =~ m/^net(\d+)$/) {
4423 # some changes can be done without hotplug
4424 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4425 $vmid, $opt, $value, $arch, $machine_type);
4426 } elsif (is_valid_drivename($opt)) {
4427 die "skip\n" if $opt eq 'efidisk0
';
4428 # some changes can be done without hotplug
4429 my $drive = parse_drive($opt, $value);
4430 if (drive_is_cloudinit($drive)) {
4431 &$apply_pending_cloudinit($opt, $value);
4433 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4434 $vmid, $opt, $value, $arch, $machine_type);
4435 } elsif ($opt =~ m/^memory$/) { #dimms
4436 die "skip\n" if !$hotplug_features->{memory};
4437 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4438 } elsif ($opt eq 'cpuunits
') {
4439 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4440 } elsif ($opt eq 'cpulimit
') {
4441 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4442 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4444 die "skip\n"; # skip non-hot-pluggable options
4448 &$add_error($opt, $err) if $err ne "skip\n";
4450 $conf->{$opt} = $value;
4451 delete $conf->{pending}->{$opt};
4455 PVE::QemuConfig->write_config($vmid, $conf);
4458 sub try_deallocate_drive {
4459 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4461 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4462 my $volid = $drive->{file};
4463 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4464 my $sid = PVE::Storage::parse_volume_id($volid);
4465 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4467 # check if the disk is really unused
4468 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4469 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4470 PVE::Storage::vdisk_free($storecfg, $volid);
4473 # If vm is not owner of this disk remove from config
4481 sub vmconfig_delete_or_detach_drive {
4482 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4484 my $drive = parse_drive($opt, $conf->{$opt});
4486 my $rpcenv = PVE::RPCEnvironment::get();
4487 my $authuser = $rpcenv->get_user();
4490 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4491 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4493 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4499 sub vmconfig_apply_pending {
4500 my ($vmid, $conf, $storecfg, $errors) = @_;
4502 my $add_apply_error = sub {
4503 my ($opt, $msg) = @_;
4504 my $err_msg = "unable to apply pending change $opt : $msg";
4505 $errors->{$opt} = $err_msg;
4511 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4512 foreach my $opt (sort keys %$pending_delete_hash) {
4513 my $force = $pending_delete_hash->{$opt}->{force};
4515 if ($opt =~ m/^unused/) {
4516 die "internal error";
4517 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4518 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4522 $add_apply_error->($opt, $err);
4524 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4525 delete $conf->{$opt};
4529 PVE::QemuConfig->cleanup_pending($conf);
4531 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4532 next if $opt eq 'delete'; # just to be sure
4534 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4535 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4539 $add_apply_error->($opt, $err);
4541 $conf->{$opt} = delete $conf->{pending}->{$opt};
4545 # write all changes at once to avoid unnecessary i/o
4546 PVE::QemuConfig->write_config($vmid, $conf);
4549 sub vmconfig_update_net {
4550 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4552 my $newnet = parse_net($value);
4554 if ($conf->{$opt}) {
4555 my $oldnet = parse_net($conf->{$opt});
4557 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4558 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4559 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4560 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4562 # for non online change, we try to hot-unplug
4563 die "skip\n" if !$hotplug;
4564 vm_deviceunplug($vmid, $conf, $opt);
4567 die "internal error" if $opt !~ m/net(\d+)/;
4568 my $iface = "tap${vmid}i$1";
4570 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4571 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4572 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4573 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4574 PVE::Network::tap_unplug($iface);
4577 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4579 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4581 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4582 # Rate can be applied on its own but any change above needs to
4583 # include the rate in tap_plug since OVS resets everything.
4584 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4587 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4588 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4596 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4602 sub vmconfig_update_disk {
4603 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4605 my $drive = parse_drive($opt, $value);
4607 if ($conf->{$opt}) {
4609 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
4611 my $media = $drive->{media} || 'disk
';
4612 my $oldmedia = $old_drive->{media} || 'disk
';
4613 die "unable to change media type\n" if $media ne $oldmedia;
4615 if (!drive_is_cdrom($old_drive)) {
4617 if ($drive->{file} ne $old_drive->{file}) {
4619 die "skip\n" if !$hotplug;
4621 # unplug and register as unused
4622 vm_deviceunplug($vmid, $conf, $opt);
4623 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4626 # update existing disk
4628 # skip non hotpluggable value
4629 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4630 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4631 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4632 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4633 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4638 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4639 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4640 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4641 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4642 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4643 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4644 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4645 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4646 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4647 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4648 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4649 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4650 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4651 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4652 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4653 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4654 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4655 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4657 qemu_block_set_io_throttle($vmid,"drive-$opt",
4658 ($drive->{mbps} || 0)*1024*1024,
4659 ($drive->{mbps_rd} || 0)*1024*1024,
4660 ($drive->{mbps_wr} || 0)*1024*1024,
4661 $drive->{iops} || 0,
4662 $drive->{iops_rd} || 0,
4663 $drive->{iops_wr} || 0,
4664 ($drive->{mbps_max} || 0)*1024*1024,
4665 ($drive->{mbps_rd_max} || 0)*1024*1024,
4666 ($drive->{mbps_wr_max} || 0)*1024*1024,
4667 $drive->{iops_max} || 0,
4668 $drive->{iops_rd_max} || 0,
4669 $drive->{iops_wr_max} || 0,
4670 $drive->{bps_max_length} || 1,
4671 $drive->{bps_rd_max_length} || 1,
4672 $drive->{bps_wr_max_length} || 1,
4673 $drive->{iops_max_length} || 1,
4674 $drive->{iops_rd_max_length} || 1,
4675 $drive->{iops_wr_max_length} || 1);
4684 if ($drive->{file} eq 'none
') {
4685 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4686 if (drive_is_cloudinit($old_drive)) {
4687 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4690 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4692 # force eject if locked
4693 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4696 mon_cmd($vmid, "blockdev-change-medium",
4697 id => "$opt", filename => "$path");
4706 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4708 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4709 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4712 # called in locked context by incoming migration
4713 sub vm_migrate_get_nbd_disks {
4714 my ($storecfg, $conf, $replicated_volumes) = @_;
4716 my $local_volumes = {};
4717 PVE::QemuConfig->foreach_volume($conf, sub {
4718 my ($ds, $drive) = @_;
4720 return if drive_is_cdrom($drive);
4722 my $volid = $drive->{file};
4726 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4728 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4729 return if $scfg->{shared};
4731 # replicated disks re-use existing state via bitmap
4732 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4733 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4735 return $local_volumes;
4738 # called in locked context by incoming migration
4739 sub vm_migrate_alloc_nbd_disks {
4740 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4745 foreach my $opt (sort keys %$source_volumes) {
4746 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4748 if ($use_existing) {
4749 $nbd->{$opt}->{drivestr} = print_drive($drive);
4750 $nbd->{$opt}->{volid} = $volid;
4751 $nbd->{$opt}->{replicated} = 1;
4755 # If a remote storage is specified and the format of the original
4756 # volume is not available there, fall back to the default format.
4757 # Otherwise use the same format as the original.
4758 if (!$storagemap->{identity}) {
4759 $storeid = map_storage($storagemap, $storeid);
4760 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4761 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4762 my $fileFormat = qemu_img_format($scfg, $volname);
4763 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4765 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4766 $format = qemu_img_format($scfg, $volname);
4769 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4770 my $newdrive = $drive;
4771 $newdrive->{format} = $format;
4772 $newdrive->{file} = $newvolid;
4773 my $drivestr = print_drive($newdrive);
4774 $nbd->{$opt}->{drivestr} = $drivestr;
4775 $nbd->{$opt}->{volid} = $newvolid;
4781 # see vm_start_nolock for parameters, additionally:
4783 # storagemap = parsed storage map for allocating NBD disks
4785 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4787 return PVE::QemuConfig->lock_config($vmid, sub {
4788 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4790 die "you can't start a vm
if it
's a template\n"
4791 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4793 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended
');
4795 PVE::QemuConfig->check_lock($conf)
4796 if !($params->{skiplock} || $has_suspended_lock);
4798 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4800 die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4802 if (my $storagemap = $migrate_opts->{storagemap}) {
4803 my $replicated = $migrate_opts->{replicated_volumes};
4804 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4805 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4807 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4808 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4812 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4818 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
4819 # skiplock => 0/1, skip checking for config lock
4820 # skiptemplate => 0/1, skip checking whether VM is template
4821 # forcemachine => to force Qemu machine (rollback/migration)
4822 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
4823 # timeout => in seconds
4824 # paused => start VM in paused state (backup)
4825 # resume => resume from hibernation
4827 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4828 # migratedfrom => source node
4829 # spice_ticket => used for spice migration, passed via tunnel/stdin
4830 # network => CIDR of migration network
4831 # type => secure/insecure - tunnel over encrypted connection or plain-text
4832 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
4833 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4834 sub vm_start_nolock {
4835 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4837 my $statefile = $params->{statefile};
4838 my $resume = $params->{resume};
4840 my $migratedfrom = $migrate_opts->{migratedfrom};
4841 my $migration_type = $migrate_opts->{type};
4845 # clean up leftover reboot request files
4846 eval { clear_reboot_request($vmid); };
4849 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4850 vmconfig_apply_pending($vmid, $conf, $storecfg);
4851 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4854 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4856 my $defaults = load_defaults();
4858 # set environment variable useful inside network script
4859 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4861 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
4863 my $forcemachine = $params->{forcemachine};
4864 my $forcecpu = $params->{forcecpu};
4866 # enforce machine and CPU type on suspended vm to ensure HW compatibility
4867 $forcemachine = $conf->{runningmachine};
4868 $forcecpu = $conf->{runningcpu};
4869 print "Resuming suspended VM\n";
4872 my ($cmd, $vollist, $spice_port) =
4873 config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4876 my $get_migration_ip = sub {
4877 my ($nodename) = @_;
4879 return $migration_ip if defined($migration_ip);
4881 my $cidr = $migrate_opts->{network};
4883 if (!defined($cidr)) {
4884 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
4885 $cidr = $dc_conf->{migration}->{network};
4888 if (defined($cidr)) {
4889 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4891 die "could not get IP: no address configured on local " .
4892 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4894 die "could not get IP: multiple addresses configured on local " .
4895 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4897 $migration_ip = @$ips[0];
4900 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4901 if !defined($migration_ip);
4903 return $migration_ip;
4908 if ($statefile eq 'tcp
') {
4909 my $localip = "localhost";
4910 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
4911 my $nodename = nodename();
4913 if (!defined($migration_type)) {
4914 if (defined($datacenterconf->{migration}->{type})) {
4915 $migration_type = $datacenterconf->{migration}->{type};
4917 $migration_type = 'secure
';
4921 if ($migration_type eq 'insecure
') {
4922 $localip = $get_migration_ip->($nodename);
4923 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4926 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4927 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4928 $migrate_uri = "tcp:${localip}:${migrate_port}";
4929 push @$cmd, '-incoming
', $migrate_uri;
4932 } elsif ($statefile eq 'unix
') {
4933 # should be default for secure migrations as a ssh TCP forward
4934 # tunnel is not deterministic reliable ready and fails regurarly
4935 # to set up in time, so use UNIX socket forwards
4936 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4937 unlink $socket_addr;
4939 $migrate_uri = "unix:$socket_addr";
4941 push @$cmd, '-incoming
', $migrate_uri;
4944 } elsif (-e $statefile) {
4945 push @$cmd, '-loadstate
', $statefile;
4947 my $statepath = PVE::Storage::path($storecfg, $statefile);
4948 push @$vollist, $statefile;
4949 push @$cmd, '-loadstate
', $statepath;
4951 } elsif ($params->{paused}) {
4956 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
4957 my $d = parse_hostpci($conf->{"hostpci$i"});
4959 my $pcidevices = $d->{pciid};
4960 foreach my $pcidevice (@$pcidevices) {
4961 my $pciid = $pcidevice->{id};
4963 my $info = PVE::SysFSTools::pci_device_info("$pciid");
4964 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4965 die "no pci device info for device '$pciid'\n" if !$info;
4968 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4969 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4971 die "can't unbind
/bind pci group to vfio
'$pciid'\n"
4972 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4973 die "can
't reset pci device '$pciid'\n"
4974 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4979 PVE::Storage::activate_volumes($storecfg, $vollist);
4982 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
4983 outfunc => sub {}, errfunc => sub {});
4985 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
4986 # timeout should be more than enough here...
4987 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4989 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4990 : $defaults->{cpuunits};
4992 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
4994 timeout => $statefile ? undef : $start_timeout,
4999 # when migrating, prefix QEMU output so other side can pick up any
5000 # errors that might occur and show the user
5001 if ($migratedfrom) {
5002 $run_params{quiet} = 1;
5003 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5007 Slice => 'qemu
.slice
',
5009 CPUShares => $cpuunits
5012 if (my $cpulimit = $conf->{cpulimit}) {
5013 $properties{CPUQuota} = int($cpulimit * 100);
5015 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5017 my $run_qemu = sub {
5018 PVE::Tools::run_fork sub {
5019 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5021 my $exitcode = run_command($cmd, %run_params);
5022 die "QEMU exited with code $exitcode\n" if $exitcode;
5026 if ($conf->{hugepages}) {
5029 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5030 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5032 PVE::QemuServer::Memory::hugepages_mount();
5033 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5035 eval { $run_qemu->() };
5037 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5041 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5043 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5046 eval { $run_qemu->() };
5050 # deactivate volumes if start fails
5051 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5052 die "start failed: $err";
5055 print "migration listens on $migrate_uri\n" if $migrate_uri;
5056 $res->{migrate_uri} = $migrate_uri;
5058 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5059 eval { mon_cmd($vmid, "cont"); };
5063 #start nbd server for storage migration
5064 if (my $nbd = $migrate_opts->{nbd}) {
5065 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5067 my $migrate_storage_uri;
5068 # nbd_protocol_version > 0 for unix socket support
5069 if ($nbd_protocol_version > 0 && $migration_type eq 'secure
') {
5070 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5071 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix
', data => { path => $socket_path } } );
5072 $migrate_storage_uri = "nbd:unix:$socket_path";
5074 my $nodename = nodename();
5075 my $localip = $get_migration_ip->($nodename);
5076 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5077 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5079 mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet
', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5080 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5081 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5084 $res->{migrate_storage_uri} = $migrate_storage_uri;
5086 foreach my $opt (sort keys %$nbd) {
5087 my $drivestr = $nbd->{$opt}->{drivestr};
5088 my $volid = $nbd->{$opt}->{volid};
5089 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5090 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5091 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5092 print "re-using replicated volume: $opt - $volid\n"
5093 if $nbd->{$opt}->{replicated};
5095 $res->{drives}->{$opt} = $nbd->{$opt};
5096 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5100 if ($migratedfrom) {
5102 set_migration_caps($vmid);
5107 print "spice listens on port $spice_port\n";
5108 $res->{spice_port} = $spice_port;
5109 if ($migrate_opts->{spice_ticket}) {
5110 mon_cmd($vmid, "set_password", protocol => 'spice
', password => $migrate_opts->{spice_ticket});
5111 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5116 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5117 if !$statefile && $conf->{balloon};
5119 foreach my $opt (keys %$conf) {
5120 next if $opt !~ m/^net\d+$/;
5121 my $nicconf = parse_net($conf->{$opt});
5122 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5126 mon_cmd($vmid, 'qom-set
',
5127 path => "machine/peripheral/balloon0",
5128 property => "guest-stats-polling-interval",
5129 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5132 print "Resumed VM, removing state\n";
5133 if (my $vmstate = $conf->{vmstate}) {
5134 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5135 PVE::Storage::vdisk_free($storecfg, $vmstate);
5137 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5138 PVE
::QemuConfig-
>write_config($vmid, $conf);
5141 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5146 sub vm_commandline
{
5147 my ($storecfg, $vmid, $snapname) = @_;
5149 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5154 my $snapshot = $conf->{snapshots
}->{$snapname};
5155 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5157 # check for machine or CPU overrides in snapshot
5158 $forcemachine = $snapshot->{runningmachine
};
5159 $forcecpu = $snapshot->{runningcpu
};
5161 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5166 my $defaults = load_defaults
();
5168 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults,
5169 $forcemachine, $forcecpu);
5171 return PVE
::Tools
::cmd2string
($cmd);
5175 my ($vmid, $skiplock) = @_;
5177 PVE
::QemuConfig-
>lock_config($vmid, sub {
5179 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5181 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5183 mon_cmd
($vmid, "system_reset");
5187 sub get_vm_volumes
{
5191 foreach_volid
($conf, sub {
5192 my ($volid, $attr) = @_;
5194 return if $volid =~ m
|^/|;
5196 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5199 push @$vollist, $volid;
5205 sub vm_stop_cleanup
{
5206 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5211 my $vollist = get_vm_volumes
($conf);
5212 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5215 foreach my $ext (qw(mon qmp pid vnc qga)) {
5216 unlink "/var/run/qemu-server/${vmid}.$ext";
5219 if ($conf->{ivshmem
}) {
5220 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5221 # just delete it for now, VMs which have this already open do not
5222 # are affected, but new VMs will get a separated one. If this
5223 # becomes an issue we either add some sort of ref-counting or just
5224 # add a "don't delete on stop" flag to the ivshmem format.
5225 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5228 foreach my $key (keys %$conf) {
5229 next if $key !~ m/^hostpci(\d+)$/;
5230 my $hostpciindex = $1;
5231 my $d = parse_hostpci
($conf->{$key});
5232 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5234 foreach my $pci (@{$d->{pciid
}}) {
5235 my $pciid = $pci->{id
};
5236 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5240 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5242 warn $@ if $@; # avoid errors - just warn
5245 # call only in locked context
5247 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5249 my $pid = check_running
($vmid, $nocheck);
5254 $conf = PVE
::QemuConfig-
>load_config($vmid);
5255 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5256 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5257 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5258 $timeout = $opts->{down
} if $opts->{down
};
5260 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5265 if (defined($conf) && parse_guest_agent
($conf)->{enabled
}) {
5266 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5268 mon_cmd
($vmid, "system_powerdown");
5271 mon_cmd
($vmid, "quit");
5277 $timeout = 60 if !defined($timeout);
5280 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5285 if ($count >= $timeout) {
5287 warn "VM still running - terminating now with SIGTERM\n";
5290 die "VM quit/powerdown failed - got timeout\n";
5293 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5297 if (!check_running
($vmid, $nocheck)) {
5298 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5302 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5305 die "VM quit/powerdown failed\n";
5313 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5318 if ($count >= $timeout) {
5319 warn "VM still running - terminating now with SIGKILL\n";
5324 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5327 # Note: use $nocheck to skip tests if VM configuration file exists.
5328 # We need that when migration VMs to other nodes (files already moved)
5329 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5331 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5333 $force = 1 if !defined($force) && !$shutdown;
5336 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5337 kill 15, $pid if $pid;
5338 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5339 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5343 PVE
::QemuConfig-
>lock_config($vmid, sub {
5344 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5349 my ($vmid, $timeout) = @_;
5351 PVE
::QemuConfig-
>lock_config($vmid, sub {
5354 # only reboot if running, as qmeventd starts it again on a stop event
5355 return if !check_running
($vmid);
5357 create_reboot_request
($vmid);
5359 my $storecfg = PVE
::Storage
::config
();
5360 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5364 # avoid that the next normal shutdown will be confused for a reboot
5365 clear_reboot_request
($vmid);
5371 # note: if using the statestorage parameter, the caller has to check privileges
5373 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5380 PVE
::QemuConfig-
>lock_config($vmid, sub {
5382 $conf = PVE
::QemuConfig-
>load_config($vmid);
5384 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5385 PVE
::QemuConfig-
>check_lock($conf)
5386 if !($skiplock || $is_backing_up);
5388 die "cannot suspend to disk during backup\n"
5389 if $is_backing_up && $includestate;
5391 if ($includestate) {
5392 $conf->{lock} = 'suspending';
5393 my $date = strftime
("%Y-%m-%d", localtime(time()));
5394 $storecfg = PVE
::Storage
::config
();
5395 if (!$statestorage) {
5396 $statestorage = find_vmstate_storage
($conf, $storecfg);
5397 # check permissions for the storage
5398 my $rpcenv = PVE
::RPCEnvironment
::get
();
5399 if ($rpcenv->{type
} ne 'cli') {
5400 my $authuser = $rpcenv->get_user();
5401 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5406 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5407 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5408 PVE
::QemuConfig-
>write_config($vmid, $conf);
5410 mon_cmd
($vmid, "stop");
5414 if ($includestate) {
5416 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5419 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5421 my $state = mon_cmd
($vmid, "query-savevm");
5422 if (!$state->{status
}) {
5423 die "savevm not active\n";
5424 } elsif ($state->{status
} eq 'active') {
5427 } elsif ($state->{status
} eq 'completed') {
5428 print "State saved, quitting\n";
5430 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5431 die "query-savevm failed with error '$state->{error}'\n"
5433 die "query-savevm returned status '$state->{status}'\n";
5439 PVE
::QemuConfig-
>lock_config($vmid, sub {
5440 $conf = PVE
::QemuConfig-
>load_config($vmid);
5442 # cleanup, but leave suspending lock, to indicate something went wrong
5444 mon_cmd
($vmid, "savevm-end");
5445 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5446 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5447 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5448 PVE
::QemuConfig-
>write_config($vmid, $conf);
5454 die "lock changed unexpectedly\n"
5455 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5457 mon_cmd
($vmid, "quit");
5458 $conf->{lock} = 'suspended';
5459 PVE
::QemuConfig-
>write_config($vmid, $conf);
5465 my ($vmid, $skiplock, $nocheck) = @_;
5467 PVE
::QemuConfig-
>lock_config($vmid, sub {
5468 my $res = mon_cmd
($vmid, 'query-status');
5469 my $resume_cmd = 'cont';
5471 if ($res->{status
} && $res->{status
} eq 'suspended') {
5472 $resume_cmd = 'system_wakeup';
5477 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5479 PVE
::QemuConfig-
>check_lock($conf)
5480 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5483 mon_cmd
($vmid, $resume_cmd);
5488 my ($vmid, $skiplock, $key) = @_;
5490 PVE
::QemuConfig-
>lock_config($vmid, sub {
5492 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5494 # there is no qmp command, so we use the human monitor command
5495 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5496 die $res if $res ne '';
5500 # vzdump restore implementaion
5502 sub tar_archive_read_firstfile
{
5503 my $archive = shift;
5505 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5507 # try to detect archive type first
5508 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5509 die "unable to open file '$archive'\n";
5510 my $firstfile = <$fh>;
5514 die "ERROR: archive contaions no data\n" if !$firstfile;
5520 sub tar_restore_cleanup
{
5521 my ($storecfg, $statfile) = @_;
5523 print STDERR
"starting cleanup\n";
5525 if (my $fd = IO
::File-
>new($statfile, "r")) {
5526 while (defined(my $line = <$fd>)) {
5527 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5530 if ($volid =~ m
|^/|) {
5531 unlink $volid || die 'unlink failed\n';
5533 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5535 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5537 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5539 print STDERR
"unable to parse line in statfile - $line";
5546 sub restore_file_archive
{
5547 my ($archive, $vmid, $user, $opts) = @_;
5549 return restore_vma_archive
($archive, $vmid, $user, $opts)
5552 my $info = PVE
::Storage
::archive_info
($archive);
5553 my $format = $opts->{format
} // $info->{format
};
5554 my $comp = $info->{compression
};
5556 # try to detect archive format
5557 if ($format eq 'tar') {
5558 return restore_tar_archive
($archive, $vmid, $user, $opts);
5560 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5564 # hepler to remove disks that will not be used after restore
5565 my $restore_cleanup_oldconf = sub {
5566 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5568 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
5569 my ($ds, $drive) = @_;
5571 return if drive_is_cdrom
($drive, 1);
5573 my $volid = $drive->{file
};
5574 return if !$volid || $volid =~ m
|^/|;
5576 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
5577 return if !$path || !$owner || ($owner != $vmid);
5579 # Note: only delete disk we want to restore
5580 # other volumes will become unused
5581 if ($virtdev_hash->{$ds}) {
5582 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
5589 # delete vmstate files, after the restore we have no snapshots anymore
5590 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5591 my $snap = $oldconf->{snapshots
}->{$snapname};
5592 if ($snap->{vmstate
}) {
5593 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
5601 # Helper to parse vzdump backup device hints
5603 # $rpcenv: Environment, used to ckeck storage permissions
5604 # $user: User ID, to check storage permissions
5605 # $storecfg: Storage configuration
5606 # $fh: the file handle for reading the configuration
5607 # $devinfo: should contain device sizes for all backu-up'ed devices
5608 # $options: backup options (pool, default storage)
5610 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5611 my $parse_backup_hints = sub {
5612 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5614 my $virtdev_hash = {};
5616 while (defined(my $line = <$fh>)) {
5617 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5618 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5619 die "archive does not contain data for drive '$virtdev'\n"
5620 if !$devinfo->{$devname};
5622 if (defined($options->{storage
})) {
5623 $storeid = $options->{storage
} || 'local';
5624 } elsif (!$storeid) {
5627 $format = 'raw' if !$format;
5628 $devinfo->{$devname}->{devname
} = $devname;
5629 $devinfo->{$devname}->{virtdev
} = $virtdev;
5630 $devinfo->{$devname}->{format
} = $format;
5631 $devinfo->{$devname}->{storeid
} = $storeid;
5633 # check permission on storage
5634 my $pool = $options->{pool
}; # todo: do we need that?
5635 if ($user ne 'root@pam') {
5636 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5639 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5640 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5642 my $drive = parse_drive
($virtdev, $2);
5643 if (drive_is_cloudinit
($drive)) {
5644 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
5645 $storeid = $options->{storage
} if defined ($options->{storage
});
5646 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5647 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
5649 $virtdev_hash->{$virtdev} = {
5651 storeid
=> $storeid,
5652 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
5659 return $virtdev_hash;
5662 # Helper to allocate and activate all volumes required for a restore
5664 # $storecfg: Storage configuration
5665 # $virtdev_hash: as returned by parse_backup_hints()
5667 # Returns: { $virtdev => $volid }
5668 my $restore_allocate_devices = sub {
5669 my ($storecfg, $virtdev_hash, $vmid) = @_;
5672 foreach my $virtdev (sort keys %$virtdev_hash) {
5673 my $d = $virtdev_hash->{$virtdev};
5674 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5675 my $storeid = $d->{storeid
};
5676 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5678 # test if requested format is supported
5679 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5680 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
5681 $d->{format
} = $defFormat if !$supported;
5684 if ($d->{is_cloudinit
}) {
5685 $name = "vm-$vmid-cloudinit";
5686 $name .= ".$d->{format}" if $d->{format
} ne 'raw';
5689 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
5691 print STDERR
"new volume ID is '$volid'\n";
5692 $d->{volid
} = $volid;
5694 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
5696 $map->{$virtdev} = $volid;
5702 my $restore_update_config_line = sub {
5703 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5705 return if $line =~ m/^\#qmdump\#/;
5706 return if $line =~ m/^\#vzdump\#/;
5707 return if $line =~ m/^lock:/;
5708 return if $line =~ m/^unused\d+:/;
5709 return if $line =~ m/^parent:/;
5711 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5712 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5713 # try to convert old 1.X settings
5714 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5715 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5716 my ($model, $macaddr) = split(/\=/, $devconfig);
5717 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5720 bridge
=> "vmbr$ind",
5721 macaddr
=> $macaddr,
5723 my $netstr = print_net
($net);
5725 print $outfd "net$cookie->{netcount}: $netstr\n";
5726 $cookie->{netcount
}++;
5728 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5729 my ($id, $netstr) = ($1, $2);
5730 my $net = parse_net
($netstr);
5731 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5732 $netstr = print_net
($net);
5733 print $outfd "$id: $netstr\n";
5734 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5737 my $di = parse_drive
($virtdev, $value);
5738 if (defined($di->{backup
}) && !$di->{backup
}) {
5739 print $outfd "#$line";
5740 } elsif ($map->{$virtdev}) {
5741 delete $di->{format
}; # format can change on restore
5742 $di->{file
} = $map->{$virtdev};
5743 $value = print_drive
($di);
5744 print $outfd "$virtdev: $value\n";
5748 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5750 if ($vmgenid ne '0') {
5751 # always generate a new vmgenid if there was a valid one setup
5752 $vmgenid = generate_uuid
();
5754 print $outfd "vmgenid: $vmgenid\n";
5755 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5756 my ($uuid, $uuid_str);
5757 UUID
::generate
($uuid);
5758 UUID
::unparse
($uuid, $uuid_str);
5759 my $smbios1 = parse_smbios1
($2);
5760 $smbios1->{uuid
} = $uuid_str;
5761 print $outfd $1.print_smbios1
($smbios1)."\n";
5767 my $restore_deactivate_volumes = sub {
5768 my ($storecfg, $devinfo) = @_;
5771 foreach my $devname (keys %$devinfo) {
5772 my $volid = $devinfo->{$devname}->{volid
};
5773 push @$vollist, $volid if $volid;
5776 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5779 my $restore_destroy_volumes = sub {
5780 my ($storecfg, $devinfo) = @_;
5782 foreach my $devname (keys %$devinfo) {
5783 my $volid = $devinfo->{$devname}->{volid
};
5786 if ($volid =~ m
|^/|) {
5787 unlink $volid || die 'unlink failed\n';
5789 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5791 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5793 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5798 my ($cfg, $vmid) = @_;
5800 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5802 my $volid_hash = {};
5803 foreach my $storeid (keys %$info) {
5804 foreach my $item (@{$info->{$storeid}}) {
5805 next if !($item->{volid
} && $item->{size
});
5806 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5807 $volid_hash->{$item->{volid
}} = $item;
5814 sub update_disk_config
{
5815 my ($vmid, $conf, $volid_hash) = @_;
5818 my $prefix = "VM $vmid";
5820 # used and unused disks
5821 my $referenced = {};
5823 # Note: it is allowed to define multiple storages with same path (alias), so
5824 # we need to check both 'volid' and real 'path' (two different volid can point
5825 # to the same path).
5827 my $referencedpath = {};
5830 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5831 my ($opt, $drive) = @_;
5833 my $volid = $drive->{file
};
5836 # mark volid as "in-use" for next step
5837 $referenced->{$volid} = 1;
5838 if ($volid_hash->{$volid} &&
5839 (my $path = $volid_hash->{$volid}->{path
})) {
5840 $referencedpath->{$path} = 1;
5843 return if drive_is_cdrom
($drive);
5844 return if !$volid_hash->{$volid};
5846 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volid_hash->{$volid}->{size
});
5847 if (defined($updated)) {
5849 $conf->{$opt} = print_drive
($updated);
5850 print "$prefix ($opt): $msg\n";
5854 # remove 'unusedX' entry if volume is used
5855 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
5856 my ($opt, $drive) = @_;
5858 my $volid = $drive->{file
};
5861 my $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
5862 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5863 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5865 delete $conf->{$opt};
5868 $referenced->{$volid} = 1;
5869 $referencedpath->{$path} = 1 if $path;
5872 foreach my $volid (sort keys %$volid_hash) {
5873 next if $volid =~ m/vm-$vmid-state-/;
5874 next if $referenced->{$volid};
5875 my $path = $volid_hash->{$volid}->{path
};
5876 next if !$path; # just to be sure
5877 next if $referencedpath->{$path};
5879 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
5880 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5881 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5888 my ($vmid, $nolock, $dryrun) = @_;
5890 my $cfg = PVE
::Storage
::config
();
5892 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5893 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5894 foreach my $stor (keys %{$cfg->{ids
}}) {
5895 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
5898 print "rescan volumes...\n";
5899 my $volid_hash = scan_volids
($cfg, $vmid);
5901 my $updatefn = sub {
5904 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5906 PVE
::QemuConfig-
>check_lock($conf);
5909 foreach my $volid (keys %$volid_hash) {
5910 my $info = $volid_hash->{$volid};
5911 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
5914 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
5916 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
5919 if (defined($vmid)) {
5923 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
5926 my $vmlist = config_list
();
5927 foreach my $vmid (keys %$vmlist) {
5931 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
5937 sub restore_proxmox_backup_archive
{
5938 my ($archive, $vmid, $user, $options) = @_;
5940 my $storecfg = PVE
::Storage
::config
();
5942 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
5943 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5945 my $server = $scfg->{server
};
5946 my $datastore = $scfg->{datastore
};
5947 my $username = $scfg->{username
} // 'root@pam';
5948 my $fingerprint = $scfg->{fingerprint
};
5949 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
5951 my $repo = "$username\@$server:$datastore";
5953 # This is only used for `pbs-restore`!
5954 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
5955 local $ENV{PBS_PASSWORD
} = $password;
5956 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
5958 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
5959 PVE
::Storage
::parse_volname
($storecfg, $archive);
5961 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
5963 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
5965 my $tmpdir = "/var/tmp/vzdumptmp$$";
5969 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
5970 my $tmpfn = "$conffile.$$.tmp";
5971 # disable interrupts (always do cleanups)
5975 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
5977 # Note: $oldconf is undef if VM does not exists
5978 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
5979 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
5981 my $rpcenv = PVE
::RPCEnvironment
::get
();
5990 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
5992 my $cfgfn = "$tmpdir/qemu-server.conf";
5993 my $firewall_config_fn = "$tmpdir/fw.conf";
5994 my $index_fn = "$tmpdir/index.json";
5996 my $cmd = "restore";
5998 my $param = [$pbs_backup_name, "index.json", $index_fn];
5999 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6000 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6001 $index = decode_json
($index);
6003 # print Dumper($index);
6004 foreach my $info (@{$index->{files
}}) {
6005 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6007 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6008 $devinfo->{$devname}->{size
} = $1;
6010 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6015 my $is_qemu_server_backup = scalar(grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}});
6016 if (!$is_qemu_server_backup) {
6017 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6019 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6021 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6022 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6024 if ($has_firewall_config) {
6025 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6026 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6028 my $pve_firewall_dir = '/etc/pve/firewall';
6029 mkdir $pve_firewall_dir; # make sure the dir exists
6030 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6033 my $fh = IO
::File-
>new($cfgfn, "r") ||
6034 die "unable to read qemu-server.conf - $!\n";
6036 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6038 # fixme: rate limit?
6040 # create empty/temp config
6041 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6043 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6046 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6048 foreach my $virtdev (sort keys %$virtdev_hash) {
6049 my $d = $virtdev_hash->{$virtdev};
6050 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6052 my $volid = $d->{volid
};
6054 my $path = PVE
::Storage
::path
($storecfg, $volid);
6056 # This is the ONLY user of the PBS_ env vars set on top of this function!
6057 my $pbs_restore_cmd = [
6058 '/usr/bin/pbs-restore',
6059 '--repository', $repo,
6061 "$d->{devname}.img.fidx",
6066 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6067 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6069 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6070 push @$pbs_restore_cmd, '--skip-zero';
6073 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6074 print "restore proxmox backup image: $dbg_cmdstring\n";
6075 run_command
($pbs_restore_cmd);
6078 $fh->seek(0, 0) || die "seek failed - $!\n";
6080 my $outfd = new IO
::File
($tmpfn, "w") ||
6081 die "unable to write config for VM $vmid\n";
6083 my $cookie = { netcount
=> 0 };
6084 while (defined(my $line = <$fh>)) {
6085 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique
});
6093 $restore_deactivate_volumes->($storecfg, $devinfo);
6099 $restore_destroy_volumes->($storecfg, $devinfo);
6103 rename($tmpfn, $conffile) ||
6104 die "unable to commit configuration file '$conffile'\n";
6106 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6108 eval { rescan
($vmid, 1); };
6112 sub restore_vma_archive
{
6113 my ($archive, $vmid, $user, $opts, $comp) = @_;
6115 my $readfrom = $archive;
6117 my $cfg = PVE
::Storage
::config
();
6119 my $bwlimit = $opts->{bwlimit
};
6121 my $dbg_cmdstring = '';
6122 my $add_pipe = sub {
6124 push @$commands, $cmd;
6125 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6126 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6131 if ($archive eq '-') {
6134 # If we use a backup from a PVE defined storage we also consider that
6135 # storage's rate limit:
6136 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6137 if (defined($volid)) {
6138 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6139 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6141 print STDERR
"applying read rate limit: $readlimit\n";
6142 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6143 $add_pipe->($cstream);
6149 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6150 my $cmd = $info->{decompressor
};
6151 push @$cmd, $readfrom;
6155 my $tmpdir = "/var/tmp/vzdumptmp$$";
6158 # disable interrupts (always do cleanups)
6162 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6164 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6165 POSIX
::mkfifo
($mapfifo, 0600);
6168 my $openfifo = sub {
6169 open($fifofh, '>', $mapfifo) || die $!;
6172 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6179 my $rpcenv = PVE
::RPCEnvironment
::get
();
6181 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6182 my $tmpfn = "$conffile.$$.tmp";
6184 # Note: $oldconf is undef if VM does not exist
6185 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6186 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6190 my $print_devmap = sub {
6191 my $cfgfn = "$tmpdir/qemu-server.conf";
6193 # we can read the config - that is already extracted
6194 my $fh = IO
::File-
>new($cfgfn, "r") ||
6195 die "unable to read qemu-server.conf - $!\n";
6197 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6199 my $pve_firewall_dir = '/etc/pve/firewall';
6200 mkdir $pve_firewall_dir; # make sure the dir exists
6201 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6204 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6206 foreach my $key (keys %storage_limits) {
6207 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6209 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6210 $storage_limits{$key} = $limit * 1024;
6213 foreach my $devname (keys %$devinfo) {
6214 die "found no device mapping information for device '$devname'\n"
6215 if !$devinfo->{$devname}->{virtdev
};
6218 # create empty/temp config
6220 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6221 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6225 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6227 # print restore information to $fifofh
6228 foreach my $virtdev (sort keys %$virtdev_hash) {
6229 my $d = $virtdev_hash->{$virtdev};
6230 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6232 my $storeid = $d->{storeid
};
6233 my $volid = $d->{volid
};
6236 if (my $limit = $storage_limits{$storeid}) {
6237 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6240 my $write_zeros = 1;
6241 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6245 my $path = PVE
::Storage
::path
($cfg, $volid);
6247 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6249 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6252 $fh->seek(0, 0) || die "seek failed - $!\n";
6254 my $outfd = new IO
::File
($tmpfn, "w") ||
6255 die "unable to write config for VM $vmid\n";
6257 my $cookie = { netcount
=> 0 };
6258 while (defined(my $line = <$fh>)) {
6259 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6272 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6273 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6275 $oldtimeout = alarm($timeout);
6282 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6283 my ($dev_id, $size, $devname) = ($1, $2, $3);
6284 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6285 } elsif ($line =~ m/^CTIME: /) {
6286 # we correctly received the vma config, so we can disable
6287 # the timeout now for disk allocation (set to 10 minutes, so
6288 # that we always timeout if something goes wrong)
6291 print $fifofh "done\n";
6292 my $tmp = $oldtimeout || 0;
6293 $oldtimeout = undef;
6299 print "restore vma archive: $dbg_cmdstring\n";
6300 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6304 alarm($oldtimeout) if $oldtimeout;
6306 $restore_deactivate_volumes->($cfg, $devinfo);
6313 $restore_destroy_volumes->($cfg, $devinfo);
6317 rename($tmpfn, $conffile) ||
6318 die "unable to commit configuration file '$conffile'\n";
6320 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6322 eval { rescan
($vmid, 1); };
6326 sub restore_tar_archive
{
6327 my ($archive, $vmid, $user, $opts) = @_;
6329 if ($archive ne '-') {
6330 my $firstfile = tar_archive_read_firstfile
($archive);
6331 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6332 if $firstfile ne 'qemu-server.conf';
6335 my $storecfg = PVE
::Storage
::config
();
6337 # avoid zombie disks when restoring over an existing VM -> cleanup first
6338 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6339 # skiplock=1 because qmrestore has set the 'create' lock itself already
6340 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6341 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6343 my $tocmd = "/usr/lib/qemu-server/qmextract";
6345 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6346 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6347 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6348 $tocmd .= ' --info' if $opts->{info
};
6350 # tar option "xf" does not autodetect compression when read from STDIN,
6351 # so we pipe to zcat
6352 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6353 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6355 my $tmpdir = "/var/tmp/vzdumptmp$$";
6358 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6359 local $ENV{VZDUMP_VMID
} = $vmid;
6360 local $ENV{VZDUMP_USER
} = $user;
6362 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6363 my $tmpfn = "$conffile.$$.tmp";
6365 # disable interrupts (always do cleanups)
6369 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6377 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6379 if ($archive eq '-') {
6380 print "extracting archive from STDIN\n";
6381 run_command
($cmd, input
=> "<&STDIN");
6383 print "extracting archive '$archive'\n";
6387 return if $opts->{info
};
6391 my $statfile = "$tmpdir/qmrestore.stat";
6392 if (my $fd = IO
::File-
>new($statfile, "r")) {
6393 while (defined (my $line = <$fd>)) {
6394 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6395 $map->{$1} = $2 if $1;
6397 print STDERR
"unable to parse line in statfile - $line\n";
6403 my $confsrc = "$tmpdir/qemu-server.conf";
6405 my $srcfd = new IO
::File
($confsrc, "r") ||
6406 die "unable to open file '$confsrc'\n";
6408 my $outfd = new IO
::File
($tmpfn, "w") ||
6409 die "unable to write config for VM $vmid\n";
6411 my $cookie = { netcount
=> 0 };
6412 while (defined (my $line = <$srcfd>)) {
6413 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6421 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6427 rename $tmpfn, $conffile ||
6428 die "unable to commit configuration file '$conffile'\n";
6430 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6432 eval { rescan
($vmid, 1); };
6436 sub foreach_storage_used_by_vm
{
6437 my ($conf, $func) = @_;
6441 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6442 my ($ds, $drive) = @_;
6443 return if drive_is_cdrom
($drive);
6445 my $volid = $drive->{file
};
6447 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6448 $sidhash->{$sid} = $sid if $sid;
6451 foreach my $sid (sort keys %$sidhash) {
6456 my $qemu_snap_storage = {
6459 sub do_snapshots_with_qemu
{
6460 my ($storecfg, $volid) = @_;
6462 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6463 my $scfg = $storecfg->{ids
}->{$storage_name};
6465 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6469 if ($volid =~ m/\.(qcow2|qed)$/){
6476 sub qga_check_running
{
6477 my ($vmid, $nowarn) = @_;
6479 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6481 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6487 sub template_create
{
6488 my ($vmid, $conf, $disk) = @_;
6490 my $storecfg = PVE
::Storage
::config
();
6492 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6493 my ($ds, $drive) = @_;
6495 return if drive_is_cdrom
($drive);
6496 return if $disk && $ds ne $disk;
6498 my $volid = $drive->{file
};
6499 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6501 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6502 $drive->{file
} = $voliddst;
6503 $conf->{$ds} = print_drive
($drive);
6504 PVE
::QemuConfig-
>write_config($vmid, $conf);
6508 sub convert_iscsi_path
{
6511 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6516 my $initiator_name = get_initiator_name
();
6518 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6519 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6522 die "cannot convert iscsi path '$path', unkown format\n";
6525 sub qemu_img_convert
{
6526 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6528 my $storecfg = PVE
::Storage
::config
();
6529 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6530 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6532 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6536 my $src_is_iscsi = 0;
6540 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6541 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6542 $src_format = qemu_img_format
($src_scfg, $src_volname);
6543 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6544 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6545 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6546 } elsif (-f
$src_volid) {
6547 $src_path = $src_volid;
6548 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6553 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6555 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6556 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6557 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6558 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6561 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6562 push @$cmd, '-l', "snapshot.name=$snapname"
6563 if $snapname && $src_format && $src_format eq "qcow2";
6564 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6565 push @$cmd, '-T', $cachemode if defined($cachemode);
6567 if ($src_is_iscsi) {
6568 push @$cmd, '--image-opts';
6569 $src_path = convert_iscsi_path
($src_path);
6570 } elsif ($src_format) {
6571 push @$cmd, '-f', $src_format;
6574 if ($dst_is_iscsi) {
6575 push @$cmd, '--target-image-opts';
6576 $dst_path = convert_iscsi_path
($dst_path);
6578 push @$cmd, '-O', $dst_format;
6581 push @$cmd, $src_path;
6583 if (!$dst_is_iscsi && $is_zero_initialized) {
6584 push @$cmd, "zeroinit:$dst_path";
6586 push @$cmd, $dst_path;
6591 if($line =~ m/\((\S+)\/100\
%\)/){
6593 my $transferred = int($size * $percent / 100);
6594 my $remaining = $size - $transferred;
6596 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6601 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6603 die "copy failed: $err" if $err;
6606 sub qemu_img_format
{
6607 my ($scfg, $volname) = @_;
6609 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6616 sub qemu_drive_mirror
{
6617 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6619 $jobs = {} if !$jobs;
6623 $jobs->{"drive-$drive"} = {};
6625 if ($dst_volid =~ /^nbd:/) {
6626 $qemu_target = $dst_volid;
6629 my $storecfg = PVE
::Storage
::config
();
6630 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6632 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6634 $format = qemu_img_format
($dst_scfg, $dst_volname);
6636 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6638 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6641 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6642 $opts->{format
} = $format if $format;
6644 if (defined($src_bitmap)) {
6645 $opts->{sync
} = 'incremental';
6646 $opts->{bitmap
} = $src_bitmap;
6647 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6650 if (defined($bwlimit)) {
6651 $opts->{speed
} = $bwlimit * 1024;
6652 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6654 print "drive mirror is starting for drive-$drive\n";
6657 # if a job already runs for this device we get an error, catch it for cleanup
6658 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
6660 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6662 die "mirroring error: $err\n";
6665 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
6668 # $completion can be either
6669 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6670 # 'cancel': wait until all jobs are ready, block-job-cancel them
6671 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6672 sub qemu_drive_mirror_monitor
{
6673 my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6675 $completion //= 'complete';
6678 my $err_complete = 0;
6681 die "storage migration timed out\n" if $err_complete > 300;
6683 my $stats = mon_cmd
($vmid, "query-block-jobs");
6685 my $running_mirror_jobs = {};
6686 foreach my $stat (@$stats) {
6687 next if $stat->{type
} ne 'mirror';
6688 $running_mirror_jobs->{$stat->{device
}} = $stat;
6691 my $readycounter = 0;
6693 foreach my $job (keys %$jobs) {
6695 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6696 print "$job : finished\n";
6697 delete $jobs->{$job};
6701 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6703 my $busy = $running_mirror_jobs->{$job}->{busy
};
6704 my $ready = $running_mirror_jobs->{$job}->{ready
};
6705 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6706 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6707 my $remaining = $total - $transferred;
6708 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6710 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6713 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6716 last if scalar(keys %$jobs) == 0;
6718 if ($readycounter == scalar(keys %$jobs)) {
6719 print "all mirroring jobs are ready \n";
6720 last if $completion eq 'skip'; #do the complete later
6722 if ($vmiddst && $vmiddst != $vmid) {
6723 my $agent_running = $qga && qga_check_running
($vmid);
6724 if ($agent_running) {
6725 print "freeze filesystem\n";
6726 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6728 print "suspend vm\n";
6729 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6732 # if we clone a disk for a new target vm, we don't switch the disk
6733 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6735 if ($agent_running) {
6736 print "unfreeze filesystem\n";
6737 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6739 print "resume vm\n";
6740 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6746 foreach my $job (keys %$jobs) {
6747 # try to switch the disk if source and destination are on the same guest
6748 print "$job: Completing block job...\n";
6751 if ($completion eq 'complete') {
6752 $op = 'block-job-complete';
6753 } elsif ($completion eq 'cancel') {
6754 $op = 'block-job-cancel';
6756 die "invalid completion value: $completion\n";
6758 eval { mon_cmd
($vmid, $op, device
=> $job) };
6759 if ($@ =~ m/cannot be completed/) {
6760 print "$job: Block job cannot be completed, try again.\n";
6763 print "$job: Completed successfully.\n";
6764 $jobs->{$job}->{complete
} = 1;
6775 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6776 die "mirroring error: $err";
6781 sub qemu_blockjobs_cancel
{
6782 my ($vmid, $jobs) = @_;
6784 foreach my $job (keys %$jobs) {
6785 print "$job: Cancelling block job\n";
6786 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6787 $jobs->{$job}->{cancel
} = 1;
6791 my $stats = mon_cmd
($vmid, "query-block-jobs");
6793 my $running_jobs = {};
6794 foreach my $stat (@$stats) {
6795 $running_jobs->{$stat->{device
}} = $stat;
6798 foreach my $job (keys %$jobs) {
6800 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6801 print "$job: Done.\n";
6802 delete $jobs->{$job};
6806 last if scalar(keys %$jobs) == 0;
6813 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6814 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6819 print "create linked clone of drive $drivename ($drive->{file})\n";
6820 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6821 push @$newvollist, $newvolid;
6824 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6825 $storeid = $storage if $storage;
6827 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6828 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
6830 print "create full clone of drive $drivename ($drive->{file})\n";
6832 if (drive_is_cloudinit
($drive)) {
6833 $name = "vm-$newvmid-cloudinit";
6834 $name .= ".$dst_format" if $dst_format ne 'raw';
6836 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
6837 } elsif ($drivename eq 'efidisk0') {
6838 $size = get_efivars_size
($conf);
6840 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6841 push @$newvollist, $newvolid;
6843 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
6845 if (drive_is_cloudinit
($drive)) {
6849 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6850 if (!$running || $snapname) {
6851 # TODO: handle bwlimits
6852 if ($drivename eq 'efidisk0') {
6853 # the relevant data on the efidisk may be smaller than the source
6854 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6855 # that is given by the OVMF_VARS.fd
6856 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
6857 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
6858 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
6860 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
6864 my $kvmver = get_running_qemu_version
($vmid);
6865 if (!min_version
($kvmver, 2, 7)) {
6866 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6867 if $drive->{iothread
};
6870 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
6875 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
6878 $disk->{format
} = undef;
6879 $disk->{file
} = $newvolid;
6880 $disk->{size
} = $size;
6885 sub get_running_qemu_version
{
6887 my $res = mon_cmd
($vmid, "query-version");
6888 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6891 sub qemu_use_old_bios_files
{
6892 my ($machine_type) = @_;
6894 return if !$machine_type;
6896 my $use_old_bios_files = undef;
6898 if ($machine_type =~ m/^(\S+)\.pxe$/) {
6900 $use_old_bios_files = 1;
6902 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, kvm_user_version
());
6903 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6904 # load new efi bios files on migration. So this hack is required to allow
6905 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6906 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6907 $use_old_bios_files = !min_version
($version, 2, 4);
6910 return ($use_old_bios_files, $machine_type);
6913 sub get_efivars_size
{
6915 my $arch = get_vm_arch
($conf);
6916 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
6917 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
6918 return -s
$ovmf_vars;
6921 sub update_efidisk_size
{
6924 return if !defined($conf->{efidisk0
});
6926 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
6927 $disk->{size
} = get_efivars_size
($conf);
6928 $conf->{efidisk0
} = print_drive
($disk);
6933 sub create_efidisk
($$$$$) {
6934 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6936 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
6937 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
6939 my $vars_size_b = -s
$ovmf_vars;
6940 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
6941 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6942 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6944 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
6945 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
6947 return ($volid, $size/1024);
6950 sub vm_iothreads_list
{
6953 my $res = mon_cmd
($vmid, 'query-iothreads');
6956 foreach my $iothread (@$res) {
6957 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
6964 my ($conf, $drive) = @_;
6968 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
6970 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
6976 my $controller = int($drive->{index} / $maxdev);
6977 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single') ?
"virtioscsi" : "scsihw";
6979 return ($maxdev, $controller, $controller_prefix);
6982 sub windows_version
{
6985 return 0 if !$ostype;
6989 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6991 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6993 } elsif ($ostype =~ m/^win(\d+)$/) {
7000 sub resolve_dst_disk_format
{
7001 my ($storecfg, $storeid, $src_volname, $format) = @_;
7002 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7005 # if no target format is specified, use the source disk format as hint
7007 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7008 $format = qemu_img_format
($scfg, $src_volname);
7014 # test if requested format is supported - else use default
7015 my $supported = grep { $_ eq $format } @$validFormats;
7016 $format = $defFormat if !$supported;
7020 # NOTE: if this logic changes, please update docs & possibly gui logic
7021 sub find_vmstate_storage
{
7022 my ($conf, $storecfg) = @_;
7024 # first, return storage from conf if set
7025 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7027 my ($target, $shared, $local);
7029 foreach_storage_used_by_vm
($conf, sub {
7031 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7032 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7033 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7036 # second, use shared storage where VM has at least one disk
7037 # third, use local storage where VM has at least one disk
7038 # fall back to local storage
7039 $target = $shared // $local // 'local';
7045 my ($uuid, $uuid_str);
7046 UUID
::generate
($uuid);
7047 UUID
::unparse
($uuid, $uuid_str);
7051 sub generate_smbios1_uuid
{
7052 return "uuid=".generate_uuid
();
7058 mon_cmd
($vmid, 'nbd-server-stop');
7061 sub create_reboot_request
{
7063 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7064 or die "failed to create reboot trigger file: $!\n";
7068 sub clear_reboot_request
{
7070 my $path = "/run/qemu-server/$vmid.reboot";
7073 $res = unlink($path);
7074 die "could not remove reboot request for $vmid: $!"
7075 if !$res && $! != POSIX
::ENOENT
;
7080 # bash completion helper
7082 sub complete_backup_archives
{
7083 my ($cmdname, $pname, $cvalue) = @_;
7085 my $cfg = PVE
::Storage
::config
();
7089 if ($cvalue =~ m/^([^:]+):/) {
7093 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7096 foreach my $id (keys %$data) {
7097 foreach my $item (@{$data->{$id}}) {
7098 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7099 push @$res, $item->{volid
} if defined($item->{volid
});
7106 my $complete_vmid_full = sub {
7109 my $idlist = vmstatus
();
7113 foreach my $id (keys %$idlist) {
7114 my $d = $idlist->{$id};
7115 if (defined($running)) {
7116 next if $d->{template
};
7117 next if $running && $d->{status
} ne 'running';
7118 next if !$running && $d->{status
} eq 'running';
7127 return &$complete_vmid_full();
7130 sub complete_vmid_stopped
{
7131 return &$complete_vmid_full(0);
7134 sub complete_vmid_running
{
7135 return &$complete_vmid_full(1);
7138 sub complete_storage
{
7140 my $cfg = PVE
::Storage
::config
();
7141 my $ids = $cfg->{ids
};
7144 foreach my $sid (keys %$ids) {
7145 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7146 next if !$ids->{$sid}->{content
}->{images
};
7153 sub complete_migration_storage
{
7154 my ($cmd, $param, $current_value, $all_args) = @_;
7156 my $targetnode = @$all_args[1];
7158 my $cfg = PVE
::Storage
::config
();
7159 my $ids = $cfg->{ids
};
7162 foreach my $sid (keys %$ids) {
7163 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7164 next if !$ids->{$sid}->{content
}->{images
};