1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
30 use PVE
::DataCenterConfig
;
31 use PVE
::Exception
qw(raise raise_param_exc);
32 use PVE
::GuestHelpers
;
34 use PVE
::JSONSchema
qw(get_standard_option);
36 use PVE
::RPCEnvironment
;
40 use PVE
::Tools
qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
44 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
45 use PVE
::QemuServer
::Cloudinit
;
46 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
47 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive foreach_drive foreach_volid);
48 use PVE
::QemuServer
::Machine
;
49 use PVE
::QemuServer
::Memory
;
50 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
51 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port);
52 use PVE
::QemuServer
::USB
qw(parse_usb_device);
54 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
57 "$EDK2_FW_BASE/OVMF_CODE.fd",
58 "$EDK2_FW_BASE/OVMF_VARS.fd"
61 "$EDK2_FW_BASE/AAVMF_CODE.fd",
62 "$EDK2_FW_BASE/AAVMF_VARS.fd"
66 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
68 # Note about locking: we use flock on the config file protect
69 # against concurent actions.
70 # Aditionaly, we have a 'lock' setting in the config file. This
71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
72 # allowed when such lock is set. But you can ignore this kind of
73 # lock with the --skiplock flag.
75 cfs_register_file
('/qemu-server/',
79 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
80 description
=> "Some command save/restore state from this location.",
86 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
87 description
=> "Specifies the Qemu machine type.",
89 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
94 #no warnings 'redefine';
97 my ($controller, $vmid, $option, $value) = @_;
99 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
100 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
106 $nodename_cache //= PVE
::INotify
::nodename
();
107 return $nodename_cache;
114 enum
=> [qw(i6300esb ib700)],
115 description
=> "Watchdog type to emulate.",
116 default => 'i6300esb',
121 enum
=> [qw(reset shutdown poweroff pause debug none)],
122 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
126 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
130 description
=> "Enable/disable Qemu GuestAgent.",
135 fstrim_cloned_disks
=> {
136 description
=> "Run fstrim after cloning/moving a disk.",
142 description
=> "Select the agent type",
146 enum
=> [qw(virtio isa)],
152 description
=> "Select the VGA type.",
157 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
160 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
172 description
=> "The size of the file in MB.",
176 pattern
=> '[a-zA-Z0-9\-]+',
178 format_description
=> 'string',
179 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
186 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
187 description
=> "Configure an audio device."
194 description
=> "Driver backend for the audio device."
198 my $spice_enhancements_fmt = {
203 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
207 enum
=> ['off', 'all', 'filter'],
210 description
=> "Enable video streaming. Uses compression for detected video streams."
217 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
219 description
=> "The file on the host to gather entropy from. In most"
220 . " cases /dev/urandom should be preferred over /dev/random"
221 . " to avoid entropy-starvation issues on the host. Using"
222 . " urandom does *not* decrease security in any meaningful"
223 . " way, as it's still seeded from real entropy, and the"
224 . " bytes provided will most likely be mixed with real"
225 . " entropy on the guest as well. /dev/hwrng can be used"
226 . " to pass through a hardware RNG from the host.",
230 description
=> "Maximum bytes of entropy injected into the guest every"
231 . " 'period' milliseconds. Prefer a lower value when using"
232 . " /dev/random as source. Use 0 to disable limiting"
233 . " (potentially dangerous!).",
236 # default is 1 KiB/s, provides enough entropy to the guest to avoid
237 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
238 # of overwhelming the host, provided we're reading from /dev/urandom
243 description
=> "Every 'period' milliseconds the entropy-injection quota"
244 . " is reset, allowing the guest to retrieve another"
245 . " 'max_bytes' of entropy.",
255 description
=> "Specifies whether a VM will be started during system bootup.",
261 description
=> "Automatic restart after crash (currently ignored).",
266 type
=> 'string', format
=> 'pve-hotplug-features',
267 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
268 default => 'network,disk,usb',
273 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
279 description
=> "Lock/unlock the VM.",
280 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
285 description
=> "Limit of CPU usage.",
286 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
294 description
=> "CPU weight for a VM.",
295 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
303 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
310 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
316 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
324 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
325 "It should not be necessary to set it.",
326 enum
=> PVE
::Tools
::kvmkeymaplist
(),
331 type
=> 'string', format
=> 'dns-name',
332 description
=> "Set a name for the VM. Only used on the configuration web interface.",
337 description
=> "SCSI controller model",
338 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
344 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
349 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
350 description
=> "Specify guest operating system.",
351 verbose_description
=> <<EODESC,
352 Specify guest operating system. This is used to enable special
353 optimization/features for specific operating systems:
356 other;; unspecified OS
357 wxp;; Microsoft Windows XP
358 w2k;; Microsoft Windows 2000
359 w2k3;; Microsoft Windows 2003
360 w2k8;; Microsoft Windows 2008
361 wvista;; Microsoft Windows Vista
362 win7;; Microsoft Windows 7
363 win8;; Microsoft Windows 8/2012/2012r2
364 win10;; Microsoft Windows 10/2016
365 l24;; Linux 2.4 Kernel
366 l26;; Linux 2.6 - 5.X Kernel
367 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
373 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
374 pattern
=> '[acdn]{1,4}',
379 type
=> 'string', format
=> 'pve-qm-bootdisk',
380 description
=> "Enable booting from specified disk.",
381 pattern
=> '(ide|sata|scsi|virtio)\d+',
386 description
=> "The number of CPUs. Please use option -sockets instead.",
393 description
=> "The number of CPU sockets.",
400 description
=> "The number of cores per socket.",
407 description
=> "Enable/disable NUMA.",
413 description
=> "Enable/disable hugepages memory.",
414 enum
=> [qw(any 2 1024)],
419 description
=> "Number of hotplugged vcpus.",
426 description
=> "Enable/disable ACPI.",
431 description
=> "Enable/disable Qemu GuestAgent and its properties.",
433 format
=> $agent_fmt,
438 description
=> "Enable/disable KVM hardware virtualization.",
444 description
=> "Enable/disable time drift fix.",
450 description
=> "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
455 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
459 type
=> 'string', format
=> $vga_fmt,
460 description
=> "Configure the VGA hardware.",
461 verbose_description
=> "Configure the VGA Hardware. If you want to use ".
462 "high resolution modes (>= 1280x1024x16) you may need to increase " .
463 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
464 "is 'std' for all OS types besides some Windows versions (XP and " .
465 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
466 "display server. For win* OS you can select how many independent " .
467 "displays you want, Linux guests can add displays them self.\n".
468 "You can also run without any graphic card, using a serial device as terminal.",
472 type
=> 'string', format
=> 'pve-qm-watchdog',
473 description
=> "Create a virtual hardware watchdog device.",
474 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled" .
475 " (by a guest action), the watchdog must be periodically polled " .
476 "by an agent inside the guest or else the watchdog will reset " .
477 "the guest (or execute the respective action specified)",
482 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
483 description
=> "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
484 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
487 startup
=> get_standard_option
('pve-startup-order'),
491 description
=> "Enable/disable Template.",
497 description
=> "Arbitrary arguments passed to kvm.",
498 verbose_description
=> <<EODESCR,
499 Arbitrary arguments passed to kvm, for example:
501 args: -no-reboot -no-hpet
503 NOTE: this option is for experts only.
510 description
=> "Enable/disable the USB tablet device.",
511 verbose_description
=> "Enable/disable the USB tablet device. This device is " .
512 "usually needed to allow absolute mouse positioning with VNC. " .
513 "Else the mouse runs out of sync with normal VNC clients. " .
514 "If you're running lots of console-only guests on one host, " .
515 "you may consider disabling this to save some context switches. " .
516 "This is turned off by default if you use spice (-vga=qxl).",
521 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
525 migrate_downtime
=> {
528 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
534 type
=> 'string', format
=> 'pve-qm-ide',
535 typetext
=> '<volume>',
536 description
=> "This is an alias for option -ide2",
540 description
=> "Emulated CPU type.",
542 format
=> $PVE::QemuServer
::CPUConfig
::cpu_fmt
,
544 parent
=> get_standard_option
('pve-snapshot-name', {
546 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
550 description
=> "Timestamp for snapshots.",
556 type
=> 'string', format
=> 'pve-volume-id',
557 description
=> "Reference to a volume which stores the VM state. This is used internally for snapshots.",
559 vmstatestorage
=> get_standard_option
('pve-storage-id', {
560 description
=> "Default storage for VM state volumes/files.",
563 runningmachine
=> get_standard_option
('pve-qemu-machine', {
564 description
=> "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
566 machine
=> get_standard_option
('pve-qemu-machine'),
568 description
=> "Virtual processor architecture. Defaults to the host.",
571 enum
=> [qw(x86_64 aarch64)],
574 description
=> "Specify SMBIOS type 1 fields.",
575 type
=> 'string', format
=> 'pve-qm-smbios1',
582 description
=> "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
588 enum
=> [ qw(seabios ovmf) ],
589 description
=> "Select BIOS implementation.",
590 default => 'seabios',
594 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
595 format_description
=> 'UUID',
596 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
597 verbose_description
=> "The VM generation ID (vmgenid) device exposes a".
598 " 128-bit integer value identifier to the guest OS. This allows to".
599 " notify the guest operating system when the virtual machine is".
600 " executed with a different configuration (e.g. snapshot execution".
601 " or creation from a template). The guest operating system notices".
602 " the change, and is then able to react as appropriate by marking".
603 " its copies of distributed databases as dirty, re-initializing its".
604 " random number generator, etc.\n".
605 "Note that auto-creation only works when done throug API/CLI create".
606 " or update methods, but not when manually editing the config file.",
607 default => "1 (autogenerated)",
612 format
=> 'pve-volume-id',
614 description
=> "Script that will be executed during various steps in the vms lifetime.",
618 format
=> $ivshmem_fmt,
619 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
624 format
=> $audio_fmt,
625 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
628 spice_enhancements
=> {
630 format
=> $spice_enhancements_fmt,
631 description
=> "Configure additional enhancements for SPICE.",
635 type
=> 'string', format
=> 'pve-tag-list',
636 description
=> 'Tags of the VM. This is only meta information.',
642 description
=> "Configure a VirtIO-based Random Number Generator.",
651 description
=> 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
652 format
=> 'pve-volume-id',
653 format_description
=> 'volume',
658 description
=> 'Specify a custom file containing all network data passed to the VM via cloud-init.',
659 format
=> 'pve-volume-id',
660 format_description
=> 'volume',
665 description
=> 'Specify a custom file containing all user data passed to the VM via cloud-init.',
666 format
=> 'pve-volume-id',
667 format_description
=> 'volume',
670 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
672 my $confdesc_cloudinit = {
676 description
=> 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
677 enum
=> ['configdrive2', 'nocloud'],
682 description
=> "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
687 description
=> 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
692 description
=> 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
693 format
=> 'pve-qm-cicustom',
698 description
=> "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
702 type
=> 'string', format
=> 'address-list',
703 description
=> "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
708 format
=> 'urlencoded',
709 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
713 # what about other qemu settings ?
715 #machine => 'string',
728 ##soundhw => 'string',
730 while (my ($k, $v) = each %$confdesc) {
731 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
734 my $MAX_USB_DEVICES = 5;
736 my $MAX_HOSTPCI_DEVICES = 16;
737 my $MAX_SERIAL_PORTS = 4;
738 my $MAX_PARALLEL_PORTS = 3;
744 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
745 description
=> "CPUs accessing this NUMA node.",
746 format_description
=> "id[-id];...",
750 description
=> "Amount of memory this NUMA node provides.",
755 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
756 description
=> "Host NUMA nodes to use.",
757 format_description
=> "id[-id];...",
762 enum
=> [qw(preferred bind interleave)],
763 description
=> "NUMA allocation policy.",
767 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
770 type
=> 'string', format
=> $numa_fmt,
771 description
=> "NUMA topology.",
773 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
775 for (my $i = 0; $i < $MAX_NUMA; $i++) {
776 $confdesc->{"numa$i"} = $numadesc;
779 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
780 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
781 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
782 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
784 my $net_fmt_bridge_descr = <<__EOD__;
785 Bridge to attach the network device to. The Proxmox VE standard bridge
788 If you do not specify a bridge, we create a kvm user (NATed) network
789 device, which provides DHCP and DNS services. The following addresses
796 The DHCP server assign addresses to the guest starting from 10.0.2.15.
800 macaddr
=> get_standard_option
('mac-addr', {
801 description
=> "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
805 description
=> "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
806 enum
=> $nic_model_list,
809 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
812 description
=> $net_fmt_bridge_descr,
813 format_description
=> 'bridge',
818 minimum
=> 0, maximum
=> 16,
819 description
=> 'Number of packet queues to be used on the device.',
825 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
830 minimum
=> 1, maximum
=> 4094,
831 description
=> 'VLAN tag to apply to packets on this interface.',
836 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
837 description
=> 'VLAN trunks to pass through this interface.',
838 format_description
=> 'vlanid[;vlanid...]',
843 description
=> 'Whether this interface should be protected by the firewall.',
848 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
855 type
=> 'string', format
=> $net_fmt,
856 description
=> "Specify network devices.",
859 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
864 format
=> 'pve-ipv4-config',
865 format_description
=> 'IPv4Format/CIDR',
866 description
=> 'IPv4 address in CIDR format.',
873 format_description
=> 'GatewayIPv4',
874 description
=> 'Default gateway for IPv4 traffic.',
880 format
=> 'pve-ipv6-config',
881 format_description
=> 'IPv6Format/CIDR',
882 description
=> 'IPv6 address in CIDR format.',
889 format_description
=> 'GatewayIPv6',
890 description
=> 'Default gateway for IPv6 traffic.',
895 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
898 type
=> 'string', format
=> 'pve-qm-ipconfig',
899 description
=> <<'EODESCR',
900 cloud-init: Specify IP addresses and gateways for the corresponding interface.
902 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
904 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
905 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
907 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
910 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
912 for (my $i = 0; $i < $MAX_NETS; $i++) {
913 $confdesc->{"net$i"} = $netdesc;
914 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
917 foreach my $key (keys %$confdesc_cloudinit) {
918 $confdesc->{$key} = $confdesc_cloudinit->{$key};
921 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
922 sub verify_volume_id_or_qm_path
{
923 my ($volid, $noerr) = @_;
925 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
929 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
930 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
932 return undef if $noerr;
941 type
=> 'string', format
=> 'pve-qm-usb-device',
942 format_description
=> 'HOSTUSBDEVICE|spice',
943 description
=> <<EODESCR,
944 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
946 'bus-port(.port)*' (decimal numbers) or
947 'vendor_id:product_id' (hexadeciaml numbers) or
950 You can use the 'lsusb -t' command to list existing usb devices.
952 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
954 The value 'spice' can be used to add a usb redirection devices for spice.
960 description
=> "Specifies whether if given host option is a USB3 device or port.",
967 type
=> 'string', format
=> $usb_fmt,
968 description
=> "Configure an USB device (n is 0 to 4).",
970 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
972 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
977 pattern
=> qr/$PCIRE(;$PCIRE)*/,
978 format_description
=> 'HOSTPCIID[;HOSTPCIID2...]',
979 description
=> <<EODESCR,
980 Host PCI device pass through. The PCI ID of a host's PCI device or a list
981 of PCI virtual functions of the host. HOSTPCIID syntax is:
983 'bus:dev.func' (hexadecimal numbers)
985 You can us the 'lspci' command to list existing PCI devices.
990 description
=> "Specify whether or not the device's ROM will be visible in the guest's memory map.",
997 format_description
=> 'string',
998 description
=> "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1003 description
=> "Choose the PCI-express bus (needs the 'q35' machine model).",
1009 description
=> "Enable vfio-vga device support.",
1015 format_description
=> 'string',
1016 pattern
=> '[^/\.:]+',
1018 description
=> <<EODESCR
1019 The type of mediated device to use.
1020 An instance of this type will be created on startup of the VM and
1021 will be cleaned up when the VM stops.
1025 PVE
::JSONSchema
::register_format
('pve-qm-hostpci', $hostpci_fmt);
1029 type
=> 'string', format
=> 'pve-qm-hostpci',
1030 description
=> "Map host PCI devices into guest.",
1031 verbose_description
=> <<EODESCR,
1032 Map host PCI devices into guest.
1034 NOTE: This option allows direct access to host hardware. So it is no longer
1035 possible to migrate such machines - use with special care.
1037 CAUTION: Experimental! User reported problems with this option.
1040 PVE
::JSONSchema
::register_standard_option
("pve-qm-hostpci", $hostpcidesc);
1045 pattern
=> '(/dev/.+|socket)',
1046 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1047 verbose_description
=> <<EODESCR,
1048 Create a serial device inside the VM (n is 0 to 3), and pass through a
1049 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1050 host side (use 'qm terminal' to open a terminal connection).
1052 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1054 CAUTION: Experimental! User reported problems with this option.
1061 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1062 description
=> "Map host parallel devices (n is 0 to 2).",
1063 verbose_description
=> <<EODESCR,
1064 Map host parallel devices (n is 0 to 2).
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1068 CAUTION: Experimental! User reported problems with this option.
1072 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1073 $confdesc->{"parallel$i"} = $paralleldesc;
1076 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1077 $confdesc->{"serial$i"} = $serialdesc;
1080 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
1081 $confdesc->{"hostpci$i"} = $hostpcidesc;
1084 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1085 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1088 for (my $i = 0; $i < $PVE::QemuServer
::Drive
::MAX_UNUSED_DISKS
; $i++) {
1089 $confdesc->{"unused$i"} = $PVE::QemuServer
::Drive
::unuseddesc
;
1092 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1093 $confdesc->{"usb$i"} = $usbdesc;
1096 my $kvm_api_version = 0;
1099 return $kvm_api_version if $kvm_api_version;
1101 open my $fh, '<', '/dev/kvm'
1104 # 0xae00 => KVM_GET_API_VERSION
1105 $kvm_api_version = ioctl($fh, 0xae00, 0);
1107 return $kvm_api_version;
1110 my $kvm_user_version = {};
1113 sub kvm_user_version
{
1116 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1117 my $st = stat($binary);
1119 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1120 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1121 $cachedmtime == $st->mtime;
1123 $kvm_user_version->{$binary} = 'unknown';
1124 $kvm_mtime->{$binary} = $st->mtime;
1128 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1129 $kvm_user_version->{$binary} = $2;
1133 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1136 return $kvm_user_version->{$binary};
1140 sub kernel_has_vhost_net
{
1141 return -c
'/dev/vhost-net';
1146 return defined($confdesc->{$key});
1150 sub get_cdrom_path
{
1152 return $cdrom_path if $cdrom_path;
1154 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1155 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1156 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1160 my ($storecfg, $vmid, $cdrom) = @_;
1162 if ($cdrom eq 'cdrom') {
1163 return get_cdrom_path
();
1164 } elsif ($cdrom eq 'none') {
1166 } elsif ($cdrom =~ m
|^/|) {
1169 return PVE
::Storage
::path
($storecfg, $cdrom);
1173 # try to convert old style file names to volume IDs
1174 sub filename_to_volume_id
{
1175 my ($vmid, $file, $media) = @_;
1177 if (!($file eq 'none' || $file eq 'cdrom' ||
1178 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1180 return undef if $file =~ m
|/|;
1182 if ($media && $media eq 'cdrom') {
1183 $file = "local:iso/$file";
1185 $file = "local:$vmid/$file";
1192 sub verify_media_type
{
1193 my ($opt, $vtype, $media) = @_;
1198 if ($media eq 'disk') {
1200 } elsif ($media eq 'cdrom') {
1203 die "internal error";
1206 return if ($vtype eq $etype);
1208 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1211 sub cleanup_drive_path
{
1212 my ($opt, $storecfg, $drive) = @_;
1214 # try to convert filesystem paths to volume IDs
1216 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1217 ($drive->{file
} !~ m
|^/dev/.+|) &&
1218 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1219 ($drive->{file
} !~ m/^\d+$/)) {
1220 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1221 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1222 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1223 verify_media_type
($opt, $vtype, $drive->{media
});
1224 $drive->{file
} = $volid;
1227 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1230 sub parse_hotplug_features
{
1235 return $res if $data eq '0';
1237 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1239 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1240 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1243 die "invalid hotplug feature '$feature'\n";
1249 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1250 sub pve_verify_hotplug_features
{
1251 my ($value, $noerr) = @_;
1253 return $value if parse_hotplug_features
($value);
1255 return undef if $noerr;
1257 die "unable to parse hotplug option\n";
1261 my($fh, $noerr) = @_;
1264 my $SG_GET_VERSION_NUM = 0x2282;
1266 my $versionbuf = "\x00" x
8;
1267 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1269 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1272 my $version = unpack("I", $versionbuf);
1273 if ($version < 30000) {
1274 die "scsi generic interface too old\n" if !$noerr;
1278 my $buf = "\x00" x
36;
1279 my $sensebuf = "\x00" x
8;
1280 my $cmd = pack("C x3 C x1", 0x12, 36);
1282 # see /usr/include/scsi/sg.h
1283 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1285 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1286 length($sensebuf), 0, length($buf), $buf,
1287 $cmd, $sensebuf, 6000);
1289 $ret = ioctl($fh, $SG_IO, $packet);
1291 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1295 my @res = unpack($sg_io_hdr_t, $packet);
1296 if ($res[17] || $res[18]) {
1297 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1302 (my $byte0, my $byte1, $res->{vendor
},
1303 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1305 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1306 $res->{type
} = $byte0 & 31;
1314 my $fh = IO
::File-
>new("+<$path") || return undef;
1315 my $res = scsi_inquiry
($fh, 1);
1321 sub print_tabletdevice_full
{
1322 my ($conf, $arch) = @_;
1324 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1326 # we use uhci for old VMs because tablet driver was buggy in older qemu
1328 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1334 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1337 sub print_keyboarddevice_full
{
1338 my ($conf, $arch, $machine) = @_;
1340 return undef if $arch ne 'aarch64';
1342 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1345 sub print_drivedevice_full
{
1346 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1351 if ($drive->{interface
} eq 'virtio') {
1352 my $pciaddr = print_pci_addr
("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1353 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1354 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread
};
1355 } elsif ($drive->{interface
} eq 'scsi') {
1357 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1358 my $unit = $drive->{index} % $maxdev;
1359 my $devicetype = 'hd';
1361 if (drive_is_cdrom
($drive)) {
1364 if ($drive->{file
} =~ m
|^/|) {
1365 $path = $drive->{file
};
1366 if (my $info = path_is_scsi
($path)) {
1367 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1368 $devicetype = 'block';
1369 } elsif ($info->{type
} == 1) { # tape
1370 $devicetype = 'generic';
1374 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1377 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1378 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, kvm_user_version
());
1379 if ($path =~ m/^iscsi\:\/\
// &&
1380 !min_version
($version, 4, 1)) {
1381 $devicetype = 'generic';
1385 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1386 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1388 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1391 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1392 $device .= ",rotation_rate=1";
1394 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1396 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1397 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1398 my $controller = int($drive->{index} / $maxdev);
1399 my $unit = $drive->{index} % $maxdev;
1400 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1402 $device = "ide-$devicetype";
1403 if ($drive->{interface
} eq 'ide') {
1404 $device .= ",bus=ide.$controller,unit=$unit";
1406 $device .= ",bus=ahci$controller.$unit";
1408 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1410 if ($devicetype eq 'hd') {
1411 if (my $model = $drive->{model
}) {
1412 $model = URI
::Escape
::uri_unescape
($model);
1413 $device .= ",model=$model";
1415 if ($drive->{ssd
}) {
1416 $device .= ",rotation_rate=1";
1419 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1420 } elsif ($drive->{interface
} eq 'usb') {
1422 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1424 die "unsupported interface type";
1427 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1429 if (my $serial = $drive->{serial
}) {
1430 $serial = URI
::Escape
::uri_unescape
($serial);
1431 $device .= ",serial=$serial";
1438 sub get_initiator_name
{
1441 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return undef;
1442 while (defined(my $line = <$fh>)) {
1443 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1452 sub print_drive_full
{
1453 my ($storecfg, $vmid, $drive) = @_;
1456 my $volid = $drive->{file
};
1459 if (drive_is_cdrom
($drive)) {
1460 $path = get_iso_path
($storecfg, $vmid, $volid);
1462 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1464 $path = PVE
::Storage
::path
($storecfg, $volid);
1465 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1466 $format = qemu_img_format
($scfg, $volname);
1474 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1475 foreach my $o (@qemu_drive_options) {
1476 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1479 # snapshot only accepts on|off
1480 if (defined($drive->{snapshot
})) {
1481 my $v = $drive->{snapshot
} ?
'on' : 'off';
1482 $opts .= ",snapshot=$v";
1485 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1486 my ($dir, $qmpname) = @$type;
1487 if (my $v = $drive->{"mbps$dir"}) {
1488 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1490 if (my $v = $drive->{"mbps${dir}_max"}) {
1491 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1493 if (my $v = $drive->{"bps${dir}_max_length"}) {
1494 $opts .= ",throttling.bps$qmpname-max-length=$v";
1496 if (my $v = $drive->{"iops${dir}"}) {
1497 $opts .= ",throttling.iops$qmpname=$v";
1499 if (my $v = $drive->{"iops${dir}_max"}) {
1500 $opts .= ",throttling.iops$qmpname-max=$v";
1502 if (my $v = $drive->{"iops${dir}_max_length"}) {
1503 $opts .= ",throttling.iops$qmpname-max-length=$v";
1507 $opts .= ",format=$format" if $format && !$drive->{format
};
1509 my $cache_direct = 0;
1511 if (my $cache = $drive->{cache
}) {
1512 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1513 } elsif (!drive_is_cdrom
($drive)) {
1514 $opts .= ",cache=none";
1518 # aio native works only with O_DIRECT
1519 if (!$drive->{aio
}) {
1521 $opts .= ",aio=native";
1523 $opts .= ",aio=threads";
1527 if (!drive_is_cdrom
($drive)) {
1529 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1530 $detectzeroes = 'off';
1531 } elsif ($drive->{discard
}) {
1532 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1534 # This used to be our default with discard not being specified:
1535 $detectzeroes = 'on';
1537 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1540 my $pathinfo = $path ?
"file=$path," : '';
1542 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1545 sub print_netdevice_full
{
1546 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1548 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
1550 my $device = $net->{model
};
1551 if ($net->{model
} eq 'virtio') {
1552 $device = 'virtio-net-pci';
1555 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1556 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1557 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1558 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1559 my $vectors = $net->{queues
} * 2 + 2;
1560 $tmpstr .= ",vectors=$vectors,mq=on";
1562 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1564 if ($use_old_bios_files) {
1566 if ($device eq 'virtio-net-pci') {
1567 $romfile = 'pxe-virtio.rom';
1568 } elsif ($device eq 'e1000') {
1569 $romfile = 'pxe-e1000.rom';
1570 } elsif ($device eq 'ne2k') {
1571 $romfile = 'pxe-ne2k_pci.rom';
1572 } elsif ($device eq 'pcnet') {
1573 $romfile = 'pxe-pcnet.rom';
1574 } elsif ($device eq 'rtl8139') {
1575 $romfile = 'pxe-rtl8139.rom';
1577 $tmpstr .= ",romfile=$romfile" if $romfile;
1583 sub print_netdev_full
{
1584 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1587 if ($netid =~ m/^net(\d+)$/) {
1591 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1593 my $ifname = "tap${vmid}i$i";
1595 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1596 die "interface name '$ifname' is too long (max 15 character)\n"
1597 if length($ifname) >= 16;
1599 my $vhostparam = '';
1600 if (is_native
($arch)) {
1601 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1604 my $vmname = $conf->{name
} || "vm$vmid";
1607 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1609 if ($net->{bridge
}) {
1610 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1612 $netdev = "type=user,id=$netid,hostname=$vmname";
1615 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1621 'cirrus' => 'cirrus-vga',
1623 'vmware' => 'vmware-svga',
1624 'virtio' => 'virtio-vga',
1627 sub print_vga_device
{
1628 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1630 my $type = $vga_map->{$vga->{type
}};
1631 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1632 $type = 'virtio-gpu';
1634 my $vgamem_mb = $vga->{memory
};
1636 my $max_outputs = '';
1638 $type = $id ?
'qxl' : 'qxl-vga';
1640 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1641 # set max outputs so linux can have up to 4 qxl displays with one device
1642 if (min_version
($machine_version, 4, 1)) {
1643 $max_outputs = ",max_outputs=4";
1648 die "no devicetype for $vga->{type}\n" if !$type;
1652 if ($vga->{type
} eq 'virtio') {
1653 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1654 $memory = ",max_hostmem=$bytes";
1656 # from https://www.spice-space.org/multiple-monitors.html
1657 $memory = ",vgamem_mb=$vga->{memory}";
1658 my $ram = $vgamem_mb * 4;
1659 my $vram = $vgamem_mb * 2;
1660 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1662 $memory = ",vgamem_mb=$vga->{memory}";
1664 } elsif ($qxlnum && $id) {
1665 $memory = ",ram_size=67108864,vram_size=33554432";
1668 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1669 my $vgaid = "vga" . ($id // '');
1672 if ($q35 && $vgaid eq 'vga') {
1673 # the first display uses pcie.0 bus on q35 machines
1674 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1676 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1679 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1682 sub parse_number_sets
{
1685 foreach my $part (split(/;/, $set)) {
1686 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1687 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1688 push @$res, [ $1, $2 ];
1690 die "invalid range: $part\n";
1699 my $res = PVE
::JSONSchema
::parse_property_string
($numa_fmt, $data);
1700 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1701 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1708 return undef if !$value;
1710 my $res = PVE
::JSONSchema
::parse_property_string
($hostpci_fmt, $value);
1712 my @idlist = split(/;/, $res->{host
});
1713 delete $res->{host
};
1714 foreach my $id (@idlist) {
1715 my $devs = PVE
::SysFSTools
::lspci
($id);
1716 die "no PCI device found for '$id'\n" if !scalar(@$devs);
1717 push @{$res->{pciid
}}, @$devs;
1722 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1726 my $res = eval { PVE
::JSONSchema
::parse_property_string
($net_fmt, $data) };
1731 if (!defined($res->{macaddr
})) {
1732 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1733 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1738 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1739 sub parse_ipconfig
{
1742 my $res = eval { PVE
::JSONSchema
::parse_property_string
($ipconfig_fmt, $data) };
1748 if ($res->{gw
} && !$res->{ip
}) {
1749 warn 'gateway specified without specifying an IP address';
1752 if ($res->{gw6
} && !$res->{ip6
}) {
1753 warn 'IPv6 gateway specified without specifying an IPv6 address';
1756 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1757 warn 'gateway specified together with DHCP';
1760 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1762 warn "IPv6 gateway specified together with $res->{ip6} address";
1766 if (!$res->{ip
} && !$res->{ip6
}) {
1767 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1776 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1779 sub add_random_macs
{
1780 my ($settings) = @_;
1782 foreach my $opt (keys %$settings) {
1783 next if $opt !~ m/^net(\d+)$/;
1784 my $net = parse_net
($settings->{$opt});
1786 $settings->{$opt} = print_net
($net);
1790 sub vm_is_volid_owner
{
1791 my ($storecfg, $vmid, $volid) = @_;
1793 if ($volid !~ m
|^/|) {
1795 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1796 if ($owner && ($owner == $vmid)) {
1804 sub vmconfig_register_unused_drive
{
1805 my ($storecfg, $vmid, $conf, $drive) = @_;
1807 if (drive_is_cloudinit
($drive)) {
1808 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1810 } elsif (!drive_is_cdrom
($drive)) {
1811 my $volid = $drive->{file
};
1812 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1813 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1818 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1822 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1823 format_description
=> 'UUID',
1824 description
=> "Set SMBIOS1 UUID.",
1829 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1830 format_description
=> 'Base64 encoded string',
1831 description
=> "Set SMBIOS1 version.",
1836 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1837 format_description
=> 'Base64 encoded string',
1838 description
=> "Set SMBIOS1 serial number.",
1843 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1844 format_description
=> 'Base64 encoded string',
1845 description
=> "Set SMBIOS1 manufacturer.",
1850 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1851 format_description
=> 'Base64 encoded string',
1852 description
=> "Set SMBIOS1 product ID.",
1857 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1858 format_description
=> 'Base64 encoded string',
1859 description
=> "Set SMBIOS1 SKU string.",
1864 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1865 format_description
=> 'Base64 encoded string',
1866 description
=> "Set SMBIOS1 family string.",
1871 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
1879 my $res = eval { PVE
::JSONSchema
::parse_property_string
($smbios1_fmt, $data) };
1886 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
1889 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
1891 sub parse_watchdog
{
1894 return undef if !$value;
1896 my $res = eval { PVE
::JSONSchema
::parse_property_string
($watchdog_fmt, $value) };
1901 sub parse_guest_agent
{
1904 return {} if !defined($value->{agent
});
1906 my $res = eval { PVE
::JSONSchema
::parse_property_string
($agent_fmt, $value->{agent
}) };
1909 # if the agent is disabled ignore the other potentially set properties
1910 return {} if !$res->{enabled
};
1917 return {} if !$value;
1918 my $res = eval { PVE
::JSONSchema
::parse_property_string
($vga_fmt, $value) };
1926 return undef if !$value;
1928 my $res = eval { PVE
::JSONSchema
::parse_property_string
($rng_fmt, $value) };
1933 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
1934 sub verify_usb_device
{
1935 my ($value, $noerr) = @_;
1937 return $value if parse_usb_device
($value);
1939 return undef if $noerr;
1941 die "unable to parse usb device\n";
1944 # add JSON properties for create and set function
1945 sub json_config_properties
{
1948 foreach my $opt (keys %$confdesc) {
1949 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
1950 $prop->{$opt} = $confdesc->{$opt};
1956 # return copy of $confdesc_cloudinit to generate documentation
1957 sub cloudinit_config_properties
{
1959 return dclone
($confdesc_cloudinit);
1963 my ($key, $value) = @_;
1965 die "unknown setting '$key'\n" if !$confdesc->{$key};
1967 my $type = $confdesc->{$key}->{type
};
1969 if (!defined($value)) {
1970 die "got undefined value\n";
1973 if ($value =~ m/[\n\r]/) {
1974 die "property contains a line feed\n";
1977 if ($type eq 'boolean') {
1978 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1979 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1980 die "type check ('boolean') failed - got '$value'\n";
1981 } elsif ($type eq 'integer') {
1982 return int($1) if $value =~ m/^(\d+)$/;
1983 die "type check ('integer') failed - got '$value'\n";
1984 } elsif ($type eq 'number') {
1985 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1986 die "type check ('number') failed - got '$value'\n";
1987 } elsif ($type eq 'string') {
1988 if (my $fmt = $confdesc->{$key}->{format
}) {
1989 PVE
::JSONSchema
::check_format
($fmt, $value);
1992 $value =~ s/^\"(.*)\"$/$1/;
1995 die "internal error"
2000 my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2002 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2004 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2006 if ($conf->{template
}) {
2007 # check if any base image is still used by a linked clone
2008 foreach_drive
($conf, sub {
2009 my ($ds, $drive) = @_;
2010 return if drive_is_cdrom
($drive);
2012 my $volid = $drive->{file
};
2013 return if !$volid || $volid =~ m
|^/|;
2015 die "base volume '$volid' is still in use by linked cloned\n"
2016 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2021 # only remove disks owned by this VM
2022 foreach_drive
($conf, sub {
2023 my ($ds, $drive) = @_;
2024 return if drive_is_cdrom
($drive, 1);
2026 my $volid = $drive->{file
};
2027 return if !$volid || $volid =~ m
|^/|;
2029 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2030 return if !$path || !$owner || ($owner != $vmid);
2032 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2033 warn "Could not remove disk '$volid', check manually: $@" if $@;
2036 # also remove unused disk
2037 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2038 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2039 my ($volid, $sid, $volname, $d) = @_;
2040 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2044 if (defined $replacement_conf) {
2045 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2047 PVE
::QemuConfig-
>destroy_config($vmid);
2051 sub parse_vm_config
{
2052 my ($filename, $raw) = @_;
2054 return undef if !defined($raw);
2057 digest
=> Digest
::SHA
::sha1_hex
($raw),
2062 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2063 || die "got strange filename '$filename'";
2071 my @lines = split(/\n/, $raw);
2072 foreach my $line (@lines) {
2073 next if $line =~ m/^\s*$/;
2075 if ($line =~ m/^\[PENDING\]\s*$/i) {
2076 $section = 'pending';
2077 if (defined($descr)) {
2079 $conf->{description
} = $descr;
2082 $conf = $res->{$section} = {};
2085 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2087 if (defined($descr)) {
2089 $conf->{description
} = $descr;
2092 $conf = $res->{snapshots
}->{$section} = {};
2096 if ($line =~ m/^\#(.*)\s*$/) {
2097 $descr = '' if !defined($descr);
2098 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2102 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2103 $descr = '' if !defined($descr);
2104 $descr .= PVE
::Tools
::decode_text
($2);
2105 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2106 $conf->{snapstate
} = $1;
2107 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2110 $conf->{$key} = $value;
2111 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2113 if ($section eq 'pending') {
2114 $conf->{delete} = $value; # we parse this later
2116 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2118 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2121 eval { $value = check_type
($key, $value); };
2123 warn "vm $vmid - unable to parse value of '$key' - $@";
2125 $key = 'ide2' if $key eq 'cdrom';
2126 my $fmt = $confdesc->{$key}->{format
};
2127 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2128 my $v = parse_drive
($key, $value);
2129 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2130 $v->{file
} = $volid;
2131 $value = print_drive
($v);
2133 warn "vm $vmid - unable to parse value of '$key'\n";
2138 $conf->{$key} = $value;
2143 if (defined($descr)) {
2145 $conf->{description
} = $descr;
2147 delete $res->{snapstate
}; # just to be sure
2152 sub write_vm_config
{
2153 my ($filename, $conf) = @_;
2155 delete $conf->{snapstate
}; # just to be sure
2157 if ($conf->{cdrom
}) {
2158 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2159 $conf->{ide2
} = $conf->{cdrom
};
2160 delete $conf->{cdrom
};
2163 # we do not use 'smp' any longer
2164 if ($conf->{sockets
}) {
2165 delete $conf->{smp
};
2166 } elsif ($conf->{smp
}) {
2167 $conf->{sockets
} = $conf->{smp
};
2168 delete $conf->{cores
};
2169 delete $conf->{smp
};
2172 my $used_volids = {};
2174 my $cleanup_config = sub {
2175 my ($cref, $pending, $snapname) = @_;
2177 foreach my $key (keys %$cref) {
2178 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2179 $key eq 'snapstate' || $key eq 'pending';
2180 my $value = $cref->{$key};
2181 if ($key eq 'delete') {
2182 die "propertry 'delete' is only allowed in [PENDING]\n"
2184 # fixme: check syntax?
2187 eval { $value = check_type
($key, $value); };
2188 die "unable to parse value of '$key' - $@" if $@;
2190 $cref->{$key} = $value;
2192 if (!$snapname && is_valid_drivename
($key)) {
2193 my $drive = parse_drive
($key, $value);
2194 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2199 &$cleanup_config($conf);
2201 &$cleanup_config($conf->{pending
}, 1);
2203 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2204 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2205 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2208 # remove 'unusedX' settings if we re-add a volume
2209 foreach my $key (keys %$conf) {
2210 my $value = $conf->{$key};
2211 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2212 delete $conf->{$key};
2216 my $generate_raw_config = sub {
2217 my ($conf, $pending) = @_;
2221 # add description as comment to top of file
2222 if (defined(my $descr = $conf->{description
})) {
2224 foreach my $cl (split(/\n/, $descr)) {
2225 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2228 $raw .= "#\n" if $pending;
2232 foreach my $key (sort keys %$conf) {
2233 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2234 $raw .= "$key: $conf->{$key}\n";
2239 my $raw = &$generate_raw_config($conf);
2241 if (scalar(keys %{$conf->{pending
}})){
2242 $raw .= "\n[PENDING]\n";
2243 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2246 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2247 $raw .= "\n[$snapname]\n";
2248 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2258 # we use static defaults from our JSON schema configuration
2259 foreach my $key (keys %$confdesc) {
2260 if (defined(my $default = $confdesc->{$key}->{default})) {
2261 $res->{$key} = $default;
2269 my $vmlist = PVE
::Cluster
::get_vmlist
();
2271 return $res if !$vmlist || !$vmlist->{ids
};
2272 my $ids = $vmlist->{ids
};
2273 my $nodename = nodename
();
2275 foreach my $vmid (keys %$ids) {
2276 my $d = $ids->{$vmid};
2277 next if !$d->{node
} || $d->{node
} ne $nodename;
2278 next if !$d->{type
} || $d->{type
} ne 'qemu';
2279 $res->{$vmid}->{exists} = 1;
2284 # test if VM uses local resources (to prevent migration)
2285 sub check_local_resources
{
2286 my ($conf, $noerr) = @_;
2290 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2291 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2293 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2295 foreach my $k (keys %$conf) {
2296 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2297 # sockets are safe: they will recreated be on the target side post-migrate
2298 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2299 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2302 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2307 # check if used storages are available on all nodes (use by migrate)
2308 sub check_storage_availability
{
2309 my ($storecfg, $conf, $node) = @_;
2311 foreach_drive
($conf, sub {
2312 my ($ds, $drive) = @_;
2314 my $volid = $drive->{file
};
2317 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2320 # check if storage is available on both nodes
2321 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2322 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2326 # list nodes where all VM images are available (used by has_feature API)
2328 my ($conf, $storecfg) = @_;
2330 my $nodelist = PVE
::Cluster
::get_nodelist
();
2331 my $nodehash = { map { $_ => 1 } @$nodelist };
2332 my $nodename = nodename
();
2334 foreach_drive
($conf, sub {
2335 my ($ds, $drive) = @_;
2337 my $volid = $drive->{file
};
2340 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2342 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2343 if ($scfg->{disable
}) {
2345 } elsif (my $avail = $scfg->{nodes
}) {
2346 foreach my $node (keys %$nodehash) {
2347 delete $nodehash->{$node} if !$avail->{$node};
2349 } elsif (!$scfg->{shared
}) {
2350 foreach my $node (keys %$nodehash) {
2351 delete $nodehash->{$node} if $node ne $nodename
2360 sub check_local_storage_availability
{
2361 my ($conf, $storecfg) = @_;
2363 my $nodelist = PVE
::Cluster
::get_nodelist
();
2364 my $nodehash = { map { $_ => {} } @$nodelist };
2366 foreach_drive
($conf, sub {
2367 my ($ds, $drive) = @_;
2369 my $volid = $drive->{file
};
2372 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2374 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2376 if ($scfg->{disable
}) {
2377 foreach my $node (keys %$nodehash) {
2378 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2380 } elsif (my $avail = $scfg->{nodes
}) {
2381 foreach my $node (keys %$nodehash) {
2382 if (!$avail->{$node}) {
2383 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2390 foreach my $node (values %$nodehash) {
2391 if (my $unavail = $node->{unavailable_storages
}) {
2392 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2399 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2401 my ($vmid, $nocheck, $node) = @_;
2403 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2404 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2409 my $vzlist = config_list
();
2411 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2413 while (defined(my $de = $fd->read)) {
2414 next if $de !~ m/^(\d+)\.pid$/;
2416 next if !defined($vzlist->{$vmid});
2417 if (my $pid = check_running
($vmid)) {
2418 $vzlist->{$vmid}->{pid
} = $pid;
2425 our $vmstatus_return_properties = {
2426 vmid
=> get_standard_option
('pve-vmid'),
2428 description
=> "Qemu process status.",
2430 enum
=> ['stopped', 'running'],
2433 description
=> "Maximum memory in bytes.",
2436 renderer
=> 'bytes',
2439 description
=> "Root disk size in bytes.",
2442 renderer
=> 'bytes',
2445 description
=> "VM name.",
2450 description
=> "Qemu QMP agent status.",
2455 description
=> "PID of running qemu process.",
2460 description
=> "Uptime.",
2463 renderer
=> 'duration',
2466 description
=> "Maximum usable CPUs.",
2471 description
=> "The current config lock, if any.",
2476 description
=> "The current configured tags, if any",
2482 my $last_proc_pid_stat;
2484 # get VM status information
2485 # This must be fast and should not block ($full == false)
2486 # We only query KVM using QMP if $full == true (this can be slow)
2488 my ($opt_vmid, $full) = @_;
2492 my $storecfg = PVE
::Storage
::config
();
2494 my $list = vzlist
();
2495 my $defaults = load_defaults
();
2497 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2499 my $cpucount = $cpuinfo->{cpus
} || 1;
2501 foreach my $vmid (keys %$list) {
2502 next if $opt_vmid && ($vmid ne $opt_vmid);
2504 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2506 my $d = { vmid
=> $vmid };
2507 $d->{pid
} = $list->{$vmid}->{pid
};
2509 # fixme: better status?
2510 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2512 my $size = PVE
::QemuServer
::Drive
::disksize
($storecfg, $conf);
2513 if (defined($size)) {
2514 $d->{disk
} = 0; # no info available
2515 $d->{maxdisk
} = $size;
2521 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2522 * ($conf->{cores
} || $defaults->{cores
});
2523 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2524 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2526 $d->{name
} = $conf->{name
} || "VM $vmid";
2527 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2528 : $defaults->{memory
}*(1024*1024);
2530 if ($conf->{balloon
}) {
2531 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2532 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2533 : $defaults->{shares
};
2544 $d->{diskwrite
} = 0;
2546 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
2548 $d->{serial
} = 1 if conf_has_serial
($conf);
2549 $d->{lock} = $conf->{lock} if $conf->{lock};
2550 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2555 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2556 foreach my $dev (keys %$netdev) {
2557 next if $dev !~ m/^tap([1-9]\d*)i/;
2559 my $d = $res->{$vmid};
2562 $d->{netout
} += $netdev->{$dev}->{receive
};
2563 $d->{netin
} += $netdev->{$dev}->{transmit
};
2566 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
2567 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
2572 my $ctime = gettimeofday
;
2574 foreach my $vmid (keys %$list) {
2576 my $d = $res->{$vmid};
2577 my $pid = $d->{pid
};
2580 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2581 next if !$pstat; # not running
2583 my $used = $pstat->{utime} + $pstat->{stime
};
2585 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2587 if ($pstat->{vsize
}) {
2588 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2591 my $old = $last_proc_pid_stat->{$pid};
2593 $last_proc_pid_stat->{$pid} = {
2601 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2603 if ($dtime > 1000) {
2604 my $dutime = $used - $old->{used
};
2606 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2607 $last_proc_pid_stat->{$pid} = {
2613 $d->{cpu
} = $old->{cpu
};
2617 return $res if !$full;
2619 my $qmpclient = PVE
::QMPClient-
>new();
2621 my $ballooncb = sub {
2622 my ($vmid, $resp) = @_;
2624 my $info = $resp->{'return'};
2625 return if !$info->{max_mem
};
2627 my $d = $res->{$vmid};
2629 # use memory assigned to VM
2630 $d->{maxmem
} = $info->{max_mem
};
2631 $d->{balloon
} = $info->{actual
};
2633 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2634 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2635 $d->{freemem
} = $info->{free_mem
};
2638 $d->{ballooninfo
} = $info;
2641 my $blockstatscb = sub {
2642 my ($vmid, $resp) = @_;
2643 my $data = $resp->{'return'} || [];
2644 my $totalrdbytes = 0;
2645 my $totalwrbytes = 0;
2647 for my $blockstat (@$data) {
2648 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2649 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2651 $blockstat->{device
} =~ s/drive-//;
2652 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2654 $res->{$vmid}->{diskread
} = $totalrdbytes;
2655 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2658 my $statuscb = sub {
2659 my ($vmid, $resp) = @_;
2661 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2662 # this fails if ballon driver is not loaded, so this must be
2663 # the last commnand (following command are aborted if this fails).
2664 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2666 my $status = 'unknown';
2667 if (!defined($status = $resp->{'return'}->{status
})) {
2668 warn "unable to get VM status\n";
2672 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2675 foreach my $vmid (keys %$list) {
2676 next if $opt_vmid && ($vmid ne $opt_vmid);
2677 next if !$res->{$vmid}->{pid
}; # not running
2678 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2681 $qmpclient->queue_execute(undef, 2);
2683 foreach my $vmid (keys %$list) {
2684 next if $opt_vmid && ($vmid ne $opt_vmid);
2685 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2691 sub conf_has_serial
{
2694 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2695 if ($conf->{"serial$i"}) {
2703 sub conf_has_audio
{
2704 my ($conf, $id) = @_;
2707 my $audio = $conf->{"audio$id"};
2708 return undef if !defined($audio);
2710 my $audioproperties = PVE
::JSONSchema
::parse_property_string
($audio_fmt, $audio);
2711 my $audiodriver = $audioproperties->{driver
} // 'spice';
2714 dev
=> $audioproperties->{device
},
2715 dev_id
=> "audiodev$id",
2716 backend
=> $audiodriver,
2717 backend_id
=> "$audiodriver-backend${id}",
2721 sub vga_conf_has_spice
{
2724 my $vgaconf = parse_vga
($vga);
2725 my $vgatype = $vgaconf->{type
};
2726 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2733 return get_host_arch
() eq $arch;
2738 return $conf->{arch
} // get_host_arch
();
2741 my $default_machines = {
2746 sub get_vm_machine
{
2747 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2749 my $machine = $forcemachine || $conf->{machine
};
2751 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2753 $machine ||= $default_machines->{$arch};
2754 if ($add_pve_version) {
2755 $kvmversion //= kvm_user_version
();
2756 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
2757 $machine .= "+pve$pvever";
2761 if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2762 # for version-pinned machines that do not include a pve-version (e.g.
2763 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2764 $machine .= '+pve0';
2770 sub get_ovmf_files
($) {
2773 my $ovmf = $OVMF->{$arch}
2774 or die "no OVMF images known for architecture '$arch'\n";
2780 aarch64
=> '/usr/bin/qemu-system-aarch64',
2781 x86_64
=> '/usr/bin/qemu-system-x86_64',
2783 sub get_command_for_arch
($) {
2785 return '/usr/bin/kvm' if is_native
($arch);
2787 my $cmd = $Arch2Qemu->{$arch}
2788 or die "don't know how to emulate architecture '$arch'\n";
2792 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2793 # to use in a QEMU command line (-cpu element), first array_intersect the result
2794 # of query_supported_ with query_understood_. This is necessary because:
2796 # a) query_understood_ returns flags the host cannot use and
2797 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2798 # flags, but CPU settings - with most of them being flags. Those settings
2799 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
2801 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2802 # expensive. If you need the value returned from this, you can get it much
2803 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2804 # $accel being 'kvm' or 'tcg'.
2806 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2807 # changes, automatically populating pmxcfs.
2809 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2810 # since kvm and tcg machines support different flags
2812 sub query_supported_cpu_flags
{
2815 $arch //= get_host_arch
();
2816 my $default_machine = $default_machines->{$arch};
2820 # FIXME: Once this is merged, the code below should work for ARM as well:
2821 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2822 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2825 my $kvm_supported = defined(kvm_version
());
2826 my $qemu_cmd = get_command_for_arch
($arch);
2828 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
2830 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2831 my $query_supported_run_qemu = sub {
2837 '-machine', $default_machine,
2839 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2840 '-mon', 'chardev=qmp,mode=control',
2841 '-pidfile', $pidfile,
2846 push @$cmd, '-accel', 'tcg';
2849 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
2850 die "QEMU flag querying VM exited with code " . $rc if $rc;
2853 my $cmd_result = mon_cmd
(
2855 'query-cpu-model-expansion',
2857 model
=> { name
=> 'host' }
2860 my $props = $cmd_result->{model
}->{props
};
2861 foreach my $prop (keys %$props) {
2862 next if $props->{$prop} ne '1';
2863 # QEMU returns some flags multiple times, with '_', '.' or '-'
2864 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2865 # We only keep those with underscores, to match /proc/cpuinfo
2866 $prop =~ s/\.|-/_/g;
2867 $flags->{$prop} = 1;
2872 # force stop with 10 sec timeout and 'nocheck'
2873 # always stop, even if QMP failed
2874 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
2878 return [ sort keys %$flags ];
2881 # We need to query QEMU twice, since KVM and TCG have different supported flags
2882 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
2883 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
2884 warn "warning: failed querying supported tcg flags: $@\n" if $@;
2886 if ($kvm_supported) {
2887 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
2888 warn "warning: failed querying supported kvm flags: $@\n" if $@;
2895 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2896 my $understood_cpu_flag_dir = "/usr/share/kvm";
2897 sub query_understood_cpu_flags
{
2898 my $arch = get_host_arch
();
2899 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2901 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2904 my $raw = file_get_contents
($filepath);
2905 $raw =~ s/^\s+|\s+$//g;
2906 my @flags = split(/\s+/, $raw);
2911 sub config_to_command
{
2912 my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2915 my $globalFlags = [];
2916 my $machineFlags = [];
2921 my $ostype = $conf->{ostype
};
2922 my $winversion = windows_version
($ostype);
2923 my $kvm = $conf->{kvm
};
2924 my $nodename = nodename
();
2926 my $arch = get_vm_arch
($conf);
2927 my $kvm_binary = get_command_for_arch
($arch);
2928 my $kvmver = kvm_user_version
($kvm_binary);
2930 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2931 $kvmver //= "undefined";
2932 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2935 my $add_pve_version = min_version
($kvmver, 4, 1);
2937 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
2938 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, $kvmver);
2939 $kvm //= 1 if is_native
($arch);
2941 $machine_version =~ m/(\d+)\.(\d+)/;
2942 my ($machine_major, $machine_minor) = ($1, $2);
2943 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2944 if !PVE
::QemuServer
::min_version
($kvmver, $machine_major, $machine_minor);
2946 if (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
2947 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
2948 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2949 . " is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2952 # if a specific +pve version is required for a feature, use $version_guard
2953 # instead of min_version to allow machines to be run with the minimum
2955 my $required_pve_version = 0;
2956 my $version_guard = sub {
2957 my ($major, $minor, $pve) = @_;
2958 return 0 if !min_version
($machine_version, $major, $minor, $pve);
2959 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2964 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2965 if !defined kvm_version
();
2968 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2969 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
2970 my $use_old_bios_files = undef;
2971 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
2973 my $cpuunits = defined($conf->{cpuunits
}) ?
2974 $conf->{cpuunits
} : $defaults->{cpuunits
};
2976 push @$cmd, $kvm_binary;
2978 push @$cmd, '-id', $vmid;
2980 my $vmname = $conf->{name
} || "vm$vmid";
2982 push @$cmd, '-name', $vmname;
2986 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
2987 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2988 push @$cmd, '-mon', "chardev=qmp,mode=control";
2990 if (min_version
($machine_version, 2, 12)) {
2991 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
2992 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
2995 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
2997 push @$cmd, '-daemonize';
2999 if ($conf->{smbios1
}) {
3000 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3001 if ($smbios_conf->{base64
}) {
3002 # Do not pass base64 flag to qemu
3003 delete $smbios_conf->{base64
};
3004 my $smbios_string = "";
3005 foreach my $key (keys %$smbios_conf) {
3007 if ($key eq "uuid") {
3008 $value = $smbios_conf->{uuid
}
3010 $value = decode_base64
($smbios_conf->{$key});
3012 # qemu accepts any binary data, only commas need escaping by double comma
3014 $smbios_string .= "," . $key . "=" . $value if $value;
3016 push @$cmd, '-smbios', "type=1" . $smbios_string;
3018 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3022 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3023 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3024 die "uefi base image not found\n" if ! -f
$ovmf_code;
3028 if (my $efidisk = $conf->{efidisk0
}) {
3029 my $d = parse_drive
('efidisk0', $efidisk);
3030 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3031 $format = $d->{format
};
3033 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3034 if (!defined($format)) {
3035 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3036 $format = qemu_img_format
($scfg, $volname);
3040 die "efidisk format must be specified\n"
3041 if !defined($format);
3044 warn "no efidisk configured! Using temporary efivars disk.\n";
3045 $path = "/tmp/$vmid-ovmf.fd";
3046 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3050 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3051 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3056 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3057 if (min_version
($machine_version, 4, 0)) {
3058 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3060 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3064 if ($conf->{vmgenid
}) {
3065 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3068 # add usb controllers
3069 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
($conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3070 push @$devices, @usbcontrollers if @usbcontrollers;
3071 my $vga = parse_vga
($conf->{vga
});
3073 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3074 $vga->{type
} = 'qxl' if $qxlnum;
3076 if (!$vga->{type
}) {
3077 if ($arch eq 'aarch64') {
3078 $vga->{type
} = 'virtio';
3079 } elsif (min_version
($machine_version, 2, 9)) {
3080 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3082 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3086 # enable absolute mouse coordinates (needed by vnc)
3088 if (defined($conf->{tablet
})) {
3089 $tablet = $conf->{tablet
};
3091 $tablet = $defaults->{tablet
};
3092 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3093 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3097 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3098 my $kbd = print_keyboarddevice_full
($conf, $arch);
3099 push @$devices, '-device', $kbd if defined($kbd);
3103 my $gpu_passthrough;
3106 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
3107 my $id = "hostpci$i";
3108 my $d = parse_hostpci
($conf->{$id});
3111 if (my $pcie = $d->{pcie
}) {
3112 die "q35 machine model is not enabled" if !$q35;
3113 # win7 wants to have the pcie devices directly on the pcie bus
3114 # instead of in the root port
3115 if ($winversion == 7) {
3116 $pciaddr = print_pcie_addr
("${id}bus0");
3118 # add more root ports if needed, 4 are present by default
3119 # by pve-q35 cfgs, rest added here on demand.
3121 push @$devices, '-device', print_pcie_root_port
($i);
3123 $pciaddr = print_pcie_addr
($id);
3126 $pciaddr = print_pci_addr
($id, $bridges, $arch, $machine_type);
3130 if ($d->{'x-vga'}) {
3131 $xvga = ',x-vga=on' if !($conf->{bios
} && $conf->{bios
} eq 'ovmf');
3133 $vga->{type
} = 'none' if !defined($conf->{vga
});
3134 $gpu_passthrough = 1;
3137 my $pcidevices = $d->{pciid
};
3138 my $multifunction = 1 if @$pcidevices > 1;
3141 if ($d->{mdev
} && scalar(@$pcidevices) == 1) {
3142 my $pci_id = $pcidevices->[0]->{id
};
3143 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $i);
3144 $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3145 } elsif ($d->{mdev
}) {
3146 warn "ignoring mediated device '$id' with multifunction device\n";
3150 foreach my $pcidevice (@$pcidevices) {
3151 my $devicestr = "vfio-pci";
3154 $devicestr .= ",sysfsdev=$sysfspath";
3156 $devicestr .= ",host=$pcidevice->{id}";
3159 my $mf_addr = $multifunction ?
".$j" : '';
3160 $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3163 $devicestr .= ',rombar=0' if defined($d->{rombar
}) && !$d->{rombar
};
3164 $devicestr .= "$xvga";
3165 $devicestr .= ",multifunction=on" if $multifunction;
3166 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile
};
3169 push @$devices, '-device', $devicestr;
3175 my $usb_dev_features = {};
3176 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3178 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
($conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features);
3179 push @$devices, @usbdevices if @usbdevices;
3181 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3182 if (my $path = $conf->{"serial$i"}) {
3183 if ($path eq 'socket') {
3184 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3185 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3186 # On aarch64, serial0 is the UART device. Qemu only allows
3187 # connecting UART devices via the '-serial' command line, as
3188 # the device has a fixed slot on the hardware...
3189 if ($arch eq 'aarch64' && $i == 0) {
3190 push @$devices, '-serial', "chardev:serial$i";
3192 push @$devices, '-device', "isa-serial,chardev=serial$i";
3195 die "no such serial device\n" if ! -c
$path;
3196 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3197 push @$devices, '-device', "isa-serial,chardev=serial$i";
3203 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3204 if (my $path = $conf->{"parallel$i"}) {
3205 die "no such parallel device\n" if ! -c
$path;
3206 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3207 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3208 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3212 if (my $audio = conf_has_audio
($conf)) {
3214 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3216 my $id = $audio->{dev_id
};
3217 if ($audio->{dev
} eq 'AC97') {
3218 push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3219 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3220 push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3221 push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3222 push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3224 die "unkown audio device '$audio->{dev}', implement me!";
3227 push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3231 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3232 $sockets = $conf->{sockets
} if $conf->{sockets
};
3234 my $cores = $conf->{cores
} || 1;
3236 my $maxcpus = $sockets * $cores;
3238 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3240 my $allowed_vcpus = $cpuinfo->{cpus
};
3242 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3243 if ($allowed_vcpus < $maxcpus);
3245 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3247 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3248 for (my $i = 2; $i <= $vcpus; $i++) {
3249 my $cpustr = print_cpu_device
($conf,$i);
3250 push @$cmd, '-device', $cpustr;
3255 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3257 push @$cmd, '-nodefaults';
3259 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
3261 my $bootindex_hash = {};
3263 foreach my $o (split(//, $bootorder)) {
3264 $bootindex_hash->{$o} = $i*100;
3268 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3270 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3272 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3274 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3275 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3276 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3277 push @$cmd, '-vnc', "unix:$socket,password";
3279 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3280 push @$cmd, '-nographic';
3284 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3286 my $useLocaltime = $conf->{localtime};
3288 if ($winversion >= 5) { # windows
3289 $useLocaltime = 1 if !defined($conf->{localtime});
3291 # use time drift fix when acpi is enabled
3292 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3293 $tdf = 1 if !defined($conf->{tdf
});
3297 if ($winversion >= 6) {
3298 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3299 push @$cmd, '-no-hpet';
3302 push @$rtcFlags, 'driftfix=slew' if $tdf;
3304 if (($conf->{startdate
}) && ($conf->{startdate
} ne 'now')) {
3305 push @$rtcFlags, "base=$conf->{startdate}";
3306 } elsif ($useLocaltime) {
3307 push @$rtcFlags, 'base=localtime';
3310 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3312 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3314 push @$cmd, '-S' if $conf->{freeze
};
3316 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3318 my $guest_agent = parse_guest_agent
($conf);
3320 if ($guest_agent->{enabled
}) {
3321 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3322 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3324 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3325 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3326 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3327 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3328 } elsif ($guest_agent->{type
} eq 'isa') {
3329 push @$devices, '-device', "isa-serial,chardev=qga0";
3333 my $rng = parse_rng
($conf->{rng0
}) if $conf->{rng0
};
3334 if ($rng && &$version_guard(4, 1, 2)) {
3335 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3336 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3338 my $limiter_str = "";
3340 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3343 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3344 die "cannot create VirtIO RNG device: source file '$rng->{source}' doesn't exist\n"
3345 if ! -e
$rng->{source
};
3347 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3349 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3350 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3358 for(my $i = 1; $i < $qxlnum; $i++){
3359 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3362 # assume other OS works like Linux
3363 my ($ram, $vram) = ("134217728", "67108864");
3364 if ($vga->{memory
}) {
3365 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3366 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3368 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3369 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3373 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3375 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3376 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3377 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3379 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3380 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3381 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3383 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3384 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3386 my $spice_enhancement = PVE
::JSONSchema
::parse_property_string
($spice_enhancements_fmt, $conf->{spice_enhancements
} // '');
3387 if ($spice_enhancement->{foldersharing
}) {
3388 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3389 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3392 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3393 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming
};
3394 push @$devices, '-spice', "$spice_opts";
3397 # enable balloon by default, unless explicitly disabled
3398 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3399 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3400 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3403 if ($conf->{watchdog
}) {
3404 my $wdopts = parse_watchdog
($conf->{watchdog
});
3405 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3406 my $watchdog = $wdopts->{model
} || 'i6300esb';
3407 push @$devices, '-device', "$watchdog$pciaddr";
3408 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3412 my $scsicontroller = {};
3413 my $ahcicontroller = {};
3414 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3416 # Add iscsi initiator name if available
3417 if (my $initiator = get_initiator_name
()) {
3418 push @$devices, '-iscsi', "initiator-name=$initiator";
3421 foreach_drive
($conf, sub {
3422 my ($ds, $drive) = @_;
3424 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3425 push @$vollist, $drive->{file
};
3428 # ignore efidisk here, already added in bios/fw handling code above
3429 return if $drive->{interface
} eq 'efidisk';
3431 $use_virtio = 1 if $ds =~ m/^virtio/;
3433 if (drive_is_cdrom
($drive)) {
3434 if ($bootindex_hash->{d
}) {
3435 $drive->{bootindex
} = $bootindex_hash->{d
};
3436 $bootindex_hash->{d
} += 1;
3439 if ($bootindex_hash->{c
}) {
3440 $drive->{bootindex
} = $bootindex_hash->{c
} if $conf->{bootdisk
} && ($conf->{bootdisk
} eq $ds);
3441 $bootindex_hash->{c
} += 1;
3445 if($drive->{interface
} eq 'virtio'){
3446 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3449 if ($drive->{interface
} eq 'scsi') {
3451 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3453 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3454 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3456 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3457 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3460 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3461 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3462 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3463 } elsif ($drive->{iothread
}) {
3464 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3468 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3469 $queues = ",num_queues=$drive->{queues}";
3472 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3473 $scsicontroller->{$controller}=1;
3476 if ($drive->{interface
} eq 'sata') {
3477 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3478 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3479 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3480 $ahcicontroller->{$controller}=1;
3483 my $drive_cmd = print_drive_full
($storecfg, $vmid, $drive);
3484 push @$devices, '-drive',$drive_cmd;
3485 push @$devices, '-device', print_drivedevice_full
($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3488 for (my $i = 0; $i < $MAX_NETS; $i++) {
3489 next if !$conf->{"net$i"};
3490 my $d = parse_net
($conf->{"net$i"});
3493 $use_virtio = 1 if $d->{model
} eq 'virtio';
3495 if ($bootindex_hash->{n
}) {
3496 $d->{bootindex
} = $bootindex_hash->{n
};
3497 $bootindex_hash->{n
} += 1;
3500 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, "net$i");
3501 push @$devices, '-netdev', $netdevfull;
3503 my $netdevicefull = print_netdevice_full
($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3504 push @$devices, '-device', $netdevicefull;
3507 if ($conf->{ivshmem
}) {
3508 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3512 $bus = print_pcie_addr
("ivshmem");
3514 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3517 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3518 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3520 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3521 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3524 # pci.4 is nested in pci.1
3525 $bridges->{1} = 1 if $bridges->{4};
3529 if (min_version
($machine_version, 2, 3)) {
3534 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3538 for my $k (sort {$b cmp $a} keys %$bridges) {
3539 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3540 $pciaddr = print_pci_addr
("pci.$k", undef, $arch, $machine_type);
3541 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3543 # add after -readconfig pve-q35.cfg
3544 splice @$devices, 2, 0, '-device', $devstr;
3546 unshift @$devices, '-device', $devstr if $k > 0;
3551 push @$machineFlags, 'accel=tcg';
3554 my $machine_type_min = $machine_type;
3555 if ($add_pve_version) {
3556 $machine_type_min =~ s/\+pve\d+$//;
3557 $machine_type_min .= "+pve$required_pve_version";
3559 push @$machineFlags, "type=${machine_type_min}";
3561 push @$cmd, @$devices;
3562 push @$cmd, '-rtc', join(',', @$rtcFlags)
3563 if scalar(@$rtcFlags);
3564 push @$cmd, '-machine', join(',', @$machineFlags)
3565 if scalar(@$machineFlags);
3566 push @$cmd, '-global', join(',', @$globalFlags)
3567 if scalar(@$globalFlags);
3569 if (my $vmstate = $conf->{vmstate
}) {
3570 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3571 push @$vollist, $vmstate;
3572 push @$cmd, '-loadstate', $statepath;
3573 print "activating and using '$vmstate' as vmstate\n";
3577 if ($conf->{args
}) {
3578 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3582 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3588 my $res = mon_cmd
($vmid, 'query-spice');
3590 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3593 sub vm_devices_list
{
3596 my $res = mon_cmd
($vmid, 'query-pci');
3597 my $devices_to_check = [];
3599 foreach my $pcibus (@$res) {
3600 push @$devices_to_check, @{$pcibus->{devices
}},
3603 while (@$devices_to_check) {
3605 for my $d (@$devices_to_check) {
3606 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3607 next if !$d->{'pci_bridge'};
3609 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3610 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3612 $devices_to_check = $to_check;
3615 my $resblock = mon_cmd
($vmid, 'query-block');
3616 foreach my $block (@$resblock) {
3617 if($block->{device
} =~ m/^drive-(\S+)/){
3622 my $resmice = mon_cmd
($vmid, 'query-mice');
3623 foreach my $mice (@$resmice) {
3624 if ($mice->{name
} eq 'QEMU HID Tablet') {
3625 $devices->{tablet
} = 1;
3630 # for usb devices there is no query-usb
3631 # but we can iterate over the entries in
3632 # qom-list path=/machine/peripheral
3633 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3634 foreach my $per (@$resperipheral) {
3635 if ($per->{name
} =~ m/^usb\d+$/) {
3636 $devices->{$per->{name
}} = 1;
3644 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3646 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3648 my $devices_list = vm_devices_list
($vmid);
3649 return 1 if defined($devices_list->{$deviceid});
3651 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3653 if ($deviceid eq 'tablet') {
3655 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
3657 } elsif ($deviceid eq 'keyboard') {
3659 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
3661 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3663 die "usb hotplug currently not reliable\n";
3664 # since we can't reliably hot unplug all added usb devices
3665 # and usb passthrough disables live migration
3666 # we disable usb hotplugging for now
3667 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device));
3669 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3671 qemu_iothread_add
($vmid, $deviceid, $device);
3673 qemu_driveadd
($storecfg, $vmid, $device);
3674 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3676 qemu_deviceadd
($vmid, $devicefull);
3677 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3679 eval { qemu_drivedel
($vmid, $deviceid); };
3684 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3687 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3688 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3689 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3691 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3693 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3694 qemu_iothread_add
($vmid, $deviceid, $device);
3695 $devicefull .= ",iothread=iothread-$deviceid";
3698 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3699 $devicefull .= ",num_queues=$device->{queues}";
3702 qemu_deviceadd
($vmid, $devicefull);
3703 qemu_deviceaddverify
($vmid, $deviceid);
3705 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3707 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3708 qemu_driveadd
($storecfg, $vmid, $device);
3710 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3711 eval { qemu_deviceadd
($vmid, $devicefull); };
3713 eval { qemu_drivedel
($vmid, $deviceid); };
3718 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3720 return undef if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
3722 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
3723 my $use_old_bios_files = undef;
3724 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3726 my $netdevicefull = print_netdevice_full
($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3727 qemu_deviceadd
($vmid, $netdevicefull);
3729 qemu_deviceaddverify
($vmid, $deviceid);
3730 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
3733 eval { qemu_netdevdel
($vmid, $deviceid); };
3738 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3741 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3742 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3744 qemu_deviceadd
($vmid, $devicefull);
3745 qemu_deviceaddverify
($vmid, $deviceid);
3748 die "can't hotplug device '$deviceid'\n";
3754 # fixme: this should raise exceptions on error!
3755 sub vm_deviceunplug
{
3756 my ($vmid, $conf, $deviceid) = @_;
3758 my $devices_list = vm_devices_list
($vmid);
3759 return 1 if !defined($devices_list->{$deviceid});
3761 die "can't unplug bootdisk" if $conf->{bootdisk
} && $conf->{bootdisk
} eq $deviceid;
3763 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3765 qemu_devicedel
($vmid, $deviceid);
3767 } elsif ($deviceid =~ m/^usb\d+$/) {
3769 die "usb hotplug currently not reliable\n";
3770 # when unplugging usb devices this way,
3771 # there may be remaining usb controllers/hubs
3772 # so we disable it for now
3773 qemu_devicedel
($vmid, $deviceid);
3774 qemu_devicedelverify
($vmid, $deviceid);
3776 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3778 qemu_devicedel
($vmid, $deviceid);
3779 qemu_devicedelverify
($vmid, $deviceid);
3780 qemu_drivedel
($vmid, $deviceid);
3781 qemu_iothread_del
($conf, $vmid, $deviceid);
3783 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3785 qemu_devicedel
($vmid, $deviceid);
3786 qemu_devicedelverify
($vmid, $deviceid);
3787 qemu_iothread_del
($conf, $vmid, $deviceid);
3789 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3791 qemu_devicedel
($vmid, $deviceid);
3792 qemu_drivedel
($vmid, $deviceid);
3793 qemu_deletescsihw
($conf, $vmid, $deviceid);
3795 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3797 qemu_devicedel
($vmid, $deviceid);
3798 qemu_devicedelverify
($vmid, $deviceid);
3799 qemu_netdevdel
($vmid, $deviceid);
3802 die "can't unplug device '$deviceid'\n";
3808 sub qemu_deviceadd
{
3809 my ($vmid, $devicefull) = @_;
3811 $devicefull = "driver=".$devicefull;
3812 my %options = split(/[=,]/, $devicefull);
3814 mon_cmd
($vmid, "device_add" , %options);
3817 sub qemu_devicedel
{
3818 my ($vmid, $deviceid) = @_;
3820 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
3823 sub qemu_iothread_add
{
3824 my($vmid, $deviceid, $device) = @_;
3826 if ($device->{iothread
}) {
3827 my $iothreads = vm_iothreads_list
($vmid);
3828 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3832 sub qemu_iothread_del
{
3833 my($conf, $vmid, $deviceid) = @_;
3835 my $confid = $deviceid;
3836 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3837 $confid = 'scsi' . $1;
3839 my $device = parse_drive
($confid, $conf->{$confid});
3840 if ($device->{iothread
}) {
3841 my $iothreads = vm_iothreads_list
($vmid);
3842 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3846 sub qemu_objectadd
{
3847 my($vmid, $objectid, $qomtype) = @_;
3849 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
3854 sub qemu_objectdel
{
3855 my($vmid, $objectid) = @_;
3857 mon_cmd
($vmid, "object-del", id
=> $objectid);
3863 my ($storecfg, $vmid, $device) = @_;
3865 my $drive = print_drive_full
($storecfg, $vmid, $device);
3866 $drive =~ s/\\/\\\\/g;
3867 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
3869 # If the command succeeds qemu prints: "OK
"
3870 return 1 if $ret =~ m/OK/s;
3872 die "adding drive failed
: $ret\n";
3876 my($vmid, $deviceid) = @_;
3878 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
3881 return 1 if $ret eq "";
3883 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3884 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3886 die "deleting drive
$deviceid failed
: $ret\n";
3889 sub qemu_deviceaddverify {
3890 my ($vmid, $deviceid) = @_;
3892 for (my $i = 0; $i <= 5; $i++) {
3893 my $devices_list = vm_devices_list($vmid);
3894 return 1 if defined($devices_list->{$deviceid});
3898 die "error on hotplug device
'$deviceid'\n";
3902 sub qemu_devicedelverify {
3903 my ($vmid, $deviceid) = @_;
3905 # need to verify that the device is correctly removed as device_del
3906 # is async and empty return is not reliable
3908 for (my $i = 0; $i <= 5; $i++) {
3909 my $devices_list = vm_devices_list($vmid);
3910 return 1 if !defined($devices_list->{$deviceid});
3914 die "error on hot-unplugging device
'$deviceid'\n";
3917 sub qemu_findorcreatescsihw {
3918 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3920 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3922 my $scsihwid="$controller_prefix$controller";
3923 my $devices_list = vm_devices_list($vmid);
3925 if(!defined($devices_list->{$scsihwid})) {
3926 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3932 sub qemu_deletescsihw {
3933 my ($conf, $vmid, $opt) = @_;
3935 my $device = parse_drive($opt, $conf->{$opt});
3937 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3938 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
3942 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3944 my $devices_list = vm_devices_list($vmid);
3945 foreach my $opt (keys %{$devices_list}) {
3946 if (is_valid_drivename($opt)) {
3947 my $drive = parse_drive($opt, $conf->{$opt});
3948 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3954 my $scsihwid="scsihw
$controller";
3956 vm_deviceunplug($vmid, $conf, $scsihwid);
3961 sub qemu_add_pci_bridge {
3962 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3968 print_pci_addr($device, $bridges, $arch, $machine_type);
3970 while (my ($k, $v) = each %$bridges) {
3973 return 1 if !defined($bridgeid) || $bridgeid < 1;
3975 my $bridge = "pci
.$bridgeid";
3976 my $devices_list = vm_devices_list($vmid);
3978 if (!defined($devices_list->{$bridge})) {
3979 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3985 sub qemu_set_link_status {
3986 my ($vmid, $device, $up) = @_;
3988 mon_cmd($vmid, "set_link
", name => $device,
3989 up => $up ? JSON::true : JSON::false);
3992 sub qemu_netdevadd {
3993 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
3995 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
3996 my %options = split(/[=,]/, $netdev);
3998 mon_cmd($vmid, "netdev_add
", %options);
4002 sub qemu_netdevdel {
4003 my ($vmid, $deviceid) = @_;
4005 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4008 sub qemu_usb_hotplug {
4009 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4013 # remove the old one first
4014 vm_deviceunplug($vmid, $conf, $deviceid);
4016 # check if xhci controller is necessary and available
4017 if ($device->{usb3}) {
4019 my $devicelist = vm_devices_list($vmid);
4021 if (!$devicelist->{xhci}) {
4022 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4023 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4026 my $d = parse_usb_device($device->{host});
4027 $d->{usb3} = $device->{usb3};
4030 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4033 sub qemu_cpu_hotplug {
4034 my ($vmid, $conf, $vcpus) = @_;
4036 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4039 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4040 $sockets = $conf->{sockets} if $conf->{sockets};
4041 my $cores = $conf->{cores} || 1;
4042 my $maxcpus = $sockets * $cores;
4044 $vcpus = $maxcpus if !$vcpus;
4046 die "you can
't add more vcpus than maxcpus\n"
4047 if $vcpus > $maxcpus;
4049 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4051 if ($vcpus < $currentvcpus) {
4053 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4055 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4056 qemu_devicedel($vmid, "cpu$i");
4058 my $currentrunningvcpus = undef;
4060 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4061 last if scalar(@{$currentrunningvcpus}) == $i-1;
4062 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4066 #update conf after each succesfull cpu unplug
4067 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4068 PVE::QemuConfig->write_config($vmid, $conf);
4071 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4077 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4078 die "vcpus in running vm does not match its configuration\n"
4079 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4081 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4083 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4084 my $cpustr = print_cpu_device($conf, $i);
4085 qemu_deviceadd($vmid, $cpustr);
4088 my $currentrunningvcpus = undef;
4090 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4091 last if scalar(@{$currentrunningvcpus}) == $i;
4092 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4096 #update conf after each succesfull cpu hotplug
4097 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4098 PVE::QemuConfig->write_config($vmid, $conf);
4102 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4103 mon_cmd($vmid, "cpu-add", id => int($i));
4108 sub qemu_block_set_io_throttle {
4109 my ($vmid, $deviceid,
4110 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4111 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4112 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4113 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4115 return if !check_running($vmid) ;
4117 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4119 bps_rd => int($bps_rd),
4120 bps_wr => int($bps_wr),
4122 iops_rd => int($iops_rd),
4123 iops_wr => int($iops_wr),
4124 bps_max => int($bps_max),
4125 bps_rd_max => int($bps_rd_max),
4126 bps_wr_max => int($bps_wr_max),
4127 iops_max => int($iops_max),
4128 iops_rd_max => int($iops_rd_max),
4129 iops_wr_max => int($iops_wr_max),
4130 bps_max_length => int($bps_max_length),
4131 bps_rd_max_length => int($bps_rd_max_length),
4132 bps_wr_max_length => int($bps_wr_max_length),
4133 iops_max_length => int($iops_max_length),
4134 iops_rd_max_length => int($iops_rd_max_length),
4135 iops_wr_max_length => int($iops_wr_max_length),
4140 # old code, only used to shutdown old VM after update
4142 my ($fh, $timeout) = @_;
4144 my $sel = new IO::Select;
4151 while (scalar (@ready = $sel->can_read($timeout))) {
4153 if ($count = $fh->sysread($buf, 8192)) {
4154 if ($buf =~ /^(.*)\(qemu\) $/s) {
4161 if (!defined($count)) {
4168 die "monitor read timeout\n" if !scalar(@ready);
4173 sub qemu_block_resize {
4174 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4176 my $running = check_running($vmid);
4178 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4180 return if !$running;
4182 my $padding = (1024 - $size % 1024) % 1024;
4183 $size = $size + $padding;
4185 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4189 sub qemu_volume_snapshot {
4190 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4192 my $running = check_running($vmid);
4194 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4195 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4197 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4201 sub qemu_volume_snapshot_delete {
4202 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4204 my $running = check_running($vmid);
4209 my $conf = PVE::QemuConfig->load_config($vmid);
4210 foreach_drive($conf, sub {
4211 my ($ds, $drive) = @_;
4212 $running = 1 if $drive->{file} eq $volid;
4216 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4217 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4219 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4223 sub set_migration_caps {
4229 "auto-converge" => 1,
4231 "x-rdma-pin-all" => 0,
4236 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4238 for my $supported_capability (@$supported_capabilities) {
4240 capability => $supported_capability->{capability},
4241 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4245 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4248 my $fast_plug_option = {
4256 'vmstatestorage
' => 1,
4261 # hotplug changes in [PENDING]
4262 # $selection hash can be used to only apply specified options, for
4263 # example: { cores => 1 } (only apply changed 'cores
')
4264 # $errors ref is used to return error messages
4265 sub vmconfig_hotplug_pending {
4266 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4268 my $defaults = load_defaults();
4269 my $arch = get_vm_arch($conf);
4270 my $machine_type = get_vm_machine($conf, undef, $arch);
4272 # commit values which do not have any impact on running VM first
4273 # Note: those option cannot raise errors, we we do not care about
4274 # $selection and always apply them.
4276 my $add_error = sub {
4277 my ($opt, $msg) = @_;
4278 $errors->{$opt} = "hotplug problem - $msg";
4282 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4283 if ($fast_plug_option->{$opt}) {
4284 $conf->{$opt} = $conf->{pending}->{$opt};
4285 delete $conf->{pending}->{$opt};
4291 PVE::QemuConfig->write_config($vmid, $conf);
4294 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4296 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4297 foreach my $opt (sort keys %$pending_delete_hash) {
4298 next if $selection && !$selection->{$opt};
4299 my $force = $pending_delete_hash->{$opt}->{force};
4301 if ($opt eq 'hotplug
') {
4302 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4303 } elsif ($opt eq 'tablet
') {
4304 die "skip\n" if !$hotplug_features->{usb};
4305 if ($defaults->{tablet}) {
4306 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4307 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4308 if $arch eq 'aarch64
';
4310 vm_deviceunplug($vmid, $conf, 'tablet
');
4311 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4313 } elsif ($opt =~ m/^usb\d+/) {
4315 # since we cannot reliably hot unplug usb devices
4316 # we are disabling it
4317 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4318 vm_deviceunplug($vmid, $conf, $opt);
4319 } elsif ($opt eq 'vcpus
') {
4320 die "skip\n" if !$hotplug_features->{cpu};
4321 qemu_cpu_hotplug($vmid, $conf, undef);
4322 } elsif ($opt eq 'balloon
') {
4323 # enable balloon device is not hotpluggable
4324 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4325 # here we reset the ballooning value to memory
4326 my $balloon = $conf->{memory} || $defaults->{memory};
4327 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4328 } elsif ($fast_plug_option->{$opt}) {
4330 } elsif ($opt =~ m/^net(\d+)$/) {
4331 die "skip\n" if !$hotplug_features->{network};
4332 vm_deviceunplug($vmid, $conf, $opt);
4333 } elsif (is_valid_drivename($opt)) {
4334 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4335 vm_deviceunplug($vmid, $conf, $opt);
4336 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4337 } elsif ($opt =~ m/^memory$/) {
4338 die "skip\n" if !$hotplug_features->{memory};
4339 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4340 } elsif ($opt eq 'cpuunits
') {
4341 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4342 } elsif ($opt eq 'cpulimit
') {
4343 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4349 &$add_error($opt, $err) if $err ne "skip\n";
4351 delete $conf->{$opt};
4352 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4356 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4357 $apply_pending_cloudinit = sub {
4358 return if $apply_pending_cloudinit_done; # once is enough
4359 $apply_pending_cloudinit_done = 1; # once is enough
4361 my ($key, $value) = @_;
4363 my @cloudinit_opts = keys %$confdesc_cloudinit;
4364 foreach my $opt (keys %{$conf->{pending}}) {
4365 next if !grep { $_ eq $opt } @cloudinit_opts;
4366 $conf->{$opt} = delete $conf->{pending}->{$opt};
4369 my $new_conf = { %$conf };
4370 $new_conf->{$key} = $value;
4371 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4374 foreach my $opt (keys %{$conf->{pending}}) {
4375 next if $selection && !$selection->{$opt};
4376 my $value = $conf->{pending}->{$opt};
4378 if ($opt eq 'hotplug
') {
4379 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4380 } elsif ($opt eq 'tablet
') {
4381 die "skip\n" if !$hotplug_features->{usb};
4383 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4384 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4385 if $arch eq 'aarch64
';
4386 } elsif ($value == 0) {
4387 vm_deviceunplug($vmid, $conf, 'tablet
');
4388 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4390 } elsif ($opt =~ m/^usb\d+$/) {
4392 # since we cannot reliably hot unplug usb devices
4393 # we are disabling it
4394 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4395 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4396 die "skip\n" if !$d;
4397 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4398 } elsif ($opt eq 'vcpus
') {
4399 die "skip\n" if !$hotplug_features->{cpu};
4400 qemu_cpu_hotplug($vmid, $conf, $value);
4401 } elsif ($opt eq 'balloon
') {
4402 # enable/disable balloning device is not hotpluggable
4403 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4404 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4405 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4407 # allow manual ballooning if shares is set to zero
4408 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4409 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4410 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4412 } elsif ($opt =~ m/^net(\d+)$/) {
4413 # some changes can be done without hotplug
4414 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4415 $vmid, $opt, $value, $arch, $machine_type);
4416 } elsif (is_valid_drivename($opt)) {
4417 die "skip\n" if $opt eq 'efidisk0
';
4418 # some changes can be done without hotplug
4419 my $drive = parse_drive($opt, $value);
4420 if (drive_is_cloudinit($drive)) {
4421 &$apply_pending_cloudinit($opt, $value);
4423 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4424 $vmid, $opt, $value, $arch, $machine_type);
4425 } elsif ($opt =~ m/^memory$/) { #dimms
4426 die "skip\n" if !$hotplug_features->{memory};
4427 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4428 } elsif ($opt eq 'cpuunits
') {
4429 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4430 } elsif ($opt eq 'cpulimit
') {
4431 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4432 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4434 die "skip\n"; # skip non-hot-pluggable options
4438 &$add_error($opt, $err) if $err ne "skip\n";
4440 $conf->{$opt} = $value;
4441 delete $conf->{pending}->{$opt};
4445 PVE::QemuConfig->write_config($vmid, $conf);
4448 sub try_deallocate_drive {
4449 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4451 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4452 my $volid = $drive->{file};
4453 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4454 my $sid = PVE::Storage::parse_volume_id($volid);
4455 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4457 # check if the disk is really unused
4458 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4459 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4460 PVE::Storage::vdisk_free($storecfg, $volid);
4463 # If vm is not owner of this disk remove from config
4471 sub vmconfig_delete_or_detach_drive {
4472 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4474 my $drive = parse_drive($opt, $conf->{$opt});
4476 my $rpcenv = PVE::RPCEnvironment::get();
4477 my $authuser = $rpcenv->get_user();
4480 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4481 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4483 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4489 sub vmconfig_apply_pending {
4490 my ($vmid, $conf, $storecfg, $errors) = @_;
4492 my $add_apply_error = sub {
4493 my ($opt, $msg) = @_;
4494 my $err_msg = "unable to apply pending change $opt : $msg";
4495 $errors->{$opt} = $err_msg;
4501 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4502 foreach my $opt (sort keys %$pending_delete_hash) {
4503 my $force = $pending_delete_hash->{$opt}->{force};
4505 if ($opt =~ m/^unused/) {
4506 die "internal error";
4507 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4508 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4512 $add_apply_error->($opt, $err);
4514 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4515 delete $conf->{$opt};
4519 PVE::QemuConfig->cleanup_pending($conf);
4521 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4522 next if $opt eq 'delete'; # just to be sure
4524 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4525 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4529 $add_apply_error->($opt, $err);
4531 $conf->{$opt} = delete $conf->{pending}->{$opt};
4535 # write all changes at once to avoid unnecessary i/o
4536 PVE::QemuConfig->write_config($vmid, $conf);
4539 my $safe_num_ne = sub {
4542 return 0 if !defined($a) && !defined($b);
4543 return 1 if !defined($a);
4544 return 1 if !defined($b);
4549 my $safe_string_ne = sub {
4552 return 0 if !defined($a) && !defined($b);
4553 return 1 if !defined($a);
4554 return 1 if !defined($b);
4559 sub vmconfig_update_net {
4560 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4562 my $newnet = parse_net($value);
4564 if ($conf->{$opt}) {
4565 my $oldnet = parse_net($conf->{$opt});
4567 if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4568 &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4569 &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4570 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4572 # for non online change, we try to hot-unplug
4573 die "skip\n" if !$hotplug;
4574 vm_deviceunplug($vmid, $conf, $opt);
4577 die "internal error" if $opt !~ m/net(\d+)/;
4578 my $iface = "tap${vmid}i$1";
4580 if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4581 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4582 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4583 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4584 PVE::Network::tap_unplug($iface);
4585 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4586 } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4587 # Rate can be applied on its own but any change above needs to
4588 # include the rate in tap_plug since OVS resets everything.
4589 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4592 if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4593 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4601 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4607 sub vmconfig_update_disk {
4608 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4610 my $drive = parse_drive($opt, $value);
4612 if ($conf->{$opt}) {
4614 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
4616 my $media = $drive->{media} || 'disk
';
4617 my $oldmedia = $old_drive->{media} || 'disk
';
4618 die "unable to change media type\n" if $media ne $oldmedia;
4620 if (!drive_is_cdrom($old_drive)) {
4622 if ($drive->{file} ne $old_drive->{file}) {
4624 die "skip\n" if !$hotplug;
4626 # unplug and register as unused
4627 vm_deviceunplug($vmid, $conf, $opt);
4628 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4631 # update existing disk
4633 # skip non hotpluggable value
4634 if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4635 &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4636 &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4637 &$safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4638 &$safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4643 if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4644 &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4645 &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4646 &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4647 &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4648 &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4649 &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4650 &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4651 &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4652 &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4653 &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4654 &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4655 &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4656 &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4657 &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4658 &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4659 &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4660 &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4662 qemu_block_set_io_throttle($vmid,"drive-$opt",
4663 ($drive->{mbps} || 0)*1024*1024,
4664 ($drive->{mbps_rd} || 0)*1024*1024,
4665 ($drive->{mbps_wr} || 0)*1024*1024,
4666 $drive->{iops} || 0,
4667 $drive->{iops_rd} || 0,
4668 $drive->{iops_wr} || 0,
4669 ($drive->{mbps_max} || 0)*1024*1024,
4670 ($drive->{mbps_rd_max} || 0)*1024*1024,
4671 ($drive->{mbps_wr_max} || 0)*1024*1024,
4672 $drive->{iops_max} || 0,
4673 $drive->{iops_rd_max} || 0,
4674 $drive->{iops_wr_max} || 0,
4675 $drive->{bps_max_length} || 1,
4676 $drive->{bps_rd_max_length} || 1,
4677 $drive->{bps_wr_max_length} || 1,
4678 $drive->{iops_max_length} || 1,
4679 $drive->{iops_rd_max_length} || 1,
4680 $drive->{iops_wr_max_length} || 1);
4689 if ($drive->{file} eq 'none
') {
4690 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4691 if (drive_is_cloudinit($old_drive)) {
4692 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4695 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4697 # force eject if locked
4698 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4701 mon_cmd($vmid, "blockdev-change-medium",
4702 id => "$opt", filename => "$path");
4711 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4713 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4714 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4718 my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4719 $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage, $timeout) = @_;
4721 PVE::QemuConfig->lock_config($vmid, sub {
4722 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4724 die "you can't start a vm
if it
's a template\n" if PVE::QemuConfig->is_template($conf);
4726 my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended
');
4728 PVE::QemuConfig->check_lock($conf)
4729 if !($skiplock || $is_suspended);
4731 die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4733 # clean up leftover reboot request files
4734 eval { clear_reboot_request($vmid); };
4737 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4738 vmconfig_apply_pending($vmid, $conf, $storecfg);
4739 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4742 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4744 my $defaults = load_defaults();
4746 # set environment variable useful inside network script
4747 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4749 my $local_volumes = {};
4751 if ($targetstorage) {
4752 foreach_drive($conf, sub {
4753 my ($ds, $drive) = @_;
4755 return if drive_is_cdrom($drive);
4757 my $volid = $drive->{file};
4761 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4763 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4764 return if $scfg->{shared};
4765 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4770 foreach my $opt (sort keys %$local_volumes) {
4772 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4773 my $drive = parse_drive($opt, $conf->{$opt});
4775 # If a remote storage is specified and the format of the original
4776 # volume is not available there, fall back to the default format.
4777 # Otherwise use the same format as the original.
4778 if ($targetstorage && $targetstorage ne "1") {
4779 $storeid = $targetstorage;
4780 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4781 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4782 my $fileFormat = qemu_img_format($scfg, $volname);
4783 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4785 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4786 $format = qemu_img_format($scfg, $volname);
4789 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4790 my $newdrive = $drive;
4791 $newdrive->{format} = $format;
4792 $newdrive->{file} = $newvolid;
4793 my $drivestr = print_drive($newdrive);
4794 $local_volumes->{$opt} = $drivestr;
4795 #pass drive to conf for command line
4796 $conf->{$opt} = $drivestr;
4800 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
4802 if ($is_suspended) {
4803 # enforce machine type on suspended vm to ensure HW compatibility
4804 $forcemachine = $conf->{runningmachine};
4805 print "Resuming suspended VM\n";
4808 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4811 my $get_migration_ip = sub {
4812 my ($cidr, $nodename) = @_;
4814 return $migration_ip if defined($migration_ip);
4816 if (!defined($cidr)) {
4817 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
4818 $cidr = $dc_conf->{migration}->{network};
4821 if (defined($cidr)) {
4822 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4824 die "could not get IP: no address configured on local " .
4825 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4827 die "could not get IP: multiple addresses configured on local " .
4828 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4830 $migration_ip = @$ips[0];
4833 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4834 if !defined($migration_ip);
4836 return $migration_ip;
4841 if ($statefile eq 'tcp
') {
4842 my $localip = "localhost";
4843 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
4844 my $nodename = nodename();
4846 if (!defined($migration_type)) {
4847 if (defined($datacenterconf->{migration}->{type})) {
4848 $migration_type = $datacenterconf->{migration}->{type};
4850 $migration_type = 'secure
';
4854 if ($migration_type eq 'insecure
') {
4855 $localip = $get_migration_ip->($migration_network, $nodename);
4856 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4859 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4860 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4861 $migrate_uri = "tcp:${localip}:${migrate_port}";
4862 push @$cmd, '-incoming
', $migrate_uri;
4865 } elsif ($statefile eq 'unix
') {
4866 # should be default for secure migrations as a ssh TCP forward
4867 # tunnel is not deterministic reliable ready and fails regurarly
4868 # to set up in time, so use UNIX socket forwards
4869 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4870 unlink $socket_addr;
4872 $migrate_uri = "unix:$socket_addr";
4874 push @$cmd, '-incoming
', $migrate_uri;
4877 } elsif (-e $statefile) {
4878 push @$cmd, '-loadstate
', $statefile;
4880 my $statepath = PVE::Storage::path($storecfg, $statefile);
4881 push @$vollist, $statefile;
4882 push @$cmd, '-loadstate
', $statepath;
4889 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
4890 my $d = parse_hostpci($conf->{"hostpci$i"});
4892 my $pcidevices = $d->{pciid};
4893 foreach my $pcidevice (@$pcidevices) {
4894 my $pciid = $pcidevice->{id};
4896 my $info = PVE::SysFSTools::pci_device_info("$pciid");
4897 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4898 die "no pci device info for device '$pciid'\n" if !$info;
4901 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4902 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4904 die "can't unbind
/bind pci group to vfio
'$pciid'\n"
4905 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4906 die "can
't reset pci device '$pciid'\n"
4907 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4912 PVE::Storage::activate_volumes($storecfg, $vollist);
4915 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
4916 outfunc => sub {}, errfunc => sub {});
4918 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
4919 # timeout should be more than enough here...
4920 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4922 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4923 : $defaults->{cpuunits};
4925 my $start_timeout = $timeout // config_aware_timeout($conf, $is_suspended);
4927 timeout => $statefile ? undef : $start_timeout,
4932 # when migrating, prefix QEMU output so other side can pick up any
4933 # errors that might occur and show the user
4934 if ($migratedfrom) {
4935 $run_params{quiet} = 1;
4936 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
4940 Slice => 'qemu
.slice
',
4942 CPUShares => $cpuunits
4945 if (my $cpulimit = $conf->{cpulimit}) {
4946 $properties{CPUQuota} = int($cpulimit * 100);
4948 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4950 my $run_qemu = sub {
4951 PVE::Tools::run_fork sub {
4952 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4954 my $exitcode = run_command($cmd, %run_params);
4955 die "QEMU exited with code $exitcode\n" if $exitcode;
4959 if ($conf->{hugepages}) {
4962 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4963 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4965 PVE::QemuServer::Memory::hugepages_mount();
4966 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4968 eval { $run_qemu->() };
4970 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4974 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4976 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4979 eval { $run_qemu->() };
4983 # deactivate volumes if start fails
4984 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4985 die "start failed: $err";
4988 print "migration listens on $migrate_uri\n" if $migrate_uri;
4990 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
4991 eval { mon_cmd($vmid, "cont"); };
4995 #start nbd server for storage migration
4996 if ($targetstorage) {
4997 my $nodename = nodename();
4998 my $localip = $get_migration_ip->($migration_network, $nodename);
4999 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5000 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5002 mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet
', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5004 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5006 foreach my $opt (sort keys %$local_volumes) {
5007 my $drivestr = $local_volumes->{$opt};
5008 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5009 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5010 print "storage migration listens on $migrate_storage_uri volume:$drivestr\n";
5014 if ($migratedfrom) {
5016 set_migration_caps($vmid);
5021 print "spice listens on port $spice_port\n";
5022 if ($spice_ticket) {
5023 mon_cmd($vmid, "set_password", protocol => 'spice
', password => $spice_ticket);
5024 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5029 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5030 if !$statefile && $conf->{balloon};
5032 foreach my $opt (keys %$conf) {
5033 next if $opt !~ m/^net\d+$/;
5034 my $nicconf = parse_net($conf->{$opt});
5035 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5039 mon_cmd($vmid, 'qom-set
',
5040 path => "machine/peripheral/balloon0",
5041 property => "guest-stats-polling-interval",
5042 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5044 if ($is_suspended) {
5045 print "Resumed VM, removing state\n";
5046 if (my $vmstate = $conf->{vmstate}) {
5047 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5048 PVE::Storage::vdisk_free($storecfg, $vmstate);
5050 delete $conf->@{qw(lock vmstate runningmachine)};
5051 PVE
::QemuConfig-
>write_config($vmid, $conf);
5054 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5058 sub vm_commandline
{
5059 my ($storecfg, $vmid, $snapname) = @_;
5061 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5065 my $snapshot = $conf->{snapshots
}->{$snapname};
5066 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5068 # check for a 'runningmachine' in snapshot
5069 $forcemachine = $snapshot->{runningmachine
} if $snapshot->{runningmachine
};
5071 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5076 my $defaults = load_defaults
();
5078 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine);
5080 return PVE
::Tools
::cmd2string
($cmd);
5084 my ($vmid, $skiplock) = @_;
5086 PVE
::QemuConfig-
>lock_config($vmid, sub {
5088 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5090 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5092 mon_cmd
($vmid, "system_reset");
5096 sub get_vm_volumes
{
5100 foreach_volid
($conf, sub {
5101 my ($volid, $attr) = @_;
5103 return if $volid =~ m
|^/|;
5105 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5108 push @$vollist, $volid;
5114 sub vm_stop_cleanup
{
5115 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5120 my $vollist = get_vm_volumes
($conf);
5121 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5124 foreach my $ext (qw(mon qmp pid vnc qga)) {
5125 unlink "/var/run/qemu-server/${vmid}.$ext";
5128 if ($conf->{ivshmem
}) {
5129 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5130 # just delete it for now, VMs which have this already open do not
5131 # are affected, but new VMs will get a separated one. If this
5132 # becomes an issue we either add some sort of ref-counting or just
5133 # add a "don't delete on stop" flag to the ivshmem format.
5134 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5137 foreach my $key (keys %$conf) {
5138 next if $key !~ m/^hostpci(\d+)$/;
5139 my $hostpciindex = $1;
5140 my $d = parse_hostpci
($conf->{$key});
5141 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5143 foreach my $pci (@{$d->{pciid
}}) {
5144 my $pciid = $pci->{id
};
5145 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5149 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5151 warn $@ if $@; # avoid errors - just warn
5154 # call only in locked context
5156 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5158 my $pid = check_running
($vmid, $nocheck);
5163 $conf = PVE
::QemuConfig-
>load_config($vmid);
5164 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5165 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5166 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5167 $timeout = $opts->{down
} if $opts->{down
};
5169 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5174 if (defined($conf) && parse_guest_agent
($conf)->{enabled
}) {
5175 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5177 mon_cmd
($vmid, "system_powerdown");
5180 mon_cmd
($vmid, "quit");
5186 $timeout = 60 if !defined($timeout);
5189 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5194 if ($count >= $timeout) {
5196 warn "VM still running - terminating now with SIGTERM\n";
5199 die "VM quit/powerdown failed - got timeout\n";
5202 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5207 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5210 die "VM quit/powerdown failed\n";
5218 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5223 if ($count >= $timeout) {
5224 warn "VM still running - terminating now with SIGKILL\n";
5229 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5232 # Note: use $nocheck to skip tests if VM configuration file exists.
5233 # We need that when migration VMs to other nodes (files already moved)
5234 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5236 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5238 $force = 1 if !defined($force) && !$shutdown;
5241 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5242 kill 15, $pid if $pid;
5243 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5244 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5248 PVE
::QemuConfig-
>lock_config($vmid, sub {
5249 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5254 my ($vmid, $timeout) = @_;
5256 PVE
::QemuConfig-
>lock_config($vmid, sub {
5259 # only reboot if running, as qmeventd starts it again on a stop event
5260 return if !check_running
($vmid);
5262 create_reboot_request
($vmid);
5264 my $storecfg = PVE
::Storage
::config
();
5265 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5269 # avoid that the next normal shutdown will be confused for a reboot
5270 clear_reboot_request
($vmid);
5276 # note: if using the statestorage parameter, the caller has to check privileges
5278 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5285 PVE
::QemuConfig-
>lock_config($vmid, sub {
5287 $conf = PVE
::QemuConfig-
>load_config($vmid);
5289 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5290 PVE
::QemuConfig-
>check_lock($conf)
5291 if !($skiplock || $is_backing_up);
5293 die "cannot suspend to disk during backup\n"
5294 if $is_backing_up && $includestate;
5296 if ($includestate) {
5297 $conf->{lock} = 'suspending';
5298 my $date = strftime
("%Y-%m-%d", localtime(time()));
5299 $storecfg = PVE
::Storage
::config
();
5300 if (!$statestorage) {
5301 $statestorage = find_vmstate_storage
($conf, $storecfg);
5302 # check permissions for the storage
5303 my $rpcenv = PVE
::RPCEnvironment
::get
();
5304 if ($rpcenv->{type
} ne 'cli') {
5305 my $authuser = $rpcenv->get_user();
5306 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5311 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5312 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5313 PVE
::QemuConfig-
>write_config($vmid, $conf);
5315 mon_cmd
($vmid, "stop");
5319 if ($includestate) {
5321 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5324 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5326 my $state = mon_cmd
($vmid, "query-savevm");
5327 if (!$state->{status
}) {
5328 die "savevm not active\n";
5329 } elsif ($state->{status
} eq 'active') {
5332 } elsif ($state->{status
} eq 'completed') {
5333 print "State saved, quitting\n";
5335 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5336 die "query-savevm failed with error '$state->{error}'\n"
5338 die "query-savevm returned status '$state->{status}'\n";
5344 PVE
::QemuConfig-
>lock_config($vmid, sub {
5345 $conf = PVE
::QemuConfig-
>load_config($vmid);
5347 # cleanup, but leave suspending lock, to indicate something went wrong
5349 mon_cmd
($vmid, "savevm-end");
5350 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5351 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5352 delete $conf->@{qw(vmstate runningmachine)};
5353 PVE
::QemuConfig-
>write_config($vmid, $conf);
5359 die "lock changed unexpectedly\n"
5360 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5362 mon_cmd
($vmid, "quit");
5363 $conf->{lock} = 'suspended';
5364 PVE
::QemuConfig-
>write_config($vmid, $conf);
5370 my ($vmid, $skiplock, $nocheck) = @_;
5372 PVE
::QemuConfig-
>lock_config($vmid, sub {
5373 my $res = mon_cmd
($vmid, 'query-status');
5374 my $resume_cmd = 'cont';
5376 if ($res->{status
} && $res->{status
} eq 'suspended') {
5377 $resume_cmd = 'system_wakeup';
5382 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5384 PVE
::QemuConfig-
>check_lock($conf)
5385 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5388 mon_cmd
($vmid, $resume_cmd);
5393 my ($vmid, $skiplock, $key) = @_;
5395 PVE
::QemuConfig-
>lock_config($vmid, sub {
5397 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5399 # there is no qmp command, so we use the human monitor command
5400 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5401 die $res if $res ne '';
5405 # vzdump restore implementaion
5407 sub tar_archive_read_firstfile
{
5408 my $archive = shift;
5410 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5412 # try to detect archive type first
5413 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5414 die "unable to open file '$archive'\n";
5415 my $firstfile = <$fh>;
5419 die "ERROR: archive contaions no data\n" if !$firstfile;
5425 sub tar_restore_cleanup
{
5426 my ($storecfg, $statfile) = @_;
5428 print STDERR
"starting cleanup\n";
5430 if (my $fd = IO
::File-
>new($statfile, "r")) {
5431 while (defined(my $line = <$fd>)) {
5432 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5435 if ($volid =~ m
|^/|) {
5436 unlink $volid || die 'unlink failed\n';
5438 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5440 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5442 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5444 print STDERR
"unable to parse line in statfile - $line";
5451 sub restore_archive
{
5452 my ($archive, $vmid, $user, $opts) = @_;
5454 my $format = $opts->{format
};
5457 if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5458 $format = 'tar' if !$format;
5460 } elsif ($archive =~ m/\.tar$/) {
5461 $format = 'tar' if !$format;
5462 } elsif ($archive =~ m/.tar.lzo$/) {
5463 $format = 'tar' if !$format;
5465 } elsif ($archive =~ m/\.vma$/) {
5466 $format = 'vma' if !$format;
5467 } elsif ($archive =~ m/\.vma\.gz$/) {
5468 $format = 'vma' if !$format;
5470 } elsif ($archive =~ m/\.vma\.lzo$/) {
5471 $format = 'vma' if !$format;
5474 $format = 'vma' if !$format; # default
5477 # try to detect archive format
5478 if ($format eq 'tar') {
5479 return restore_tar_archive
($archive, $vmid, $user, $opts);
5481 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5485 sub restore_update_config_line
{
5486 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5488 return if $line =~ m/^\#qmdump\#/;
5489 return if $line =~ m/^\#vzdump\#/;
5490 return if $line =~ m/^lock:/;
5491 return if $line =~ m/^unused\d+:/;
5492 return if $line =~ m/^parent:/;
5494 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5495 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5496 # try to convert old 1.X settings
5497 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5498 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5499 my ($model, $macaddr) = split(/\=/, $devconfig);
5500 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5503 bridge
=> "vmbr$ind",
5504 macaddr
=> $macaddr,
5506 my $netstr = print_net
($net);
5508 print $outfd "net$cookie->{netcount}: $netstr\n";
5509 $cookie->{netcount
}++;
5511 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5512 my ($id, $netstr) = ($1, $2);
5513 my $net = parse_net
($netstr);
5514 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5515 $netstr = print_net
($net);
5516 print $outfd "$id: $netstr\n";
5517 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5520 my $di = parse_drive
($virtdev, $value);
5521 if (defined($di->{backup
}) && !$di->{backup
}) {
5522 print $outfd "#$line";
5523 } elsif ($map->{$virtdev}) {
5524 delete $di->{format
}; # format can change on restore
5525 $di->{file
} = $map->{$virtdev};
5526 $value = print_drive
($di);
5527 print $outfd "$virtdev: $value\n";
5531 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5533 if ($vmgenid ne '0') {
5534 # always generate a new vmgenid if there was a valid one setup
5535 $vmgenid = generate_uuid
();
5537 print $outfd "vmgenid: $vmgenid\n";
5538 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5539 my ($uuid, $uuid_str);
5540 UUID
::generate
($uuid);
5541 UUID
::unparse
($uuid, $uuid_str);
5542 my $smbios1 = parse_smbios1
($2);
5543 $smbios1->{uuid
} = $uuid_str;
5544 print $outfd $1.print_smbios1
($smbios1)."\n";
5551 my ($cfg, $vmid) = @_;
5553 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5555 my $volid_hash = {};
5556 foreach my $storeid (keys %$info) {
5557 foreach my $item (@{$info->{$storeid}}) {
5558 next if !($item->{volid
} && $item->{size
});
5559 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5560 $volid_hash->{$item->{volid
}} = $item;
5567 sub update_disk_config
{
5568 my ($vmid, $conf, $volid_hash) = @_;
5571 my $prefix = "VM $vmid:";
5573 # used and unused disks
5574 my $referenced = {};
5576 # Note: it is allowed to define multiple storages with same path (alias), so
5577 # we need to check both 'volid' and real 'path' (two different volid can point
5578 # to the same path).
5580 my $referencedpath = {};
5583 foreach my $opt (keys %$conf) {
5584 if (is_valid_drivename
($opt)) {
5585 my $drive = parse_drive
($opt, $conf->{$opt});
5586 my $volid = $drive->{file
};
5589 # mark volid as "in-use" for next step
5590 $referenced->{$volid} = 1;
5591 if ($volid_hash->{$volid} &&
5592 (my $path = $volid_hash->{$volid}->{path
})) {
5593 $referencedpath->{$path} = 1;
5596 next if drive_is_cdrom
($drive);
5597 next if !$volid_hash->{$volid};
5599 my ($updated, $old_size, $new_size) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volid_hash);
5600 if (defined($updated)) {
5602 $conf->{$opt} = print_drive
($updated);
5603 print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5608 # remove 'unusedX' entry if volume is used
5609 foreach my $opt (keys %$conf) {
5610 next if $opt !~ m/^unused\d+$/;
5611 my $volid = $conf->{$opt};
5612 my $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
5613 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5614 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5616 delete $conf->{$opt};
5619 $referenced->{$volid} = 1;
5620 $referencedpath->{$path} = 1 if $path;
5623 foreach my $volid (sort keys %$volid_hash) {
5624 next if $volid =~ m/vm-$vmid-state-/;
5625 next if $referenced->{$volid};
5626 my $path = $volid_hash->{$volid}->{path
};
5627 next if !$path; # just to be sure
5628 next if $referencedpath->{$path};
5630 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
5631 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5632 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5639 my ($vmid, $nolock, $dryrun) = @_;
5641 my $cfg = PVE
::Storage
::config
();
5643 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5644 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5645 foreach my $stor (keys %{$cfg->{ids
}}) {
5646 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
5649 print "rescan volumes...\n";
5650 my $volid_hash = scan_volids
($cfg, $vmid);
5652 my $updatefn = sub {
5655 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5657 PVE
::QemuConfig-
>check_lock($conf);
5660 foreach my $volid (keys %$volid_hash) {
5661 my $info = $volid_hash->{$volid};
5662 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
5665 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
5667 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
5670 if (defined($vmid)) {
5674 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
5677 my $vmlist = config_list
();
5678 foreach my $vmid (keys %$vmlist) {
5682 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
5688 sub restore_vma_archive
{
5689 my ($archive, $vmid, $user, $opts, $comp) = @_;
5691 my $readfrom = $archive;
5693 my $cfg = PVE
::Storage
::config
();
5695 my $bwlimit = $opts->{bwlimit
};
5697 my $dbg_cmdstring = '';
5698 my $add_pipe = sub {
5700 push @$commands, $cmd;
5701 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5702 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
5707 if ($archive eq '-') {
5710 # If we use a backup from a PVE defined storage we also consider that
5711 # storage's rate limit:
5712 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
5713 if (defined($volid)) {
5714 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
5715 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
5717 print STDERR
"applying read rate limit: $readlimit\n";
5718 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5719 $add_pipe->($cstream);
5726 if ($comp eq 'gzip') {
5727 $cmd = ['zcat', $readfrom];
5728 } elsif ($comp eq 'lzop') {
5729 $cmd = ['lzop', '-d', '-c', $readfrom];
5731 die "unknown compression method '$comp'\n";
5736 my $tmpdir = "/var/tmp/vzdumptmp$$";
5739 # disable interrupts (always do cleanups)
5743 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
5745 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5746 POSIX
::mkfifo
($mapfifo, 0600);
5749 my $openfifo = sub {
5750 open($fifofh, '>', $mapfifo) || die $!;
5753 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5760 my $rpcenv = PVE
::RPCEnvironment
::get
();
5762 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
5763 my $tmpfn = "$conffile.$$.tmp";
5765 # Note: $oldconf is undef if VM does not exist
5766 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
5767 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
5771 my $print_devmap = sub {
5772 my $virtdev_hash = {};
5774 my $cfgfn = "$tmpdir/qemu-server.conf";
5776 # we can read the config - that is already extracted
5777 my $fh = IO
::File-
>new($cfgfn, "r") ||
5778 "unable to read qemu-server.conf - $!\n";
5780 my $fwcfgfn = "$tmpdir/qemu-server.fw";
5782 my $pve_firewall_dir = '/etc/pve/firewall';
5783 mkdir $pve_firewall_dir; # make sure the dir exists
5784 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5787 while (defined(my $line = <$fh>)) {
5788 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5789 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5790 die "archive does not contain data for drive '$virtdev'\n"
5791 if !$devinfo->{$devname};
5792 if (defined($opts->{storage
})) {
5793 $storeid = $opts->{storage
} || 'local';
5794 } elsif (!$storeid) {
5797 $format = 'raw' if !$format;
5798 $devinfo->{$devname}->{devname
} = $devname;
5799 $devinfo->{$devname}->{virtdev
} = $virtdev;
5800 $devinfo->{$devname}->{format
} = $format;
5801 $devinfo->{$devname}->{storeid
} = $storeid;
5803 # check permission on storage
5804 my $pool = $opts->{pool
}; # todo: do we need that?
5805 if ($user ne 'root@pam') {
5806 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5809 $storage_limits{$storeid} = $bwlimit;
5811 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5812 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5814 my $drive = parse_drive
($virtdev, $2);
5815 if (drive_is_cloudinit
($drive)) {
5816 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
5817 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
5818 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
5822 storeid
=> $opts->{storage
} // $storeid,
5823 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
5824 file
=> $drive->{file
}, # to make drive_is_cloudinit check possible
5825 name
=> "vm-$vmid-cloudinit",
5828 $virtdev_hash->{$virtdev} = $d;
5833 foreach my $key (keys %storage_limits) {
5834 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
5836 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
5837 $storage_limits{$key} = $limit * 1024;
5840 foreach my $devname (keys %$devinfo) {
5841 die "found no device mapping information for device '$devname'\n"
5842 if !$devinfo->{$devname}->{virtdev
};
5845 # create empty/temp config
5847 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
5848 foreach_drive
($oldconf, sub {
5849 my ($ds, $drive) = @_;
5851 return if drive_is_cdrom
($drive, 1);
5853 my $volid = $drive->{file
};
5854 return if !$volid || $volid =~ m
|^/|;
5856 my ($path, $owner) = PVE
::Storage
::path
($cfg, $volid);
5857 return if !$path || !$owner || ($owner != $vmid);
5859 # Note: only delete disk we want to restore
5860 # other volumes will become unused
5861 if ($virtdev_hash->{$ds}) {
5862 eval { PVE
::Storage
::vdisk_free
($cfg, $volid); };
5869 # delete vmstate files, after the restore we have no snapshots anymore
5870 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5871 my $snap = $oldconf->{snapshots
}->{$snapname};
5872 if ($snap->{vmstate
}) {
5873 eval { PVE
::Storage
::vdisk_free
($cfg, $snap->{vmstate
}); };
5882 foreach my $virtdev (sort keys %$virtdev_hash) {
5883 my $d = $virtdev_hash->{$virtdev};
5884 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5885 my $storeid = $d->{storeid
};
5886 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
5889 if (my $limit = $storage_limits{$storeid}) {
5890 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5893 # test if requested format is supported
5894 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($cfg, $storeid);
5895 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
5896 $d->{format
} = $defFormat if !$supported;
5899 if ($d->{is_cloudinit
}) {
5901 $name .= ".$d->{format}" if $d->{format
} ne 'raw';
5904 my $volid = PVE
::Storage
::vdisk_alloc
($cfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
5905 print STDERR
"new volume ID is '$volid'\n";
5906 $d->{volid
} = $volid;
5908 PVE
::Storage
::activate_volumes
($cfg, [$volid]);
5910 my $write_zeros = 1;
5911 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
5915 if (!$d->{is_cloudinit
}) {
5916 my $path = PVE
::Storage
::path
($cfg, $volid);
5918 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5920 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5922 $map->{$virtdev} = $volid;
5925 $fh->seek(0, 0) || die "seek failed - $!\n";
5927 my $outfd = new IO
::File
($tmpfn, "w") ||
5928 die "unable to write config for VM $vmid\n";
5930 my $cookie = { netcount
=> 0 };
5931 while (defined(my $line = <$fh>)) {
5932 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
5945 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
5946 local $SIG{ALRM
} = sub { die "got timeout\n"; };
5948 $oldtimeout = alarm($timeout);
5955 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5956 my ($dev_id, $size, $devname) = ($1, $2, $3);
5957 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
5958 } elsif ($line =~ m/^CTIME: /) {
5959 # we correctly received the vma config, so we can disable
5960 # the timeout now for disk allocation (set to 10 minutes, so
5961 # that we always timeout if something goes wrong)
5964 print $fifofh "done\n";
5965 my $tmp = $oldtimeout || 0;
5966 $oldtimeout = undef;
5972 print "restore vma archive: $dbg_cmdstring\n";
5973 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
5977 alarm($oldtimeout) if $oldtimeout;
5980 foreach my $devname (keys %$devinfo) {
5981 my $volid = $devinfo->{$devname}->{volid
};
5982 push @$vollist, $volid if $volid;
5985 PVE
::Storage
::deactivate_volumes
($cfg, $vollist);
5993 foreach my $devname (keys %$devinfo) {
5994 my $volid = $devinfo->{$devname}->{volid
};
5997 if ($volid =~ m
|^/|) {
5998 unlink $volid || die 'unlink failed\n';
6000 PVE
::Storage
::vdisk_free
($cfg, $volid);
6002 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6004 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6011 rename($tmpfn, $conffile) ||
6012 die "unable to commit configuration file '$conffile'\n";
6014 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6016 eval { rescan
($vmid, 1); };
6020 sub restore_tar_archive
{
6021 my ($archive, $vmid, $user, $opts) = @_;
6023 if ($archive ne '-') {
6024 my $firstfile = tar_archive_read_firstfile
($archive);
6025 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6026 if $firstfile ne 'qemu-server.conf';
6029 my $storecfg = PVE
::Storage
::config
();
6031 # avoid zombie disks when restoring over an existing VM -> cleanup first
6032 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6033 # skiplock=1 because qmrestore has set the 'create' lock itself already
6034 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6035 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6037 my $tocmd = "/usr/lib/qemu-server/qmextract";
6039 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6040 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6041 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6042 $tocmd .= ' --info' if $opts->{info
};
6044 # tar option "xf" does not autodetect compression when read from STDIN,
6045 # so we pipe to zcat
6046 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6047 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6049 my $tmpdir = "/var/tmp/vzdumptmp$$";
6052 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6053 local $ENV{VZDUMP_VMID
} = $vmid;
6054 local $ENV{VZDUMP_USER
} = $user;
6056 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6057 my $tmpfn = "$conffile.$$.tmp";
6059 # disable interrupts (always do cleanups)
6063 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6071 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6073 if ($archive eq '-') {
6074 print "extracting archive from STDIN\n";
6075 run_command
($cmd, input
=> "<&STDIN");
6077 print "extracting archive '$archive'\n";
6081 return if $opts->{info
};
6085 my $statfile = "$tmpdir/qmrestore.stat";
6086 if (my $fd = IO
::File-
>new($statfile, "r")) {
6087 while (defined (my $line = <$fd>)) {
6088 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6089 $map->{$1} = $2 if $1;
6091 print STDERR
"unable to parse line in statfile - $line\n";
6097 my $confsrc = "$tmpdir/qemu-server.conf";
6099 my $srcfd = new IO
::File
($confsrc, "r") ||
6100 die "unable to open file '$confsrc'\n";
6102 my $outfd = new IO
::File
($tmpfn, "w") ||
6103 die "unable to write config for VM $vmid\n";
6105 my $cookie = { netcount
=> 0 };
6106 while (defined (my $line = <$srcfd>)) {
6107 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6115 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6121 rename $tmpfn, $conffile ||
6122 die "unable to commit configuration file '$conffile'\n";
6124 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6126 eval { rescan
($vmid, 1); };
6130 sub foreach_storage_used_by_vm
{
6131 my ($conf, $func) = @_;
6135 foreach_drive
($conf, sub {
6136 my ($ds, $drive) = @_;
6137 return if drive_is_cdrom
($drive);
6139 my $volid = $drive->{file
};
6141 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6142 $sidhash->{$sid} = $sid if $sid;
6145 foreach my $sid (sort keys %$sidhash) {
6150 my $qemu_snap_storage = {
6153 sub do_snapshots_with_qemu
{
6154 my ($storecfg, $volid) = @_;
6156 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6157 my $scfg = $storecfg->{ids
}->{$storage_name};
6159 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6163 if ($volid =~ m/\.(qcow2|qed)$/){
6170 sub qga_check_running
{
6171 my ($vmid, $nowarn) = @_;
6173 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6175 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6181 sub template_create
{
6182 my ($vmid, $conf, $disk) = @_;
6184 my $storecfg = PVE
::Storage
::config
();
6186 foreach_drive
($conf, sub {
6187 my ($ds, $drive) = @_;
6189 return if drive_is_cdrom
($drive);
6190 return if $disk && $ds ne $disk;
6192 my $volid = $drive->{file
};
6193 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6195 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6196 $drive->{file
} = $voliddst;
6197 $conf->{$ds} = print_drive
($drive);
6198 PVE
::QemuConfig-
>write_config($vmid, $conf);
6202 sub convert_iscsi_path
{
6205 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6210 my $initiator_name = get_initiator_name
();
6212 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6213 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6216 die "cannot convert iscsi path '$path', unkown format\n";
6219 sub qemu_img_convert
{
6220 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6222 my $storecfg = PVE
::Storage
::config
();
6223 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6224 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6226 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6230 my $src_is_iscsi = 0;
6234 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6235 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6236 $src_format = qemu_img_format
($src_scfg, $src_volname);
6237 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6238 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6239 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6240 } elsif (-f
$src_volid) {
6241 $src_path = $src_volid;
6242 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6247 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6249 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6250 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6251 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6252 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6255 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6256 push @$cmd, '-l', "snapshot.name=$snapname"
6257 if $snapname && $src_format && $src_format eq "qcow2";
6258 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6259 push @$cmd, '-T', $cachemode if defined($cachemode);
6261 if ($src_is_iscsi) {
6262 push @$cmd, '--image-opts';
6263 $src_path = convert_iscsi_path
($src_path);
6264 } elsif ($src_format) {
6265 push @$cmd, '-f', $src_format;
6268 if ($dst_is_iscsi) {
6269 push @$cmd, '--target-image-opts';
6270 $dst_path = convert_iscsi_path
($dst_path);
6272 push @$cmd, '-O', $dst_format;
6275 push @$cmd, $src_path;
6277 if (!$dst_is_iscsi && $is_zero_initialized) {
6278 push @$cmd, "zeroinit:$dst_path";
6280 push @$cmd, $dst_path;
6285 if($line =~ m/\((\S+)\/100\
%\)/){
6287 my $transferred = int($size * $percent / 100);
6288 my $remaining = $size - $transferred;
6290 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6295 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6297 die "copy failed: $err" if $err;
6300 sub qemu_img_format
{
6301 my ($scfg, $volname) = @_;
6303 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6310 sub qemu_drive_mirror
{
6311 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6313 $jobs = {} if !$jobs;
6317 $jobs->{"drive-$drive"} = {};
6319 if ($dst_volid =~ /^nbd:/) {
6320 $qemu_target = $dst_volid;
6323 my $storecfg = PVE
::Storage
::config
();
6324 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6326 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6328 $format = qemu_img_format
($dst_scfg, $dst_volname);
6330 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6332 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6335 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6336 $opts->{format
} = $format if $format;
6338 if (defined($bwlimit)) {
6339 $opts->{speed
} = $bwlimit * 1024;
6340 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6342 print "drive mirror is starting for drive-$drive\n";
6345 # if a job already runs for this device we get an error, catch it for cleanup
6346 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
6348 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6350 die "mirroring error: $err\n";
6353 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6356 sub qemu_drive_mirror_monitor
{
6357 my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6360 my $err_complete = 0;
6363 die "storage migration timed out\n" if $err_complete > 300;
6365 my $stats = mon_cmd
($vmid, "query-block-jobs");
6367 my $running_mirror_jobs = {};
6368 foreach my $stat (@$stats) {
6369 next if $stat->{type
} ne 'mirror';
6370 $running_mirror_jobs->{$stat->{device
}} = $stat;
6373 my $readycounter = 0;
6375 foreach my $job (keys %$jobs) {
6377 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6378 print "$job : finished\n";
6379 delete $jobs->{$job};
6383 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6385 my $busy = $running_mirror_jobs->{$job}->{busy
};
6386 my $ready = $running_mirror_jobs->{$job}->{ready
};
6387 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6388 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6389 my $remaining = $total - $transferred;
6390 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6392 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6395 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6398 last if scalar(keys %$jobs) == 0;
6400 if ($readycounter == scalar(keys %$jobs)) {
6401 print "all mirroring jobs are ready \n";
6402 last if $skipcomplete; #do the complete later
6404 if ($vmiddst && $vmiddst != $vmid) {
6405 my $agent_running = $qga && qga_check_running
($vmid);
6406 if ($agent_running) {
6407 print "freeze filesystem\n";
6408 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6410 print "suspend vm\n";
6411 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6414 # if we clone a disk for a new target vm, we don't switch the disk
6415 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6417 if ($agent_running) {
6418 print "unfreeze filesystem\n";
6419 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6421 print "resume vm\n";
6422 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6428 foreach my $job (keys %$jobs) {
6429 # try to switch the disk if source and destination are on the same guest
6430 print "$job: Completing block job...\n";
6432 eval { mon_cmd
($vmid, "block-job-complete", device
=> $job) };
6433 if ($@ =~ m/cannot be completed/) {
6434 print "$job: Block job cannot be completed, try again.\n";
6437 print "$job: Completed successfully.\n";
6438 $jobs->{$job}->{complete
} = 1;
6449 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6450 die "mirroring error: $err";
6455 sub qemu_blockjobs_cancel
{
6456 my ($vmid, $jobs) = @_;
6458 foreach my $job (keys %$jobs) {
6459 print "$job: Cancelling block job\n";
6460 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6461 $jobs->{$job}->{cancel
} = 1;
6465 my $stats = mon_cmd
($vmid, "query-block-jobs");
6467 my $running_jobs = {};
6468 foreach my $stat (@$stats) {
6469 $running_jobs->{$stat->{device
}} = $stat;
6472 foreach my $job (keys %$jobs) {
6474 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6475 print "$job: Done.\n";
6476 delete $jobs->{$job};
6480 last if scalar(keys %$jobs) == 0;
6487 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6488 $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6493 print "create linked clone of drive $drivename ($drive->{file})\n";
6494 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6495 push @$newvollist, $newvolid;
6498 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6499 $storeid = $storage if $storage;
6501 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6502 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
6504 print "create full clone of drive $drivename ($drive->{file})\n";
6506 if (drive_is_cloudinit
($drive)) {
6507 $name = "vm-$newvmid-cloudinit";
6508 $name .= ".$dst_format" if $dst_format ne 'raw';
6510 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
6512 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6513 push @$newvollist, $newvolid;
6515 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
6517 if (drive_is_cloudinit
($drive)) {
6521 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6522 if (!$running || $snapname) {
6523 # TODO: handle bwlimits
6524 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
6527 my $kvmver = get_running_qemu_version
($vmid);
6528 if (!min_version
($kvmver, 2, 7)) {
6529 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6530 if $drive->{iothread
};
6533 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
6538 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
6541 $disk->{format
} = undef;
6542 $disk->{file
} = $newvolid;
6543 $disk->{size
} = $size;
6548 sub get_running_qemu_version
{
6550 my $res = mon_cmd
($vmid, "query-version");
6551 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6554 sub qemu_use_old_bios_files
{
6555 my ($machine_type) = @_;
6557 return if !$machine_type;
6559 my $use_old_bios_files = undef;
6561 if ($machine_type =~ m/^(\S+)\.pxe$/) {
6563 $use_old_bios_files = 1;
6565 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, kvm_user_version
());
6566 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6567 # load new efi bios files on migration. So this hack is required to allow
6568 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6569 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6570 $use_old_bios_files = !min_version
($version, 2, 4);
6573 return ($use_old_bios_files, $machine_type);
6576 sub create_efidisk
($$$$$) {
6577 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6579 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
6580 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
6582 my $vars_size_b = -s
$ovmf_vars;
6583 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
6584 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6585 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6587 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
6588 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
6590 return ($volid, $size/1024);
6593 sub vm_iothreads_list
{
6596 my $res = mon_cmd
($vmid, 'query-iothreads');
6599 foreach my $iothread (@$res) {
6600 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
6607 my ($conf, $drive) = @_;
6611 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
6613 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
6619 my $controller = int($drive->{index} / $maxdev);
6620 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single') ?
"virtioscsi" : "scsihw";
6622 return ($maxdev, $controller, $controller_prefix);
6625 sub windows_version
{
6628 return 0 if !$ostype;
6632 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6634 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6636 } elsif ($ostype =~ m/^win(\d+)$/) {
6643 sub resolve_dst_disk_format
{
6644 my ($storecfg, $storeid, $src_volname, $format) = @_;
6645 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6648 # if no target format is specified, use the source disk format as hint
6650 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6651 $format = qemu_img_format
($scfg, $src_volname);
6657 # test if requested format is supported - else use default
6658 my $supported = grep { $_ eq $format } @$validFormats;
6659 $format = $defFormat if !$supported;
6663 # NOTE: if this logic changes, please update docs & possibly gui logic
6664 sub find_vmstate_storage
{
6665 my ($conf, $storecfg) = @_;
6667 # first, return storage from conf if set
6668 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
6670 my ($target, $shared, $local);
6672 foreach_storage_used_by_vm
($conf, sub {
6674 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
6675 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
6676 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
6679 # second, use shared storage where VM has at least one disk
6680 # third, use local storage where VM has at least one disk
6681 # fall back to local storage
6682 $target = $shared // $local // 'local';
6688 my ($uuid, $uuid_str);
6689 UUID
::generate
($uuid);
6690 UUID
::unparse
($uuid, $uuid_str);
6694 sub generate_smbios1_uuid
{
6695 return "uuid=".generate_uuid
();
6701 mon_cmd
($vmid, 'nbd-server-stop');
6704 sub create_reboot_request
{
6706 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
6707 or die "failed to create reboot trigger file: $!\n";
6711 sub clear_reboot_request
{
6713 my $path = "/run/qemu-server/$vmid.reboot";
6716 $res = unlink($path);
6717 die "could not remove reboot request for $vmid: $!"
6718 if !$res && $! != POSIX
::ENOENT
;
6723 # bash completion helper
6725 sub complete_backup_archives
{
6726 my ($cmdname, $pname, $cvalue) = @_;
6728 my $cfg = PVE
::Storage
::config
();
6732 if ($cvalue =~ m/^([^:]+):/) {
6736 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
6739 foreach my $id (keys %$data) {
6740 foreach my $item (@{$data->{$id}}) {
6741 next if $item->{format
} !~ m/^vma\.(gz|lzo)$/;
6742 push @$res, $item->{volid
} if defined($item->{volid
});
6749 my $complete_vmid_full = sub {
6752 my $idlist = vmstatus
();
6756 foreach my $id (keys %$idlist) {
6757 my $d = $idlist->{$id};
6758 if (defined($running)) {
6759 next if $d->{template
};
6760 next if $running && $d->{status
} ne 'running';
6761 next if !$running && $d->{status
} eq 'running';
6770 return &$complete_vmid_full();
6773 sub complete_vmid_stopped
{
6774 return &$complete_vmid_full(0);
6777 sub complete_vmid_running
{
6778 return &$complete_vmid_full(1);
6781 sub complete_storage
{
6783 my $cfg = PVE
::Storage
::config
();
6784 my $ids = $cfg->{ids
};
6787 foreach my $sid (keys %$ids) {
6788 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
6789 next if !$ids->{$sid}->{content
}->{images
};
6796 sub complete_migration_storage
{
6797 my ($cmd, $param, $current_value, $all_args) = @_;
6799 my $targetnode = @$all_args[1];
6801 my $cfg = PVE
::Storage
::config
();
6802 my $ids = $cfg->{ids
};
6805 foreach my $sid (keys %$ids) {
6806 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
6807 next if !$ids->{$sid}->{content
}->{images
};