1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
31 use PVE
::DataCenterConfig
;
32 use PVE
::Exception
qw(raise raise_param_exc);
33 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
35 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
38 use PVE
::RPCEnvironment
;
42 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
46 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
47 use PVE
::QemuServer
::Cloudinit
;
48 use PVE
::QemuServer
::CGroup
;
49 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
50 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
51 use PVE
::QemuServer
::Machine
;
52 use PVE
::QemuServer
::Memory
;
53 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
54 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
55 use PVE
::QemuServer
::USB
qw(parse_usb_device);
59 require PVE
::Network
::SDN
::Zones
;
63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
66 "$EDK2_FW_BASE/OVMF_CODE.fd",
67 "$EDK2_FW_BASE/OVMF_VARS.fd"
70 "$EDK2_FW_BASE/AAVMF_CODE.fd",
71 "$EDK2_FW_BASE/AAVMF_VARS.fd"
75 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
77 # Note about locking: we use flock on the config file protect
78 # against concurent actions.
79 # Aditionaly, we have a 'lock' setting in the config file. This
80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
81 # allowed when such lock is set. But you can ignore this kind of
82 # lock with the --skiplock flag.
84 cfs_register_file
('/qemu-server/',
88 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
89 description
=> "Some command save/restore state from this location.",
95 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
96 description
=> "Specifies the Qemu machine type.",
98 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
105 my ($map, $source) = @_;
107 return $source if !defined($map);
109 return $map->{entries
}->{$source}
110 if $map->{entries
} && defined($map->{entries
}->{$source});
112 return $map->{default} if $map->{default};
114 # identity (fallback)
118 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
119 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
121 format
=> 'storagepair-list',
125 #no warnings 'redefine';
129 $nodename_cache //= PVE
::INotify
::nodename
();
130 return $nodename_cache;
137 enum
=> [qw(i6300esb ib700)],
138 description
=> "Watchdog type to emulate.",
139 default => 'i6300esb',
144 enum
=> [qw(reset shutdown poweroff pause debug none)],
145 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
149 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
153 description
=> "Enable/disable Qemu GuestAgent.",
158 fstrim_cloned_disks
=> {
159 description
=> "Run fstrim after moving a disk or migrating the VM.",
165 description
=> "Select the agent type",
169 enum
=> [qw(virtio isa)],
175 description
=> "Select the VGA type.",
180 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
183 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
195 description
=> "The size of the file in MB.",
199 pattern
=> '[a-zA-Z0-9\-]+',
201 format_description
=> 'string',
202 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
209 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
210 description
=> "Configure an audio device."
214 enum
=> ['spice', 'none'],
217 description
=> "Driver backend for the audio device."
221 my $spice_enhancements_fmt = {
226 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
230 enum
=> ['off', 'all', 'filter'],
233 description
=> "Enable video streaming. Uses compression for detected video streams."
240 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
242 description
=> "The file on the host to gather entropy from. In most"
243 . " cases /dev/urandom should be preferred over /dev/random"
244 . " to avoid entropy-starvation issues on the host. Using"
245 . " urandom does *not* decrease security in any meaningful"
246 . " way, as it's still seeded from real entropy, and the"
247 . " bytes provided will most likely be mixed with real"
248 . " entropy on the guest as well. /dev/hwrng can be used"
249 . " to pass through a hardware RNG from the host.",
253 description
=> "Maximum bytes of entropy injected into the guest every"
254 . " 'period' milliseconds. Prefer a lower value when using"
255 . " /dev/random as source. Use 0 to disable limiting"
256 . " (potentially dangerous!).",
259 # default is 1 KiB/s, provides enough entropy to the guest to avoid
260 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
261 # of overwhelming the host, provided we're reading from /dev/urandom
266 description
=> "Every 'period' milliseconds the entropy-injection quota"
267 . " is reset, allowing the guest to retrieve another"
268 . " 'max_bytes' of entropy.",
278 description
=> "Specifies whether a VM will be started during system bootup.",
284 description
=> "Automatic restart after crash (currently ignored).",
289 type
=> 'string', format
=> 'pve-hotplug-features',
290 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
291 default => 'network,disk,usb',
296 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
302 description
=> "Lock/unlock the VM.",
303 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
308 description
=> "Limit of CPU usage.",
309 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
317 description
=> "CPU weight for a VM.",
318 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
326 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
333 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
339 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
347 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
348 "It should not be necessary to set it.",
349 enum
=> PVE
::Tools
::kvmkeymaplist
(),
354 type
=> 'string', format
=> 'dns-name',
355 description
=> "Set a name for the VM. Only used on the configuration web interface.",
360 description
=> "SCSI controller model",
361 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
367 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
372 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
373 description
=> "Specify guest operating system.",
374 verbose_description
=> <<EODESC,
375 Specify guest operating system. This is used to enable special
376 optimization/features for specific operating systems:
379 other;; unspecified OS
380 wxp;; Microsoft Windows XP
381 w2k;; Microsoft Windows 2000
382 w2k3;; Microsoft Windows 2003
383 w2k8;; Microsoft Windows 2008
384 wvista;; Microsoft Windows Vista
385 win7;; Microsoft Windows 7
386 win8;; Microsoft Windows 8/2012/2012r2
387 win10;; Microsoft Windows 10/2016
388 l24;; Linux 2.4 Kernel
389 l26;; Linux 2.6 - 5.X Kernel
390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
395 type
=> 'string', format
=> 'pve-qm-boot',
396 description
=> "Specify guest boot order. Use with 'order=', usage with"
397 . " no key or 'legacy=' is deprecated.",
401 type
=> 'string', format
=> 'pve-qm-bootdisk',
402 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
403 pattern
=> '(ide|sata|scsi|virtio)\d+',
408 description
=> "The number of CPUs. Please use option -sockets instead.",
415 description
=> "The number of CPU sockets.",
422 description
=> "The number of cores per socket.",
429 description
=> "Enable/disable NUMA.",
435 description
=> "Enable/disable hugepages memory.",
436 enum
=> [qw(any 2 1024)],
442 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
443 ." after VM shutdown and can be used for subsequent starts.",
448 description
=> "Number of hotplugged vcpus.",
455 description
=> "Enable/disable ACPI.",
460 description
=> "Enable/disable Qemu GuestAgent and its properties.",
462 format
=> $agent_fmt,
467 description
=> "Enable/disable KVM hardware virtualization.",
473 description
=> "Enable/disable time drift fix.",
479 description
=> "Set the real time clock to local time. This is enabled by default if ostype"
480 ." indicates a Microsoft OS.",
485 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
489 type
=> 'string', format
=> $vga_fmt,
490 description
=> "Configure the VGA hardware.",
491 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
492 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
493 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
494 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
495 ." display server. For win* OS you can select how many independent displays you want,"
496 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
497 ." using a serial device as terminal.",
501 type
=> 'string', format
=> 'pve-qm-watchdog',
502 description
=> "Create a virtual hardware watchdog device.",
503 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
504 ." action), the watchdog must be periodically polled by an agent inside the guest or"
505 ." else the watchdog will reset the guest (or execute the respective action specified)",
510 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
511 description
=> "Set the initial date of the real time clock. Valid format for date are:"
512 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
513 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
516 startup
=> get_standard_option
('pve-startup-order'),
520 description
=> "Enable/disable Template.",
526 description
=> "Arbitrary arguments passed to kvm.",
527 verbose_description
=> <<EODESCR,
528 Arbitrary arguments passed to kvm, for example:
530 args: -no-reboot -no-hpet
532 NOTE: this option is for experts only.
539 description
=> "Enable/disable the USB tablet device.",
540 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
541 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
542 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
543 ." may consider disabling this to save some context switches. This is turned off by"
544 ." default if you use spice (`qm set <vmid> --vga qxl`).",
549 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
553 migrate_downtime
=> {
556 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
562 type
=> 'string', format
=> 'pve-qm-ide',
563 typetext
=> '<volume>',
564 description
=> "This is an alias for option -ide2",
568 description
=> "Emulated CPU type.",
570 format
=> 'pve-vm-cpu-conf',
572 parent
=> get_standard_option
('pve-snapshot-name', {
574 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
578 description
=> "Timestamp for snapshots.",
584 type
=> 'string', format
=> 'pve-volume-id',
585 description
=> "Reference to a volume which stores the VM state. This is used internally"
588 vmstatestorage
=> get_standard_option
('pve-storage-id', {
589 description
=> "Default storage for VM state volumes/files.",
592 runningmachine
=> get_standard_option
('pve-qemu-machine', {
593 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
597 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
598 ." internally for snapshots.",
601 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
602 format_description
=> 'QEMU -cpu parameter'
604 machine
=> get_standard_option
('pve-qemu-machine'),
606 description
=> "Virtual processor architecture. Defaults to the host.",
609 enum
=> [qw(x86_64 aarch64)],
612 description
=> "Specify SMBIOS type 1 fields.",
613 type
=> 'string', format
=> 'pve-qm-smbios1',
620 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
621 ." remove disk operations.",
627 enum
=> [ qw(seabios ovmf) ],
628 description
=> "Select BIOS implementation.",
629 default => 'seabios',
633 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
634 format_description
=> 'UUID',
635 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
636 ." to disable explicitly.",
637 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
638 ." value identifier to the guest OS. This allows to notify the guest operating system"
639 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
640 ." execution or creation from a template). The guest operating system notices the"
641 ." change, and is then able to react as appropriate by marking its copies of"
642 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
643 ."Note that auto-creation only works when done through API/CLI create or update methods"
644 .", but not when manually editing the config file.",
645 default => "1 (autogenerated)",
650 format
=> 'pve-volume-id',
652 description
=> "Script that will be executed during various steps in the vms lifetime.",
656 format
=> $ivshmem_fmt,
657 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
663 format
=> $audio_fmt,
664 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
667 spice_enhancements
=> {
669 format
=> $spice_enhancements_fmt,
670 description
=> "Configure additional enhancements for SPICE.",
674 type
=> 'string', format
=> 'pve-tag-list',
675 description
=> 'Tags of the VM. This is only meta information.',
681 description
=> "Configure a VirtIO-based Random Number Generator.",
690 description
=> 'Specify a custom file containing all meta data passed to the VM via"
691 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
692 format
=> 'pve-volume-id',
693 format_description
=> 'volume',
698 description
=> 'Specify a custom file containing all network data passed to the VM via'
700 format
=> 'pve-volume-id',
701 format_description
=> 'volume',
706 description
=> 'Specify a custom file containing all user data passed to the VM via'
708 format
=> 'pve-volume-id',
709 format_description
=> 'volume',
712 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
714 my $confdesc_cloudinit = {
718 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
719 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
720 .' and `configdrive2` for windows.',
721 enum
=> ['configdrive2', 'nocloud'],
726 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
727 ." image's configured default user.",
732 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
733 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
734 .' support hashed passwords.',
739 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
741 format
=> 'pve-qm-cicustom',
746 description
=> "cloud-init: Sets DNS search domains for a container. Create will'
747 .' automatically use the setting from the host if neither searchdomain nor nameserver'
752 type
=> 'string', format
=> 'address-list',
753 description
=> "cloud-init: Sets DNS server IP address for a container. Create will'
754 .' automatically use the setting from the host if neither searchdomain nor nameserver'
760 format
=> 'urlencoded',
761 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
765 # what about other qemu settings ?
767 #machine => 'string',
780 ##soundhw => 'string',
782 while (my ($k, $v) = each %$confdesc) {
783 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
786 my $MAX_USB_DEVICES = 5;
788 my $MAX_SERIAL_PORTS = 4;
789 my $MAX_PARALLEL_PORTS = 3;
795 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
796 description
=> "CPUs accessing this NUMA node.",
797 format_description
=> "id[-id];...",
801 description
=> "Amount of memory this NUMA node provides.",
806 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
807 description
=> "Host NUMA nodes to use.",
808 format_description
=> "id[-id];...",
813 enum
=> [qw(preferred bind interleave)],
814 description
=> "NUMA allocation policy.",
818 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
821 type
=> 'string', format
=> $numa_fmt,
822 description
=> "NUMA topology.",
824 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
826 for (my $i = 0; $i < $MAX_NUMA; $i++) {
827 $confdesc->{"numa$i"} = $numadesc;
830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
831 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
832 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
835 my $net_fmt_bridge_descr = <<__EOD__;
836 Bridge to attach the network device to. The Proxmox VE standard bridge
839 If you do not specify a bridge, we create a kvm user (NATed) network
840 device, which provides DHCP and DNS services. The following addresses
847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
851 macaddr
=> get_standard_option
('mac-addr', {
852 description
=> "MAC address. That address must be unique withing your network. This is"
853 ." automatically generated if not specified.",
857 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
858 ." very low CPU overhead. If your guest does not support this driver, it is usually"
859 ." best to use 'e1000'.",
860 enum
=> $nic_model_list,
863 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
866 description
=> $net_fmt_bridge_descr,
867 format_description
=> 'bridge',
868 pattern
=> '[-_.\w\d]+',
873 minimum
=> 0, maximum
=> 16,
874 description
=> 'Number of packet queues to be used on the device.',
880 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
885 minimum
=> 1, maximum
=> 4094,
886 description
=> 'VLAN tag to apply to packets on this interface.',
891 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
892 description
=> 'VLAN trunks to pass through this interface.',
893 format_description
=> 'vlanid[;vlanid...]',
898 description
=> 'Whether this interface should be protected by the firewall.',
903 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
908 minimum
=> 1, maximum
=> 65520,
909 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
916 type
=> 'string', format
=> $net_fmt,
917 description
=> "Specify network devices.",
920 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
925 format
=> 'pve-ipv4-config',
926 format_description
=> 'IPv4Format/CIDR',
927 description
=> 'IPv4 address in CIDR format.',
934 format_description
=> 'GatewayIPv4',
935 description
=> 'Default gateway for IPv4 traffic.',
941 format
=> 'pve-ipv6-config',
942 format_description
=> 'IPv6Format/CIDR',
943 description
=> 'IPv6 address in CIDR format.',
950 format_description
=> 'GatewayIPv6',
951 description
=> 'Default gateway for IPv6 traffic.',
956 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
959 type
=> 'string', format
=> 'pve-qm-ipconfig',
960 description
=> <<'EODESCR',
961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
966 gateway should be provided.
967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
973 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
975 for (my $i = 0; $i < $MAX_NETS; $i++) {
976 $confdesc->{"net$i"} = $netdesc;
977 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
980 foreach my $key (keys %$confdesc_cloudinit) {
981 $confdesc->{$key} = $confdesc_cloudinit->{$key};
984 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
985 sub verify_volume_id_or_qm_path
{
986 my ($volid, $noerr) = @_;
988 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
992 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
993 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1004 type
=> 'string', format
=> 'pve-qm-usb-device',
1005 format_description
=> 'HOSTUSBDEVICE|spice',
1006 description
=> <<EODESCR,
1007 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1009 'bus-port(.port)*' (decimal numbers) or
1010 'vendor_id:product_id' (hexadeciaml numbers) or
1013 You can use the 'lsusb -t' command to list existing usb devices.
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1016 machines - use with special care.
1018 The value 'spice' can be used to add a usb redirection devices for spice.
1024 description
=> "Specifies whether if given host option is a USB3 device or port.",
1031 type
=> 'string', format
=> $usb_fmt,
1032 description
=> "Configure an USB device (n is 0 to 4).",
1034 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1039 pattern
=> '(/dev/.+|socket)',
1040 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1041 verbose_description
=> <<EODESCR,
1042 Create a serial device inside the VM (n is 0 to 3), and pass through a
1043 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1044 host side (use 'qm terminal' to open a terminal connection).
1046 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1047 use with special care.
1049 CAUTION: Experimental! User reported problems with this option.
1056 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1057 description
=> "Map host parallel devices (n is 0 to 2).",
1058 verbose_description
=> <<EODESCR,
1059 Map host parallel devices (n is 0 to 2).
1061 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1062 machines - use with special care.
1064 CAUTION: Experimental! User reported problems with this option.
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1069 $confdesc->{"parallel$i"} = $paralleldesc;
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1073 $confdesc->{"serial$i"} = $serialdesc;
1076 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1077 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1080 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1081 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1084 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1085 $confdesc->{"usb$i"} = $usbdesc;
1093 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1094 . " Deprecated, use 'order=' instead.",
1095 pattern
=> '[acdn]{1,4}',
1096 format_description
=> "[acdn]{1,4}",
1098 # note: this is also the fallback if boot: is not given at all
1104 format
=> 'pve-qm-bootdev-list',
1105 format_description
=> "device[;device...]",
1106 description
=> <<EODESC,
1107 The guest will attempt to boot from devices in the order they appear here.
1109 Disks, optical drives and passed-through storage USB devices will be directly
1110 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1111 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1113 Note that only devices in this list will be marked as bootable and thus loaded
1114 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1115 (e.g. software-raid), you need to specify all of them here.
1117 Overrides the deprecated 'legacy=[acdn]*' value when given.
1121 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1123 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1124 sub verify_bootdev
{
1125 my ($dev, $noerr) = @_;
1127 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && $dev !~ m/^efidisk/;
1131 return 0 if $dev !~ m/^$base\d+$/;
1132 return 0 if !$confdesc->{$dev};
1136 return $dev if $check->("net");
1137 return $dev if $check->("usb");
1138 return $dev if $check->("hostpci");
1141 die "invalid boot device '$dev'\n";
1144 sub print_bootorder
{
1146 return "" if !@$devs;
1147 my $data = { order
=> join(';', @$devs) };
1148 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1151 my $kvm_api_version = 0;
1154 return $kvm_api_version if $kvm_api_version;
1156 open my $fh, '<', '/dev/kvm' or return;
1158 # 0xae00 => KVM_GET_API_VERSION
1159 $kvm_api_version = ioctl($fh, 0xae00, 0);
1162 return $kvm_api_version;
1165 my $kvm_user_version = {};
1168 sub kvm_user_version
{
1171 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1172 my $st = stat($binary);
1174 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1175 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1176 $cachedmtime == $st->mtime;
1178 $kvm_user_version->{$binary} = 'unknown';
1179 $kvm_mtime->{$binary} = $st->mtime;
1183 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1184 $kvm_user_version->{$binary} = $2;
1188 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1191 return $kvm_user_version->{$binary};
1194 my sub extract_version
{
1195 my ($machine_type, $version) = @_;
1196 $version = kvm_user_version
() if !defined($version);
1197 PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1200 sub kernel_has_vhost_net
{
1201 return -c
'/dev/vhost-net';
1206 return defined($confdesc->{$key});
1210 sub get_cdrom_path
{
1212 return $cdrom_path if $cdrom_path;
1214 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1215 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1216 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1220 my ($storecfg, $vmid, $cdrom) = @_;
1222 if ($cdrom eq 'cdrom') {
1223 return get_cdrom_path
();
1224 } elsif ($cdrom eq 'none') {
1226 } elsif ($cdrom =~ m
|^/|) {
1229 return PVE
::Storage
::path
($storecfg, $cdrom);
1233 # try to convert old style file names to volume IDs
1234 sub filename_to_volume_id
{
1235 my ($vmid, $file, $media) = @_;
1237 if (!($file eq 'none' || $file eq 'cdrom' ||
1238 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1240 return if $file =~ m
|/|;
1242 if ($media && $media eq 'cdrom') {
1243 $file = "local:iso/$file";
1245 $file = "local:$vmid/$file";
1252 sub verify_media_type
{
1253 my ($opt, $vtype, $media) = @_;
1258 if ($media eq 'disk') {
1260 } elsif ($media eq 'cdrom') {
1263 die "internal error";
1266 return if ($vtype eq $etype);
1268 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1271 sub cleanup_drive_path
{
1272 my ($opt, $storecfg, $drive) = @_;
1274 # try to convert filesystem paths to volume IDs
1276 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1277 ($drive->{file
} !~ m
|^/dev/.+|) &&
1278 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1279 ($drive->{file
} !~ m/^\d+$/)) {
1280 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1281 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1283 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1284 verify_media_type
($opt, $vtype, $drive->{media
});
1285 $drive->{file
} = $volid;
1288 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1291 sub parse_hotplug_features
{
1296 return $res if $data eq '0';
1298 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1300 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1301 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1304 die "invalid hotplug feature '$feature'\n";
1310 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1311 sub pve_verify_hotplug_features
{
1312 my ($value, $noerr) = @_;
1314 return $value if parse_hotplug_features
($value);
1318 die "unable to parse hotplug option\n";
1322 my($fh, $noerr) = @_;
1325 my $SG_GET_VERSION_NUM = 0x2282;
1327 my $versionbuf = "\x00" x
8;
1328 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1330 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1333 my $version = unpack("I", $versionbuf);
1334 if ($version < 30000) {
1335 die "scsi generic interface too old\n" if !$noerr;
1339 my $buf = "\x00" x
36;
1340 my $sensebuf = "\x00" x
8;
1341 my $cmd = pack("C x3 C x1", 0x12, 36);
1343 # see /usr/include/scsi/sg.h
1344 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1346 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1347 length($sensebuf), 0, length($buf), $buf,
1348 $cmd, $sensebuf, 6000);
1350 $ret = ioctl($fh, $SG_IO, $packet);
1352 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1356 my @res = unpack($sg_io_hdr_t, $packet);
1357 if ($res[17] || $res[18]) {
1358 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363 (my $byte0, my $byte1, $res->{vendor
},
1364 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1366 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1367 $res->{type
} = $byte0 & 31;
1375 my $fh = IO
::File-
>new("+<$path") || return;
1376 my $res = scsi_inquiry
($fh, 1);
1382 sub print_tabletdevice_full
{
1383 my ($conf, $arch) = @_;
1385 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1387 # we use uhci for old VMs because tablet driver was buggy in older qemu
1389 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1395 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1398 sub print_keyboarddevice_full
{
1399 my ($conf, $arch, $machine) = @_;
1401 return if $arch ne 'aarch64';
1403 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1406 sub print_drivedevice_full
{
1407 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1412 my $drive_id = "$drive->{interface}$drive->{index}";
1413 if ($drive->{interface
} eq 'virtio') {
1414 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1415 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1416 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1417 } elsif ($drive->{interface
} eq 'scsi') {
1419 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1420 my $unit = $drive->{index} % $maxdev;
1421 my $devicetype = 'hd';
1423 if (drive_is_cdrom
($drive)) {
1426 if ($drive->{file
} =~ m
|^/|) {
1427 $path = $drive->{file
};
1428 if (my $info = path_is_scsi
($path)) {
1429 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1430 $devicetype = 'block';
1431 } elsif ($info->{type
} == 1) { # tape
1432 $devicetype = 'generic';
1436 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1439 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1440 my $version = extract_version
($machine_type, kvm_user_version
());
1441 if ($path =~ m/^iscsi\:\/\
// &&
1442 !min_version
($version, 4, 1)) {
1443 $devicetype = 'generic';
1447 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1448 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1450 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1451 .",lun=$drive->{index}";
1453 $device .= ",drive=drive-$drive_id,id=$drive_id";
1455 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1456 $device .= ",rotation_rate=1";
1458 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1460 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1461 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1462 my $controller = int($drive->{index} / $maxdev);
1463 my $unit = $drive->{index} % $maxdev;
1464 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1466 $device = "ide-$devicetype";
1467 if ($drive->{interface
} eq 'ide') {
1468 $device .= ",bus=ide.$controller,unit=$unit";
1470 $device .= ",bus=ahci$controller.$unit";
1472 $device .= ",drive=drive-$drive_id,id=$drive_id";
1474 if ($devicetype eq 'hd') {
1475 if (my $model = $drive->{model
}) {
1476 $model = URI
::Escape
::uri_unescape
($model);
1477 $device .= ",model=$model";
1479 if ($drive->{ssd
}) {
1480 $device .= ",rotation_rate=1";
1483 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1484 } elsif ($drive->{interface
} eq 'usb') {
1486 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1488 die "unsupported interface type";
1491 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1493 if (my $serial = $drive->{serial
}) {
1494 $serial = URI
::Escape
::uri_unescape
($serial);
1495 $device .= ",serial=$serial";
1502 sub get_initiator_name
{
1505 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1506 while (defined(my $line = <$fh>)) {
1507 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1516 sub print_drive_commandline_full
{
1517 my ($storecfg, $vmid, $drive) = @_;
1520 my $volid = $drive->{file
};
1523 if (drive_is_cdrom
($drive)) {
1524 $path = get_iso_path
($storecfg, $vmid, $volid);
1526 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1528 $path = PVE
::Storage
::path
($storecfg, $volid);
1529 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1530 $format = qemu_img_format
($scfg, $volname);
1538 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1539 foreach my $o (@qemu_drive_options) {
1540 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1543 # snapshot only accepts on|off
1544 if (defined($drive->{snapshot
})) {
1545 my $v = $drive->{snapshot
} ?
'on' : 'off';
1546 $opts .= ",snapshot=$v";
1549 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1550 my ($dir, $qmpname) = @$type;
1551 if (my $v = $drive->{"mbps$dir"}) {
1552 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1554 if (my $v = $drive->{"mbps${dir}_max"}) {
1555 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1557 if (my $v = $drive->{"bps${dir}_max_length"}) {
1558 $opts .= ",throttling.bps$qmpname-max-length=$v";
1560 if (my $v = $drive->{"iops${dir}"}) {
1561 $opts .= ",throttling.iops$qmpname=$v";
1563 if (my $v = $drive->{"iops${dir}_max"}) {
1564 $opts .= ",throttling.iops$qmpname-max=$v";
1566 if (my $v = $drive->{"iops${dir}_max_length"}) {
1567 $opts .= ",throttling.iops$qmpname-max-length=$v";
1571 $opts .= ",format=$format" if $format && !$drive->{format
};
1573 my $cache_direct = 0;
1575 if (my $cache = $drive->{cache
}) {
1576 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1577 } elsif (!drive_is_cdrom
($drive)) {
1578 $opts .= ",cache=none";
1582 # aio native works only with O_DIRECT
1583 if (!$drive->{aio
}) {
1585 $opts .= ",aio=native";
1587 $opts .= ",aio=threads";
1591 if (!drive_is_cdrom
($drive)) {
1593 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1594 $detectzeroes = 'off';
1595 } elsif ($drive->{discard
}) {
1596 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1598 # This used to be our default with discard not being specified:
1599 $detectzeroes = 'on';
1601 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1604 my $pathinfo = $path ?
"file=$path," : '';
1606 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1609 sub print_netdevice_full
{
1610 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1612 my $device = $net->{model
};
1613 if ($net->{model
} eq 'virtio') {
1614 $device = 'virtio-net-pci';
1617 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1618 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1619 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1620 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1621 # and out of each queue plus one config interrupt and control vector queue
1622 my $vectors = $net->{queues
} * 2 + 2;
1623 $tmpstr .= ",vectors=$vectors,mq=on";
1625 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1627 if (my $mtu = $net->{mtu
}) {
1628 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1629 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1632 } elsif ($mtu < 576) {
1633 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1634 } elsif ($mtu > $bridge_mtu) {
1635 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1637 $tmpstr .= ",host_mtu=$mtu";
1639 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1643 if ($use_old_bios_files) {
1645 if ($device eq 'virtio-net-pci') {
1646 $romfile = 'pxe-virtio.rom';
1647 } elsif ($device eq 'e1000') {
1648 $romfile = 'pxe-e1000.rom';
1649 } elsif ($device eq 'ne2k') {
1650 $romfile = 'pxe-ne2k_pci.rom';
1651 } elsif ($device eq 'pcnet') {
1652 $romfile = 'pxe-pcnet.rom';
1653 } elsif ($device eq 'rtl8139') {
1654 $romfile = 'pxe-rtl8139.rom';
1656 $tmpstr .= ",romfile=$romfile" if $romfile;
1662 sub print_netdev_full
{
1663 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1666 if ($netid =~ m/^net(\d+)$/) {
1670 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1672 my $ifname = "tap${vmid}i$i";
1674 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1675 die "interface name '$ifname' is too long (max 15 character)\n"
1676 if length($ifname) >= 16;
1678 my $vhostparam = '';
1679 if (is_native
($arch)) {
1680 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1683 my $vmname = $conf->{name
} || "vm$vmid";
1686 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1688 if ($net->{bridge
}) {
1689 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1690 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1692 $netdev = "type=user,id=$netid,hostname=$vmname";
1695 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1701 'cirrus' => 'cirrus-vga',
1703 'vmware' => 'vmware-svga',
1704 'virtio' => 'virtio-vga',
1707 sub print_vga_device
{
1708 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1710 my $type = $vga_map->{$vga->{type
}};
1711 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1712 $type = 'virtio-gpu';
1714 my $vgamem_mb = $vga->{memory
};
1716 my $max_outputs = '';
1718 $type = $id ?
'qxl' : 'qxl-vga';
1720 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1721 # set max outputs so linux can have up to 4 qxl displays with one device
1722 if (min_version
($machine_version, 4, 1)) {
1723 $max_outputs = ",max_outputs=4";
1728 die "no devicetype for $vga->{type}\n" if !$type;
1732 if ($vga->{type
} eq 'virtio') {
1733 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1734 $memory = ",max_hostmem=$bytes";
1736 # from https://www.spice-space.org/multiple-monitors.html
1737 $memory = ",vgamem_mb=$vga->{memory}";
1738 my $ram = $vgamem_mb * 4;
1739 my $vram = $vgamem_mb * 2;
1740 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1742 $memory = ",vgamem_mb=$vga->{memory}";
1744 } elsif ($qxlnum && $id) {
1745 $memory = ",ram_size=67108864,vram_size=33554432";
1749 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1750 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1753 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1754 my $vgaid = "vga" . ($id // '');
1757 if ($q35 && $vgaid eq 'vga') {
1758 # the first display uses pcie.0 bus on q35 machines
1759 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1761 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1764 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1767 sub parse_number_sets
{
1770 foreach my $part (split(/;/, $set)) {
1771 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1772 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1773 push @$res, [ $1, $2 ];
1775 die "invalid range: $part\n";
1784 my $res = parse_property_string
($numa_fmt, $data);
1785 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1786 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1790 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1794 my $res = eval { parse_property_string
($net_fmt, $data) };
1799 if (!defined($res->{macaddr
})) {
1800 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1801 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1806 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1807 sub parse_ipconfig
{
1810 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1816 if ($res->{gw
} && !$res->{ip
}) {
1817 warn 'gateway specified without specifying an IP address';
1820 if ($res->{gw6
} && !$res->{ip6
}) {
1821 warn 'IPv6 gateway specified without specifying an IPv6 address';
1824 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1825 warn 'gateway specified together with DHCP';
1828 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1830 warn "IPv6 gateway specified together with $res->{ip6} address";
1834 if (!$res->{ip
} && !$res->{ip6
}) {
1835 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1844 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1847 sub add_random_macs
{
1848 my ($settings) = @_;
1850 foreach my $opt (keys %$settings) {
1851 next if $opt !~ m/^net(\d+)$/;
1852 my $net = parse_net
($settings->{$opt});
1854 $settings->{$opt} = print_net
($net);
1858 sub vm_is_volid_owner
{
1859 my ($storecfg, $vmid, $volid) = @_;
1861 if ($volid !~ m
|^/|) {
1863 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1864 if ($owner && ($owner == $vmid)) {
1872 sub vmconfig_register_unused_drive
{
1873 my ($storecfg, $vmid, $conf, $drive) = @_;
1875 if (drive_is_cloudinit
($drive)) {
1876 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1878 } elsif (!drive_is_cdrom
($drive)) {
1879 my $volid = $drive->{file
};
1880 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1881 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1886 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1890 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1891 format_description
=> 'UUID',
1892 description
=> "Set SMBIOS1 UUID.",
1897 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1898 format_description
=> 'Base64 encoded string',
1899 description
=> "Set SMBIOS1 version.",
1904 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1905 format_description
=> 'Base64 encoded string',
1906 description
=> "Set SMBIOS1 serial number.",
1911 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1912 format_description
=> 'Base64 encoded string',
1913 description
=> "Set SMBIOS1 manufacturer.",
1918 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1919 format_description
=> 'Base64 encoded string',
1920 description
=> "Set SMBIOS1 product ID.",
1925 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1926 format_description
=> 'Base64 encoded string',
1927 description
=> "Set SMBIOS1 SKU string.",
1932 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1933 format_description
=> 'Base64 encoded string',
1934 description
=> "Set SMBIOS1 family string.",
1939 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
1947 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
1954 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
1957 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
1959 sub parse_watchdog
{
1964 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
1969 sub parse_guest_agent
{
1972 return {} if !defined($conf->{agent
});
1974 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
1977 # if the agent is disabled ignore the other potentially set properties
1978 return {} if !$res->{enabled
};
1983 my ($conf, $key) = @_;
1984 return undef if !defined($conf->{agent
});
1986 my $agent = parse_guest_agent
($conf);
1987 return $agent->{$key};
1993 return {} if !$value;
1994 my $res = eval { parse_property_string
($vga_fmt, $value) };
2004 my $res = eval { parse_property_string
($rng_fmt, $value) };
2009 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2010 sub verify_usb_device
{
2011 my ($value, $noerr) = @_;
2013 return $value if parse_usb_device
($value);
2017 die "unable to parse usb device\n";
2020 # add JSON properties for create and set function
2021 sub json_config_properties
{
2024 foreach my $opt (keys %$confdesc) {
2025 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2026 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2027 $prop->{$opt} = $confdesc->{$opt};
2033 # return copy of $confdesc_cloudinit to generate documentation
2034 sub cloudinit_config_properties
{
2036 return dclone
($confdesc_cloudinit);
2040 my ($key, $value) = @_;
2042 die "unknown setting '$key'\n" if !$confdesc->{$key};
2044 my $type = $confdesc->{$key}->{type
};
2046 if (!defined($value)) {
2047 die "got undefined value\n";
2050 if ($value =~ m/[\n\r]/) {
2051 die "property contains a line feed\n";
2054 if ($type eq 'boolean') {
2055 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2056 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2057 die "type check ('boolean') failed - got '$value'\n";
2058 } elsif ($type eq 'integer') {
2059 return int($1) if $value =~ m/^(\d+)$/;
2060 die "type check ('integer') failed - got '$value'\n";
2061 } elsif ($type eq 'number') {
2062 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2063 die "type check ('number') failed - got '$value'\n";
2064 } elsif ($type eq 'string') {
2065 if (my $fmt = $confdesc->{$key}->{format
}) {
2066 PVE
::JSONSchema
::check_format
($fmt, $value);
2069 $value =~ s/^\"(.*)\"$/$1/;
2072 die "internal error"
2077 my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2079 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2081 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2083 if ($conf->{template
}) {
2084 # check if any base image is still used by a linked clone
2085 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2086 my ($ds, $drive) = @_;
2087 return if drive_is_cdrom
($drive);
2089 my $volid = $drive->{file
};
2090 return if !$volid || $volid =~ m
|^/|;
2092 die "base volume '$volid' is still in use by linked cloned\n"
2093 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2098 # only remove disks owned by this VM
2099 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2100 my ($ds, $drive) = @_;
2101 return if drive_is_cdrom
($drive, 1);
2103 my $volid = $drive->{file
};
2104 return if !$volid || $volid =~ m
|^/|;
2106 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2107 return if !$path || !$owner || ($owner != $vmid);
2109 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2110 warn "Could not remove disk '$volid', check manually: $@" if $@;
2113 # also remove unused disk
2114 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2115 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2116 my ($volid, $sid, $volname, $d) = @_;
2117 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2121 if (defined $replacement_conf) {
2122 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2124 PVE
::QemuConfig-
>destroy_config($vmid);
2128 sub parse_vm_config
{
2129 my ($filename, $raw) = @_;
2131 return if !defined($raw);
2134 digest
=> Digest
::SHA
::sha1_hex
($raw),
2139 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2140 || die "got strange filename '$filename'";
2148 my @lines = split(/\n/, $raw);
2149 foreach my $line (@lines) {
2150 next if $line =~ m/^\s*$/;
2152 if ($line =~ m/^\[PENDING\]\s*$/i) {
2153 $section = 'pending';
2154 if (defined($descr)) {
2156 $conf->{description
} = $descr;
2159 $conf = $res->{$section} = {};
2162 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2164 if (defined($descr)) {
2166 $conf->{description
} = $descr;
2169 $conf = $res->{snapshots
}->{$section} = {};
2173 if ($line =~ m/^\#(.*)\s*$/) {
2174 $descr = '' if !defined($descr);
2175 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2179 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2180 $descr = '' if !defined($descr);
2181 $descr .= PVE
::Tools
::decode_text
($2);
2182 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2183 $conf->{snapstate
} = $1;
2184 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2187 $conf->{$key} = $value;
2188 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2190 if ($section eq 'pending') {
2191 $conf->{delete} = $value; # we parse this later
2193 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2195 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2198 eval { $value = check_type
($key, $value); };
2200 warn "vm $vmid - unable to parse value of '$key' - $@";
2202 $key = 'ide2' if $key eq 'cdrom';
2203 my $fmt = $confdesc->{$key}->{format
};
2204 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2205 my $v = parse_drive
($key, $value);
2206 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2207 $v->{file
} = $volid;
2208 $value = print_drive
($v);
2210 warn "vm $vmid - unable to parse value of '$key'\n";
2215 $conf->{$key} = $value;
2220 if (defined($descr)) {
2222 $conf->{description
} = $descr;
2224 delete $res->{snapstate
}; # just to be sure
2229 sub write_vm_config
{
2230 my ($filename, $conf) = @_;
2232 delete $conf->{snapstate
}; # just to be sure
2234 if ($conf->{cdrom
}) {
2235 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2236 $conf->{ide2
} = $conf->{cdrom
};
2237 delete $conf->{cdrom
};
2240 # we do not use 'smp' any longer
2241 if ($conf->{sockets
}) {
2242 delete $conf->{smp
};
2243 } elsif ($conf->{smp
}) {
2244 $conf->{sockets
} = $conf->{smp
};
2245 delete $conf->{cores
};
2246 delete $conf->{smp
};
2249 my $used_volids = {};
2251 my $cleanup_config = sub {
2252 my ($cref, $pending, $snapname) = @_;
2254 foreach my $key (keys %$cref) {
2255 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2256 $key eq 'snapstate' || $key eq 'pending';
2257 my $value = $cref->{$key};
2258 if ($key eq 'delete') {
2259 die "propertry 'delete' is only allowed in [PENDING]\n"
2261 # fixme: check syntax?
2264 eval { $value = check_type
($key, $value); };
2265 die "unable to parse value of '$key' - $@" if $@;
2267 $cref->{$key} = $value;
2269 if (!$snapname && is_valid_drivename
($key)) {
2270 my $drive = parse_drive
($key, $value);
2271 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2276 &$cleanup_config($conf);
2278 &$cleanup_config($conf->{pending
}, 1);
2280 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2281 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2282 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2285 # remove 'unusedX' settings if we re-add a volume
2286 foreach my $key (keys %$conf) {
2287 my $value = $conf->{$key};
2288 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2289 delete $conf->{$key};
2293 my $generate_raw_config = sub {
2294 my ($conf, $pending) = @_;
2298 # add description as comment to top of file
2299 if (defined(my $descr = $conf->{description
})) {
2301 foreach my $cl (split(/\n/, $descr)) {
2302 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2305 $raw .= "#\n" if $pending;
2309 foreach my $key (sort keys %$conf) {
2310 next if $key =~ /^(digest|description|pending|snapshots)$/;
2311 $raw .= "$key: $conf->{$key}\n";
2316 my $raw = &$generate_raw_config($conf);
2318 if (scalar(keys %{$conf->{pending
}})){
2319 $raw .= "\n[PENDING]\n";
2320 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2323 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2324 $raw .= "\n[$snapname]\n";
2325 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2335 # we use static defaults from our JSON schema configuration
2336 foreach my $key (keys %$confdesc) {
2337 if (defined(my $default = $confdesc->{$key}->{default})) {
2338 $res->{$key} = $default;
2346 my $vmlist = PVE
::Cluster
::get_vmlist
();
2348 return $res if !$vmlist || !$vmlist->{ids
};
2349 my $ids = $vmlist->{ids
};
2350 my $nodename = nodename
();
2352 foreach my $vmid (keys %$ids) {
2353 my $d = $ids->{$vmid};
2354 next if !$d->{node
} || $d->{node
} ne $nodename;
2355 next if !$d->{type
} || $d->{type
} ne 'qemu';
2356 $res->{$vmid}->{exists} = 1;
2361 # test if VM uses local resources (to prevent migration)
2362 sub check_local_resources
{
2363 my ($conf, $noerr) = @_;
2367 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2368 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2370 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2372 foreach my $k (keys %$conf) {
2373 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2374 # sockets are safe: they will recreated be on the target side post-migrate
2375 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2376 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2379 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2384 # check if used storages are available on all nodes (use by migrate)
2385 sub check_storage_availability
{
2386 my ($storecfg, $conf, $node) = @_;
2388 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2389 my ($ds, $drive) = @_;
2391 my $volid = $drive->{file
};
2394 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2397 # check if storage is available on both nodes
2398 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2399 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2403 # list nodes where all VM images are available (used by has_feature API)
2405 my ($conf, $storecfg) = @_;
2407 my $nodelist = PVE
::Cluster
::get_nodelist
();
2408 my $nodehash = { map { $_ => 1 } @$nodelist };
2409 my $nodename = nodename
();
2411 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2412 my ($ds, $drive) = @_;
2414 my $volid = $drive->{file
};
2417 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2419 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2420 if ($scfg->{disable
}) {
2422 } elsif (my $avail = $scfg->{nodes
}) {
2423 foreach my $node (keys %$nodehash) {
2424 delete $nodehash->{$node} if !$avail->{$node};
2426 } elsif (!$scfg->{shared
}) {
2427 foreach my $node (keys %$nodehash) {
2428 delete $nodehash->{$node} if $node ne $nodename
2437 sub check_local_storage_availability
{
2438 my ($conf, $storecfg) = @_;
2440 my $nodelist = PVE
::Cluster
::get_nodelist
();
2441 my $nodehash = { map { $_ => {} } @$nodelist };
2443 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2444 my ($ds, $drive) = @_;
2446 my $volid = $drive->{file
};
2449 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2451 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2453 if ($scfg->{disable
}) {
2454 foreach my $node (keys %$nodehash) {
2455 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2457 } elsif (my $avail = $scfg->{nodes
}) {
2458 foreach my $node (keys %$nodehash) {
2459 if (!$avail->{$node}) {
2460 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2467 foreach my $node (values %$nodehash) {
2468 if (my $unavail = $node->{unavailable_storages
}) {
2469 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2476 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2478 my ($vmid, $nocheck, $node) = @_;
2480 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2481 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2486 my $vzlist = config_list
();
2488 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2490 while (defined(my $de = $fd->read)) {
2491 next if $de !~ m/^(\d+)\.pid$/;
2493 next if !defined($vzlist->{$vmid});
2494 if (my $pid = check_running
($vmid)) {
2495 $vzlist->{$vmid}->{pid
} = $pid;
2502 our $vmstatus_return_properties = {
2503 vmid
=> get_standard_option
('pve-vmid'),
2505 description
=> "Qemu process status.",
2507 enum
=> ['stopped', 'running'],
2510 description
=> "Maximum memory in bytes.",
2513 renderer
=> 'bytes',
2516 description
=> "Root disk size in bytes.",
2519 renderer
=> 'bytes',
2522 description
=> "VM name.",
2527 description
=> "Qemu QMP agent status.",
2532 description
=> "PID of running qemu process.",
2537 description
=> "Uptime.",
2540 renderer
=> 'duration',
2543 description
=> "Maximum usable CPUs.",
2548 description
=> "The current config lock, if any.",
2553 description
=> "The current configured tags, if any",
2559 my $last_proc_pid_stat;
2561 # get VM status information
2562 # This must be fast and should not block ($full == false)
2563 # We only query KVM using QMP if $full == true (this can be slow)
2565 my ($opt_vmid, $full) = @_;
2569 my $storecfg = PVE
::Storage
::config
();
2571 my $list = vzlist
();
2572 my $defaults = load_defaults
();
2574 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2576 my $cpucount = $cpuinfo->{cpus
} || 1;
2578 foreach my $vmid (keys %$list) {
2579 next if $opt_vmid && ($vmid ne $opt_vmid);
2581 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2583 my $d = { vmid
=> $vmid };
2584 $d->{pid
} = $list->{$vmid}->{pid
};
2586 # fixme: better status?
2587 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2589 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2590 if (defined($size)) {
2591 $d->{disk
} = 0; # no info available
2592 $d->{maxdisk
} = $size;
2598 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2599 * ($conf->{cores
} || $defaults->{cores
});
2600 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2601 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2603 $d->{name
} = $conf->{name
} || "VM $vmid";
2604 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2605 : $defaults->{memory
}*(1024*1024);
2607 if ($conf->{balloon
}) {
2608 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2609 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2610 : $defaults->{shares
};
2621 $d->{diskwrite
} = 0;
2623 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
2625 $d->{serial
} = 1 if conf_has_serial
($conf);
2626 $d->{lock} = $conf->{lock} if $conf->{lock};
2627 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2632 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2633 foreach my $dev (keys %$netdev) {
2634 next if $dev !~ m/^tap([1-9]\d*)i/;
2636 my $d = $res->{$vmid};
2639 $d->{netout
} += $netdev->{$dev}->{receive
};
2640 $d->{netin
} += $netdev->{$dev}->{transmit
};
2643 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
2644 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
2649 my $ctime = gettimeofday
;
2651 foreach my $vmid (keys %$list) {
2653 my $d = $res->{$vmid};
2654 my $pid = $d->{pid
};
2657 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2658 next if !$pstat; # not running
2660 my $used = $pstat->{utime} + $pstat->{stime
};
2662 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2664 if ($pstat->{vsize
}) {
2665 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2668 my $old = $last_proc_pid_stat->{$pid};
2670 $last_proc_pid_stat->{$pid} = {
2678 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2680 if ($dtime > 1000) {
2681 my $dutime = $used - $old->{used
};
2683 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2684 $last_proc_pid_stat->{$pid} = {
2690 $d->{cpu
} = $old->{cpu
};
2694 return $res if !$full;
2696 my $qmpclient = PVE
::QMPClient-
>new();
2698 my $ballooncb = sub {
2699 my ($vmid, $resp) = @_;
2701 my $info = $resp->{'return'};
2702 return if !$info->{max_mem
};
2704 my $d = $res->{$vmid};
2706 # use memory assigned to VM
2707 $d->{maxmem
} = $info->{max_mem
};
2708 $d->{balloon
} = $info->{actual
};
2710 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2711 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2712 $d->{freemem
} = $info->{free_mem
};
2715 $d->{ballooninfo
} = $info;
2718 my $blockstatscb = sub {
2719 my ($vmid, $resp) = @_;
2720 my $data = $resp->{'return'} || [];
2721 my $totalrdbytes = 0;
2722 my $totalwrbytes = 0;
2724 for my $blockstat (@$data) {
2725 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2726 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2728 $blockstat->{device
} =~ s/drive-//;
2729 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2731 $res->{$vmid}->{diskread
} = $totalrdbytes;
2732 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2735 my $statuscb = sub {
2736 my ($vmid, $resp) = @_;
2738 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2739 # this fails if ballon driver is not loaded, so this must be
2740 # the last commnand (following command are aborted if this fails).
2741 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2743 my $status = 'unknown';
2744 if (!defined($status = $resp->{'return'}->{status
})) {
2745 warn "unable to get VM status\n";
2749 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2752 foreach my $vmid (keys %$list) {
2753 next if $opt_vmid && ($vmid ne $opt_vmid);
2754 next if !$res->{$vmid}->{pid
}; # not running
2755 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2758 $qmpclient->queue_execute(undef, 2);
2760 foreach my $vmid (keys %$list) {
2761 next if $opt_vmid && ($vmid ne $opt_vmid);
2762 next if !$res->{$vmid}->{pid
}; #not running
2764 # we can't use the $qmpclient since it might have already aborted on
2765 # 'query-balloon', but this might also fail for older versions...
2766 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
2767 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2770 foreach my $vmid (keys %$list) {
2771 next if $opt_vmid && ($vmid ne $opt_vmid);
2772 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2778 sub conf_has_serial
{
2781 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2782 if ($conf->{"serial$i"}) {
2790 sub conf_has_audio
{
2791 my ($conf, $id) = @_;
2794 my $audio = $conf->{"audio$id"};
2795 return if !defined($audio);
2797 my $audioproperties = parse_property_string
($audio_fmt, $audio);
2798 my $audiodriver = $audioproperties->{driver
} // 'spice';
2801 dev
=> $audioproperties->{device
},
2802 dev_id
=> "audiodev$id",
2803 backend
=> $audiodriver,
2804 backend_id
=> "$audiodriver-backend${id}",
2809 my ($audio, $audiopciaddr, $machine_version) = @_;
2813 my $id = $audio->{dev_id
};
2815 if (min_version
($machine_version, 4, 2)) {
2816 $audiodev = ",audiodev=$audio->{backend_id}";
2819 if ($audio->{dev
} eq 'AC97') {
2820 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2821 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
2822 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2823 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2824 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2826 die "unkown audio device '$audio->{dev}', implement me!";
2829 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2834 sub vga_conf_has_spice
{
2837 my $vgaconf = parse_vga
($vga);
2838 my $vgatype = $vgaconf->{type
};
2839 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2846 return get_host_arch
() eq $arch;
2851 return $conf->{arch
} // get_host_arch
();
2854 my $default_machines = {
2859 sub get_vm_machine
{
2860 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2862 my $machine = $forcemachine || $conf->{machine
};
2864 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2866 $machine ||= $default_machines->{$arch};
2867 if ($add_pve_version) {
2868 $kvmversion //= kvm_user_version
();
2869 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
2870 $machine .= "+pve$pvever";
2874 if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2875 # for version-pinned machines that do not include a pve-version (e.g.
2876 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2877 $machine .= '+pve0';
2883 sub get_ovmf_files
($) {
2886 my $ovmf = $OVMF->{$arch}
2887 or die "no OVMF images known for architecture '$arch'\n";
2893 aarch64
=> '/usr/bin/qemu-system-aarch64',
2894 x86_64
=> '/usr/bin/qemu-system-x86_64',
2896 sub get_command_for_arch
($) {
2898 return '/usr/bin/kvm' if is_native
($arch);
2900 my $cmd = $Arch2Qemu->{$arch}
2901 or die "don't know how to emulate architecture '$arch'\n";
2905 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2906 # to use in a QEMU command line (-cpu element), first array_intersect the result
2907 # of query_supported_ with query_understood_. This is necessary because:
2909 # a) query_understood_ returns flags the host cannot use and
2910 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2911 # flags, but CPU settings - with most of them being flags. Those settings
2912 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
2914 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2915 # expensive. If you need the value returned from this, you can get it much
2916 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2917 # $accel being 'kvm' or 'tcg'.
2919 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2920 # changes, automatically populating pmxcfs.
2922 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2923 # since kvm and tcg machines support different flags
2925 sub query_supported_cpu_flags
{
2928 $arch //= get_host_arch
();
2929 my $default_machine = $default_machines->{$arch};
2933 # FIXME: Once this is merged, the code below should work for ARM as well:
2934 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2935 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2938 my $kvm_supported = defined(kvm_version
());
2939 my $qemu_cmd = get_command_for_arch
($arch);
2941 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
2943 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2944 my $query_supported_run_qemu = sub {
2950 '-machine', $default_machine,
2952 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2953 '-mon', 'chardev=qmp,mode=control',
2954 '-pidfile', $pidfile,
2959 push @$cmd, '-accel', 'tcg';
2962 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
2963 die "QEMU flag querying VM exited with code " . $rc if $rc;
2966 my $cmd_result = mon_cmd
(
2968 'query-cpu-model-expansion',
2970 model
=> { name
=> 'host' }
2973 my $props = $cmd_result->{model
}->{props
};
2974 foreach my $prop (keys %$props) {
2975 next if $props->{$prop} ne '1';
2976 # QEMU returns some flags multiple times, with '_', '.' or '-'
2977 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2978 # We only keep those with underscores, to match /proc/cpuinfo
2979 $prop =~ s/\.|-/_/g;
2980 $flags->{$prop} = 1;
2985 # force stop with 10 sec timeout and 'nocheck'
2986 # always stop, even if QMP failed
2987 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
2991 return [ sort keys %$flags ];
2994 # We need to query QEMU twice, since KVM and TCG have different supported flags
2995 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
2996 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
2997 warn "warning: failed querying supported tcg flags: $@\n" if $@;
2999 if ($kvm_supported) {
3000 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3001 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3008 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3009 my $understood_cpu_flag_dir = "/usr/share/kvm";
3010 sub query_understood_cpu_flags
{
3011 my $arch = get_host_arch
();
3012 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3014 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3017 my $raw = file_get_contents
($filepath);
3018 $raw =~ s/^\s+|\s+$//g;
3019 my @flags = split(/\s+/, $raw);
3024 sub config_to_command
{
3025 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3028 my $globalFlags = [];
3029 my $machineFlags = [];
3034 my $ostype = $conf->{ostype
};
3035 my $winversion = windows_version
($ostype);
3036 my $kvm = $conf->{kvm
};
3037 my $nodename = nodename
();
3039 my $arch = get_vm_arch
($conf);
3040 my $kvm_binary = get_command_for_arch
($arch);
3041 my $kvmver = kvm_user_version
($kvm_binary);
3043 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3044 $kvmver //= "undefined";
3045 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3048 my $add_pve_version = min_version
($kvmver, 4, 1);
3050 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3051 my $machine_version = extract_version
($machine_type, $kvmver);
3052 $kvm //= 1 if is_native
($arch);
3054 $machine_version =~ m/(\d+)\.(\d+)/;
3055 my ($machine_major, $machine_minor) = ($1, $2);
3057 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3058 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3059 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3060 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3061 ." please upgrade node '$nodename'\n"
3062 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3063 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3064 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3065 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3066 ." node '$nodename'\n";
3069 # if a specific +pve version is required for a feature, use $version_guard
3070 # instead of min_version to allow machines to be run with the minimum
3072 my $required_pve_version = 0;
3073 my $version_guard = sub {
3074 my ($major, $minor, $pve) = @_;
3075 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3076 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3077 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3078 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3082 if ($kvm && !defined kvm_version
()) {
3083 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3084 ." or enable in BIOS.\n";
3087 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3088 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3089 my $use_old_bios_files = undef;
3090 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3092 my $cpuunits = defined($conf->{cpuunits
}) ?
3093 $conf->{cpuunits
} : $defaults->{cpuunits
};
3095 push @$cmd, $kvm_binary;
3097 push @$cmd, '-id', $vmid;
3099 my $vmname = $conf->{name
} || "vm$vmid";
3101 push @$cmd, '-name', $vmname;
3103 push @$cmd, '-no-shutdown';
3107 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3108 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3109 push @$cmd, '-mon', "chardev=qmp,mode=control";
3111 if (min_version
($machine_version, 2, 12)) {
3112 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3113 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3116 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3118 push @$cmd, '-daemonize';
3120 if ($conf->{smbios1
}) {
3121 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3122 if ($smbios_conf->{base64
}) {
3123 # Do not pass base64 flag to qemu
3124 delete $smbios_conf->{base64
};
3125 my $smbios_string = "";
3126 foreach my $key (keys %$smbios_conf) {
3128 if ($key eq "uuid") {
3129 $value = $smbios_conf->{uuid
}
3131 $value = decode_base64
($smbios_conf->{$key});
3133 # qemu accepts any binary data, only commas need escaping by double comma
3135 $smbios_string .= "," . $key . "=" . $value if $value;
3137 push @$cmd, '-smbios', "type=1" . $smbios_string;
3139 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3143 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3144 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3145 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3147 my ($path, $format);
3148 if (my $efidisk = $conf->{efidisk0
}) {
3149 my $d = parse_drive
('efidisk0', $efidisk);
3150 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3151 $format = $d->{format
};
3153 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3154 if (!defined($format)) {
3155 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3156 $format = qemu_img_format
($scfg, $volname);
3160 die "efidisk format must be specified\n"
3161 if !defined($format);
3164 warn "no efidisk configured! Using temporary efivars disk.\n";
3165 $path = "/tmp/$vmid-ovmf.fd";
3166 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3172 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3173 $size_str = ",size=" . (-s
$ovmf_vars);
3176 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3177 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3182 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3183 if (min_version
($machine_version, 4, 0)) {
3184 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3186 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3190 if ($conf->{vmgenid
}) {
3191 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3194 # add usb controllers
3195 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3196 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3197 push @$devices, @usbcontrollers if @usbcontrollers;
3198 my $vga = parse_vga
($conf->{vga
});
3200 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3201 $vga->{type
} = 'qxl' if $qxlnum;
3203 if (!$vga->{type
}) {
3204 if ($arch eq 'aarch64') {
3205 $vga->{type
} = 'virtio';
3206 } elsif (min_version
($machine_version, 2, 9)) {
3207 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3209 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3213 # enable absolute mouse coordinates (needed by vnc)
3215 if (defined($conf->{tablet
})) {
3216 $tablet = $conf->{tablet
};
3218 $tablet = $defaults->{tablet
};
3219 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3220 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3224 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3225 my $kbd = print_keyboarddevice_full
($conf, $arch);
3226 push @$devices, '-device', $kbd if defined($kbd);
3229 my $bootorder = device_bootorder
($conf);
3231 # host pci device passthrough
3232 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3233 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3236 my $usb_dev_features = {};
3237 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3239 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3240 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3241 push @$devices, @usbdevices if @usbdevices;
3244 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3245 if (my $path = $conf->{"serial$i"}) {
3246 if ($path eq 'socket') {
3247 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3248 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3249 # On aarch64, serial0 is the UART device. Qemu only allows
3250 # connecting UART devices via the '-serial' command line, as
3251 # the device has a fixed slot on the hardware...
3252 if ($arch eq 'aarch64' && $i == 0) {
3253 push @$devices, '-serial', "chardev:serial$i";
3255 push @$devices, '-device', "isa-serial,chardev=serial$i";
3258 die "no such serial device\n" if ! -c
$path;
3259 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3260 push @$devices, '-device', "isa-serial,chardev=serial$i";
3266 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3267 if (my $path = $conf->{"parallel$i"}) {
3268 die "no such parallel device\n" if ! -c
$path;
3269 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3270 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3271 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3275 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3276 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3277 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3278 push @$devices, @$audio_devs;
3282 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3283 $sockets = $conf->{sockets
} if $conf->{sockets
};
3285 my $cores = $conf->{cores
} || 1;
3287 my $maxcpus = $sockets * $cores;
3289 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3291 my $allowed_vcpus = $cpuinfo->{cpus
};
3293 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3294 if ($allowed_vcpus < $maxcpus);
3296 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3298 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3299 for (my $i = 2; $i <= $vcpus; $i++) {
3300 my $cpustr = print_cpu_device
($conf,$i);
3301 push @$cmd, '-device', $cpustr;
3306 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3308 push @$cmd, '-nodefaults';
3310 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3312 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3314 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3316 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3317 push @$devices, '-device', print_vga_device
(
3318 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3319 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3320 push @$cmd, '-vnc', "unix:$socket,password";
3322 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3323 push @$cmd, '-nographic';
3327 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3328 my $useLocaltime = $conf->{localtime};
3330 if ($winversion >= 5) { # windows
3331 $useLocaltime = 1 if !defined($conf->{localtime});
3333 # use time drift fix when acpi is enabled
3334 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3335 $tdf = 1 if !defined($conf->{tdf
});
3339 if ($winversion >= 6) {
3340 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3341 push @$cmd, '-no-hpet';
3344 push @$rtcFlags, 'driftfix=slew' if $tdf;
3346 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3347 push @$rtcFlags, "base=$conf->{startdate}";
3348 } elsif ($useLocaltime) {
3349 push @$rtcFlags, 'base=localtime';
3353 push @$cmd, '-cpu', $forcecpu;
3355 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3358 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3360 push @$cmd, '-S' if $conf->{freeze
};
3362 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3364 my $guest_agent = parse_guest_agent
($conf);
3366 if ($guest_agent->{enabled
}) {
3367 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3368 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3370 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3371 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3372 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3373 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3374 } elsif ($guest_agent->{type
} eq 'isa') {
3375 push @$devices, '-device', "isa-serial,chardev=qga0";
3379 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3380 if ($rng && $version_guard->(4, 1, 2)) {
3381 check_rng_source
($rng->{source
});
3383 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3384 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3385 my $limiter_str = "";
3387 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3390 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3391 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3392 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3400 for (my $i = 1; $i < $qxlnum; $i++){
3401 push @$devices, '-device', print_vga_device
(
3402 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3405 # assume other OS works like Linux
3406 my ($ram, $vram) = ("134217728", "67108864");
3407 if ($vga->{memory
}) {
3408 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3409 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3411 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3412 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3416 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3418 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3419 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3420 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3422 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3423 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3424 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3426 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3427 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3429 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3430 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3431 if ($spice_enhancement->{foldersharing
}) {
3432 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3433 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3436 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3437 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3438 if $spice_enhancement->{videostreaming
};
3440 push @$devices, '-spice', "$spice_opts";
3443 # enable balloon by default, unless explicitly disabled
3444 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3445 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3446 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3449 if ($conf->{watchdog
}) {
3450 my $wdopts = parse_watchdog
($conf->{watchdog
});
3451 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3452 my $watchdog = $wdopts->{model
} || 'i6300esb';
3453 push @$devices, '-device', "$watchdog$pciaddr";
3454 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3458 my $scsicontroller = {};
3459 my $ahcicontroller = {};
3460 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3462 # Add iscsi initiator name if available
3463 if (my $initiator = get_initiator_name
()) {
3464 push @$devices, '-iscsi', "initiator-name=$initiator";
3467 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3468 my ($ds, $drive) = @_;
3470 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3471 push @$vollist, $drive->{file
};
3474 # ignore efidisk here, already added in bios/fw handling code above
3475 return if $drive->{interface
} eq 'efidisk';
3477 $use_virtio = 1 if $ds =~ m/^virtio/;
3479 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3481 if ($drive->{interface
} eq 'virtio'){
3482 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3485 if ($drive->{interface
} eq 'scsi') {
3487 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3489 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3490 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3492 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3493 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3496 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3497 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3498 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3499 } elsif ($drive->{iothread
}) {
3500 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3504 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3505 $queues = ",num_queues=$drive->{queues}";
3508 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3509 if !$scsicontroller->{$controller};
3510 $scsicontroller->{$controller}=1;
3513 if ($drive->{interface
} eq 'sata') {
3514 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3515 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3516 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3517 if !$ahcicontroller->{$controller};
3518 $ahcicontroller->{$controller}=1;
3521 my $drive_cmd = print_drive_commandline_full
($storecfg, $vmid, $drive);
3522 $drive_cmd .= ',readonly' if PVE
::QemuConfig-
>is_template($conf);
3524 push @$devices, '-drive',$drive_cmd;
3525 push @$devices, '-device', print_drivedevice_full
(
3526 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3529 for (my $i = 0; $i < $MAX_NETS; $i++) {
3530 my $netname = "net$i";
3532 next if !$conf->{$netname};
3533 my $d = parse_net
($conf->{$netname});
3536 $use_virtio = 1 if $d->{model
} eq 'virtio';
3538 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
3540 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
3541 push @$devices, '-netdev', $netdevfull;
3543 my $netdevicefull = print_netdevice_full
(
3544 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3546 push @$devices, '-device', $netdevicefull;
3549 if ($conf->{ivshmem
}) {
3550 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3554 $bus = print_pcie_addr
("ivshmem");
3556 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3559 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3560 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3562 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3563 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3564 .",size=$ivshmem->{size}M";
3567 # pci.4 is nested in pci.1
3568 $bridges->{1} = 1 if $bridges->{4};
3572 if (min_version
($machine_version, 2, 3)) {
3577 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3581 for my $k (sort {$b cmp $a} keys %$bridges) {
3582 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3585 if ($k == 2 && $legacy_igd) {
3588 $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
3590 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3592 # add after -readconfig pve-q35.cfg
3593 splice @$devices, 2, 0, '-device', $devstr;
3595 unshift @$devices, '-device', $devstr if $k > 0;
3600 push @$machineFlags, 'accel=tcg';
3603 my $machine_type_min = $machine_type;
3604 if ($add_pve_version) {
3605 $machine_type_min =~ s/\+pve\d+$//;
3606 $machine_type_min .= "+pve$required_pve_version";
3608 push @$machineFlags, "type=${machine_type_min}";
3610 push @$cmd, @$devices;
3611 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3612 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3613 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3615 if (my $vmstate = $conf->{vmstate
}) {
3616 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3617 push @$vollist, $vmstate;
3618 push @$cmd, '-loadstate', $statepath;
3619 print "activating and using '$vmstate' as vmstate\n";
3623 if ($conf->{args
}) {
3624 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3628 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3631 sub check_rng_source
{
3634 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3635 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3638 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3639 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
3640 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3641 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3642 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3643 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3651 my $res = mon_cmd
($vmid, 'query-spice');
3653 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3656 sub vm_devices_list
{
3659 my $res = mon_cmd
($vmid, 'query-pci');
3660 my $devices_to_check = [];
3662 foreach my $pcibus (@$res) {
3663 push @$devices_to_check, @{$pcibus->{devices
}},
3666 while (@$devices_to_check) {
3668 for my $d (@$devices_to_check) {
3669 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3670 next if !$d->{'pci_bridge'};
3672 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3673 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3675 $devices_to_check = $to_check;
3678 my $resblock = mon_cmd
($vmid, 'query-block');
3679 foreach my $block (@$resblock) {
3680 if($block->{device
} =~ m/^drive-(\S+)/){
3685 my $resmice = mon_cmd
($vmid, 'query-mice');
3686 foreach my $mice (@$resmice) {
3687 if ($mice->{name
} eq 'QEMU HID Tablet') {
3688 $devices->{tablet
} = 1;
3693 # for usb devices there is no query-usb
3694 # but we can iterate over the entries in
3695 # qom-list path=/machine/peripheral
3696 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3697 foreach my $per (@$resperipheral) {
3698 if ($per->{name
} =~ m/^usb\d+$/) {
3699 $devices->{$per->{name
}} = 1;
3707 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3709 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3711 my $devices_list = vm_devices_list
($vmid);
3712 return 1 if defined($devices_list->{$deviceid});
3714 # add PCI bridge if we need it for the device
3715 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3717 if ($deviceid eq 'tablet') {
3719 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
3721 } elsif ($deviceid eq 'keyboard') {
3723 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
3725 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3727 die "usb hotplug currently not reliable\n";
3728 # since we can't reliably hot unplug all added usb devices and usb
3729 # passthrough breaks live migration we disable usb hotplugging for now
3730 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3732 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3734 qemu_iothread_add
($vmid, $deviceid, $device);
3736 qemu_driveadd
($storecfg, $vmid, $device);
3737 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3739 qemu_deviceadd
($vmid, $devicefull);
3740 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3742 eval { qemu_drivedel
($vmid, $deviceid); };
3747 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3750 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3751 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3752 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3754 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3756 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3757 qemu_iothread_add
($vmid, $deviceid, $device);
3758 $devicefull .= ",iothread=iothread-$deviceid";
3761 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3762 $devicefull .= ",num_queues=$device->{queues}";
3765 qemu_deviceadd
($vmid, $devicefull);
3766 qemu_deviceaddverify
($vmid, $deviceid);
3768 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3770 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3771 qemu_driveadd
($storecfg, $vmid, $device);
3773 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3774 eval { qemu_deviceadd
($vmid, $devicefull); };
3776 eval { qemu_drivedel
($vmid, $deviceid); };
3781 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3783 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
3785 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
3786 my $use_old_bios_files = undef;
3787 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3789 my $netdevicefull = print_netdevice_full
(
3790 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3791 qemu_deviceadd
($vmid, $netdevicefull);
3793 qemu_deviceaddverify
($vmid, $deviceid);
3794 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
3797 eval { qemu_netdevdel
($vmid, $deviceid); };
3802 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3805 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3806 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3808 qemu_deviceadd
($vmid, $devicefull);
3809 qemu_deviceaddverify
($vmid, $deviceid);
3812 die "can't hotplug device '$deviceid'\n";
3818 # fixme: this should raise exceptions on error!
3819 sub vm_deviceunplug
{
3820 my ($vmid, $conf, $deviceid) = @_;
3822 my $devices_list = vm_devices_list
($vmid);
3823 return 1 if !defined($devices_list->{$deviceid});
3825 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
3826 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3828 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3830 qemu_devicedel
($vmid, $deviceid);
3832 } elsif ($deviceid =~ m/^usb\d+$/) {
3834 die "usb hotplug currently not reliable\n";
3835 # when unplugging usb devices this way, there may be remaining usb
3836 # controllers/hubs so we disable it for now
3837 #qemu_devicedel($vmid, $deviceid);
3838 #qemu_devicedelverify($vmid, $deviceid);
3840 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3842 qemu_devicedel
($vmid, $deviceid);
3843 qemu_devicedelverify
($vmid, $deviceid);
3844 qemu_drivedel
($vmid, $deviceid);
3845 qemu_iothread_del
($conf, $vmid, $deviceid);
3847 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3849 qemu_devicedel
($vmid, $deviceid);
3850 qemu_devicedelverify
($vmid, $deviceid);
3851 qemu_iothread_del
($conf, $vmid, $deviceid);
3853 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3855 qemu_devicedel
($vmid, $deviceid);
3856 qemu_drivedel
($vmid, $deviceid);
3857 qemu_deletescsihw
($conf, $vmid, $deviceid);
3859 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3861 qemu_devicedel
($vmid, $deviceid);
3862 qemu_devicedelverify
($vmid, $deviceid);
3863 qemu_netdevdel
($vmid, $deviceid);
3866 die "can't unplug device '$deviceid'\n";
3872 sub qemu_deviceadd
{
3873 my ($vmid, $devicefull) = @_;
3875 $devicefull = "driver=".$devicefull;
3876 my %options = split(/[=,]/, $devicefull);
3878 mon_cmd
($vmid, "device_add" , %options);
3881 sub qemu_devicedel
{
3882 my ($vmid, $deviceid) = @_;
3884 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
3887 sub qemu_iothread_add
{
3888 my($vmid, $deviceid, $device) = @_;
3890 if ($device->{iothread
}) {
3891 my $iothreads = vm_iothreads_list
($vmid);
3892 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3896 sub qemu_iothread_del
{
3897 my($conf, $vmid, $deviceid) = @_;
3899 my $confid = $deviceid;
3900 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3901 $confid = 'scsi' . $1;
3903 my $device = parse_drive
($confid, $conf->{$confid});
3904 if ($device->{iothread
}) {
3905 my $iothreads = vm_iothreads_list
($vmid);
3906 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3910 sub qemu_objectadd
{
3911 my($vmid, $objectid, $qomtype) = @_;
3913 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
3918 sub qemu_objectdel
{
3919 my($vmid, $objectid) = @_;
3921 mon_cmd
($vmid, "object-del", id
=> $objectid);
3927 my ($storecfg, $vmid, $device) = @_;
3929 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device);
3930 $drive =~ s/\\/\\\\/g;
3931 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
3933 # If the command succeeds qemu prints: "OK
"
3934 return 1 if $ret =~ m/OK/s;
3936 die "adding drive failed
: $ret\n";
3940 my($vmid, $deviceid) = @_;
3942 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
3945 return 1 if $ret eq "";
3947 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3948 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3950 die "deleting drive
$deviceid failed
: $ret\n";
3953 sub qemu_deviceaddverify {
3954 my ($vmid, $deviceid) = @_;
3956 for (my $i = 0; $i <= 5; $i++) {
3957 my $devices_list = vm_devices_list($vmid);
3958 return 1 if defined($devices_list->{$deviceid});
3962 die "error on hotplug device
'$deviceid'\n";
3966 sub qemu_devicedelverify {
3967 my ($vmid, $deviceid) = @_;
3969 # need to verify that the device is correctly removed as device_del
3970 # is async and empty return is not reliable
3972 for (my $i = 0; $i <= 5; $i++) {
3973 my $devices_list = vm_devices_list($vmid);
3974 return 1 if !defined($devices_list->{$deviceid});
3978 die "error on hot-unplugging device
'$deviceid'\n";
3981 sub qemu_findorcreatescsihw {
3982 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3984 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3986 my $scsihwid="$controller_prefix$controller";
3987 my $devices_list = vm_devices_list($vmid);
3989 if(!defined($devices_list->{$scsihwid})) {
3990 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3996 sub qemu_deletescsihw {
3997 my ($conf, $vmid, $opt) = @_;
3999 my $device = parse_drive($opt, $conf->{$opt});
4001 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4002 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4006 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4008 my $devices_list = vm_devices_list($vmid);
4009 foreach my $opt (keys %{$devices_list}) {
4010 if (is_valid_drivename($opt)) {
4011 my $drive = parse_drive($opt, $conf->{$opt});
4012 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4018 my $scsihwid="scsihw
$controller";
4020 vm_deviceunplug($vmid, $conf, $scsihwid);
4025 sub qemu_add_pci_bridge {
4026 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4032 print_pci_addr($device, $bridges, $arch, $machine_type);
4034 while (my ($k, $v) = each %$bridges) {
4037 return 1 if !defined($bridgeid) || $bridgeid < 1;
4039 my $bridge = "pci
.$bridgeid";
4040 my $devices_list = vm_devices_list($vmid);
4042 if (!defined($devices_list->{$bridge})) {
4043 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4049 sub qemu_set_link_status {
4050 my ($vmid, $device, $up) = @_;
4052 mon_cmd($vmid, "set_link
", name => $device,
4053 up => $up ? JSON::true : JSON::false);
4056 sub qemu_netdevadd {
4057 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4059 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4060 my %options = split(/[=,]/, $netdev);
4062 if (defined(my $vhost = $options{vhost})) {
4063 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4066 if (defined(my $queues = $options{queues})) {
4067 $options{queues} = $queues + 0;
4070 mon_cmd($vmid, "netdev_add
", %options);
4074 sub qemu_netdevdel {
4075 my ($vmid, $deviceid) = @_;
4077 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4080 sub qemu_usb_hotplug {
4081 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4085 # remove the old one first
4086 vm_deviceunplug($vmid, $conf, $deviceid);
4088 # check if xhci controller is necessary and available
4089 if ($device->{usb3}) {
4091 my $devicelist = vm_devices_list($vmid);
4093 if (!$devicelist->{xhci}) {
4094 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4095 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4098 my $d = parse_usb_device($device->{host});
4099 $d->{usb3} = $device->{usb3};
4102 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4105 sub qemu_cpu_hotplug {
4106 my ($vmid, $conf, $vcpus) = @_;
4108 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4111 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4112 $sockets = $conf->{sockets} if $conf->{sockets};
4113 my $cores = $conf->{cores} || 1;
4114 my $maxcpus = $sockets * $cores;
4116 $vcpus = $maxcpus if !$vcpus;
4118 die "you can
't add more vcpus than maxcpus\n"
4119 if $vcpus > $maxcpus;
4121 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4123 if ($vcpus < $currentvcpus) {
4125 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4127 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4128 qemu_devicedel($vmid, "cpu$i");
4130 my $currentrunningvcpus = undef;
4132 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4133 last if scalar(@{$currentrunningvcpus}) == $i-1;
4134 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4138 #update conf after each succesfull cpu unplug
4139 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4140 PVE::QemuConfig->write_config($vmid, $conf);
4143 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4149 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4150 die "vcpus in running vm does not match its configuration\n"
4151 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4153 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4155 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4156 my $cpustr = print_cpu_device($conf, $i);
4157 qemu_deviceadd($vmid, $cpustr);
4160 my $currentrunningvcpus = undef;
4162 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4163 last if scalar(@{$currentrunningvcpus}) == $i;
4164 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4168 #update conf after each succesfull cpu hotplug
4169 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4170 PVE::QemuConfig->write_config($vmid, $conf);
4174 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4175 mon_cmd($vmid, "cpu-add", id => int($i));
4180 sub qemu_block_set_io_throttle {
4181 my ($vmid, $deviceid,
4182 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4183 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4184 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4185 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4187 return if !check_running($vmid) ;
4189 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4191 bps_rd => int($bps_rd),
4192 bps_wr => int($bps_wr),
4194 iops_rd => int($iops_rd),
4195 iops_wr => int($iops_wr),
4196 bps_max => int($bps_max),
4197 bps_rd_max => int($bps_rd_max),
4198 bps_wr_max => int($bps_wr_max),
4199 iops_max => int($iops_max),
4200 iops_rd_max => int($iops_rd_max),
4201 iops_wr_max => int($iops_wr_max),
4202 bps_max_length => int($bps_max_length),
4203 bps_rd_max_length => int($bps_rd_max_length),
4204 bps_wr_max_length => int($bps_wr_max_length),
4205 iops_max_length => int($iops_max_length),
4206 iops_rd_max_length => int($iops_rd_max_length),
4207 iops_wr_max_length => int($iops_wr_max_length),
4212 sub qemu_block_resize {
4213 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4215 my $running = check_running($vmid);
4217 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4219 return if !$running;
4221 my $padding = (1024 - $size % 1024) % 1024;
4222 $size = $size + $padding;
4224 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4228 sub qemu_volume_snapshot {
4229 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4231 my $running = check_running($vmid);
4233 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4234 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4236 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4240 sub qemu_volume_snapshot_delete {
4241 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4243 my $running = check_running($vmid);
4248 my $conf = PVE::QemuConfig->load_config($vmid);
4249 PVE::QemuConfig->foreach_volume($conf, sub {
4250 my ($ds, $drive) = @_;
4251 $running = 1 if $drive->{file} eq $volid;
4255 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4256 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4258 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4262 sub set_migration_caps {
4265 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4270 "auto-converge" => 1,
4272 "x-rdma-pin-all" => 0,
4275 "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration
'} ? 1 : 0,
4278 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4280 for my $supported_capability (@$supported_capabilities) {
4282 capability => $supported_capability->{capability},
4283 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4287 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4291 my ($conf, $func, @param) = @_;
4295 my $test_volid = sub {
4296 my ($key, $drive, $snapname) = @_;
4298 my $volid = $drive->{file};
4301 $volhash->{$volid}->{cdrom} //= 1;
4302 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4304 my $replicate = $drive->{replicate} // 1;
4305 $volhash->{$volid}->{replicate} //= 0;
4306 $volhash->{$volid}->{replicate} = 1 if $replicate;
4308 $volhash->{$volid}->{shared} //= 0;
4309 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4311 $volhash->{$volid}->{referenced_in_config} //= 0;
4312 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4314 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4315 if defined($snapname);
4317 my $size = $drive->{size};
4318 $volhash->{$volid}->{size} //= $size if $size;
4320 $volhash->{$volid}->{is_vmstate} //= 0;
4321 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4323 $volhash->{$volid}->{is_unused} //= 0;
4324 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4327 my $include_opts = {
4328 extra_keys => ['vmstate
'],
4329 include_unused => 1,
4332 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4333 foreach my $snapname (keys %{$conf->{snapshots}}) {
4334 my $snap = $conf->{snapshots}->{$snapname};
4335 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4338 foreach my $volid (keys %$volhash) {
4339 &$func($volid, $volhash->{$volid}, @param);
4343 my $fast_plug_option = {
4351 'vmstatestorage
' => 1,
4356 # hotplug changes in [PENDING]
4357 # $selection hash can be used to only apply specified options, for
4358 # example: { cores => 1 } (only apply changed 'cores
')
4359 # $errors ref is used to return error messages
4360 sub vmconfig_hotplug_pending {
4361 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4363 my $defaults = load_defaults();
4364 my $arch = get_vm_arch($conf);
4365 my $machine_type = get_vm_machine($conf, undef, $arch);
4367 # commit values which do not have any impact on running VM first
4368 # Note: those option cannot raise errors, we we do not care about
4369 # $selection and always apply them.
4371 my $add_error = sub {
4372 my ($opt, $msg) = @_;
4373 $errors->{$opt} = "hotplug problem - $msg";
4377 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4378 if ($fast_plug_option->{$opt}) {
4379 $conf->{$opt} = $conf->{pending}->{$opt};
4380 delete $conf->{pending}->{$opt};
4386 PVE::QemuConfig->write_config($vmid, $conf);
4389 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4391 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4392 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4393 foreach my $opt (sort keys %$pending_delete_hash) {
4394 next if $selection && !$selection->{$opt};
4395 my $force = $pending_delete_hash->{$opt}->{force};
4397 if ($opt eq 'hotplug
') {
4398 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4399 } elsif ($opt eq 'tablet
') {
4400 die "skip\n" if !$hotplug_features->{usb};
4401 if ($defaults->{tablet}) {
4402 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4403 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4404 if $arch eq 'aarch64
';
4406 vm_deviceunplug($vmid, $conf, 'tablet
');
4407 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4409 } elsif ($opt =~ m/^usb\d+/) {
4411 # since we cannot reliably hot unplug usb devices we are disabling it
4412 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4413 #vm_deviceunplug($vmid, $conf, $opt);
4414 } elsif ($opt eq 'vcpus
') {
4415 die "skip\n" if !$hotplug_features->{cpu};
4416 qemu_cpu_hotplug($vmid, $conf, undef);
4417 } elsif ($opt eq 'balloon
') {
4418 # enable balloon device is not hotpluggable
4419 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4420 # here we reset the ballooning value to memory
4421 my $balloon = $conf->{memory} || $defaults->{memory};
4422 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4423 } elsif ($fast_plug_option->{$opt}) {
4425 } elsif ($opt =~ m/^net(\d+)$/) {
4426 die "skip\n" if !$hotplug_features->{network};
4427 vm_deviceunplug($vmid, $conf, $opt);
4428 } elsif (is_valid_drivename($opt)) {
4429 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4430 vm_deviceunplug($vmid, $conf, $opt);
4431 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4432 } elsif ($opt =~ m/^memory$/) {
4433 die "skip\n" if !$hotplug_features->{memory};
4434 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4435 } elsif ($opt eq 'cpuunits
') {
4436 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4437 } elsif ($opt eq 'cpulimit
') {
4438 $cgroup->change_cpu_quota(-1, 100000);
4444 &$add_error($opt, $err) if $err ne "skip\n";
4446 delete $conf->{$opt};
4447 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4451 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4452 $apply_pending_cloudinit = sub {
4453 return if $apply_pending_cloudinit_done; # once is enough
4454 $apply_pending_cloudinit_done = 1; # once is enough
4456 my ($key, $value) = @_;
4458 my @cloudinit_opts = keys %$confdesc_cloudinit;
4459 foreach my $opt (keys %{$conf->{pending}}) {
4460 next if !grep { $_ eq $opt } @cloudinit_opts;
4461 $conf->{$opt} = delete $conf->{pending}->{$opt};
4464 my $new_conf = { %$conf };
4465 $new_conf->{$key} = $value;
4466 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4469 foreach my $opt (keys %{$conf->{pending}}) {
4470 next if $selection && !$selection->{$opt};
4471 my $value = $conf->{pending}->{$opt};
4473 if ($opt eq 'hotplug
') {
4474 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4475 } elsif ($opt eq 'tablet
') {
4476 die "skip\n" if !$hotplug_features->{usb};
4478 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4479 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4480 if $arch eq 'aarch64
';
4481 } elsif ($value == 0) {
4482 vm_deviceunplug($vmid, $conf, 'tablet
');
4483 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4485 } elsif ($opt =~ m/^usb\d+$/) {
4487 # since we cannot reliably hot unplug usb devices we disable it for now
4488 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4489 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4490 #die "skip\n" if !$d;
4491 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4492 } elsif ($opt eq 'vcpus
') {
4493 die "skip\n" if !$hotplug_features->{cpu};
4494 qemu_cpu_hotplug($vmid, $conf, $value);
4495 } elsif ($opt eq 'balloon
') {
4496 # enable/disable balloning device is not hotpluggable
4497 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4498 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4499 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4501 # allow manual ballooning if shares is set to zero
4502 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4503 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4504 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4506 } elsif ($opt =~ m/^net(\d+)$/) {
4507 # some changes can be done without hotplug
4508 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4509 $vmid, $opt, $value, $arch, $machine_type);
4510 } elsif (is_valid_drivename($opt)) {
4511 die "skip\n" if $opt eq 'efidisk0
';
4512 # some changes can be done without hotplug
4513 my $drive = parse_drive($opt, $value);
4514 if (drive_is_cloudinit($drive)) {
4515 &$apply_pending_cloudinit($opt, $value);
4517 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4518 $vmid, $opt, $value, $arch, $machine_type);
4519 } elsif ($opt =~ m/^memory$/) { #dimms
4520 die "skip\n" if !$hotplug_features->{memory};
4521 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4522 } elsif ($opt eq 'cpuunits
') {
4523 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4524 } elsif ($opt eq 'cpulimit
') {
4525 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4526 $cgroup->change_cpu_quota($cpulimit, 100000);
4528 die "skip\n"; # skip non-hot-pluggable options
4532 &$add_error($opt, $err) if $err ne "skip\n";
4534 $conf->{$opt} = $value;
4535 delete $conf->{pending}->{$opt};
4539 PVE::QemuConfig->write_config($vmid, $conf);
4542 sub try_deallocate_drive {
4543 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4545 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4546 my $volid = $drive->{file};
4547 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4548 my $sid = PVE::Storage::parse_volume_id($volid);
4549 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4551 # check if the disk is really unused
4552 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4553 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4554 PVE::Storage::vdisk_free($storecfg, $volid);
4557 # If vm is not owner of this disk remove from config
4565 sub vmconfig_delete_or_detach_drive {
4566 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4568 my $drive = parse_drive($opt, $conf->{$opt});
4570 my $rpcenv = PVE::RPCEnvironment::get();
4571 my $authuser = $rpcenv->get_user();
4574 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4575 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4577 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4583 sub vmconfig_apply_pending {
4584 my ($vmid, $conf, $storecfg, $errors) = @_;
4586 my $add_apply_error = sub {
4587 my ($opt, $msg) = @_;
4588 my $err_msg = "unable to apply pending change $opt : $msg";
4589 $errors->{$opt} = $err_msg;
4595 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4596 foreach my $opt (sort keys %$pending_delete_hash) {
4597 my $force = $pending_delete_hash->{$opt}->{force};
4599 if ($opt =~ m/^unused/) {
4600 die "internal error";
4601 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4602 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4606 $add_apply_error->($opt, $err);
4608 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4609 delete $conf->{$opt};
4613 PVE::QemuConfig->cleanup_pending($conf);
4615 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4616 next if $opt eq 'delete'; # just to be sure
4618 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4619 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4623 $add_apply_error->($opt, $err);
4625 $conf->{$opt} = delete $conf->{pending}->{$opt};
4629 # write all changes at once to avoid unnecessary i/o
4630 PVE::QemuConfig->write_config($vmid, $conf);
4633 sub vmconfig_update_net {
4634 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4636 my $newnet = parse_net($value);
4638 if ($conf->{$opt}) {
4639 my $oldnet = parse_net($conf->{$opt});
4641 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4642 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4643 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4644 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4646 # for non online change, we try to hot-unplug
4647 die "skip\n" if !$hotplug;
4648 vm_deviceunplug($vmid, $conf, $opt);
4651 die "internal error" if $opt !~ m/net(\d+)/;
4652 my $iface = "tap${vmid}i$1";
4654 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4655 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4656 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4657 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4658 PVE::Network::tap_unplug($iface);
4661 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4663 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4665 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4666 # Rate can be applied on its own but any change above needs to
4667 # include the rate in tap_plug since OVS resets everything.
4668 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4671 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4672 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4680 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4686 sub vmconfig_update_disk {
4687 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4689 my $drive = parse_drive($opt, $value);
4691 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4692 my $media = $drive->{media} || 'disk
';
4693 my $oldmedia = $old_drive->{media} || 'disk
';
4694 die "unable to change media type\n" if $media ne $oldmedia;
4696 if (!drive_is_cdrom($old_drive)) {
4698 if ($drive->{file} ne $old_drive->{file}) {
4700 die "skip\n" if !$hotplug;
4702 # unplug and register as unused
4703 vm_deviceunplug($vmid, $conf, $opt);
4704 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4707 # update existing disk
4709 # skip non hotpluggable value
4710 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4711 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4712 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4713 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4714 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4719 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4720 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4721 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4722 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4723 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4724 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4725 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4726 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4727 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4728 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4729 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4730 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4731 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4732 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4733 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4734 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4735 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4736 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4738 qemu_block_set_io_throttle(
4740 ($drive->{mbps} || 0)*1024*1024,
4741 ($drive->{mbps_rd} || 0)*1024*1024,
4742 ($drive->{mbps_wr} || 0)*1024*1024,
4743 $drive->{iops} || 0,
4744 $drive->{iops_rd} || 0,
4745 $drive->{iops_wr} || 0,
4746 ($drive->{mbps_max} || 0)*1024*1024,
4747 ($drive->{mbps_rd_max} || 0)*1024*1024,
4748 ($drive->{mbps_wr_max} || 0)*1024*1024,
4749 $drive->{iops_max} || 0,
4750 $drive->{iops_rd_max} || 0,
4751 $drive->{iops_wr_max} || 0,
4752 $drive->{bps_max_length} || 1,
4753 $drive->{bps_rd_max_length} || 1,
4754 $drive->{bps_wr_max_length} || 1,
4755 $drive->{iops_max_length} || 1,
4756 $drive->{iops_rd_max_length} || 1,
4757 $drive->{iops_wr_max_length} || 1,
4767 if ($drive->{file} eq 'none
') {
4768 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4769 if (drive_is_cloudinit($old_drive)) {
4770 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4773 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4775 # force eject if locked
4776 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4779 mon_cmd($vmid, "blockdev-change-medium",
4780 id => "$opt", filename => "$path");
4788 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4790 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4791 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4794 # called in locked context by incoming migration
4795 sub vm_migrate_get_nbd_disks {
4796 my ($storecfg, $conf, $replicated_volumes) = @_;
4798 my $local_volumes = {};
4799 PVE::QemuConfig->foreach_volume($conf, sub {
4800 my ($ds, $drive) = @_;
4802 return if drive_is_cdrom($drive);
4804 my $volid = $drive->{file};
4808 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4810 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4811 return if $scfg->{shared};
4813 # replicated disks re-use existing state via bitmap
4814 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4815 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4817 return $local_volumes;
4820 # called in locked context by incoming migration
4821 sub vm_migrate_alloc_nbd_disks {
4822 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4827 foreach my $opt (sort keys %$source_volumes) {
4828 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4830 if ($use_existing) {
4831 $nbd->{$opt}->{drivestr} = print_drive($drive);
4832 $nbd->{$opt}->{volid} = $volid;
4833 $nbd->{$opt}->{replicated} = 1;
4837 # If a remote storage is specified and the format of the original
4838 # volume is not available there, fall back to the default format.
4839 # Otherwise use the same format as the original.
4840 if (!$storagemap->{identity}) {
4841 $storeid = map_storage($storagemap, $storeid);
4842 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4843 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4844 my $fileFormat = qemu_img_format($scfg, $volname);
4845 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4847 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4848 $format = qemu_img_format($scfg, $volname);
4851 my $size = $drive->{size} / 1024;
4852 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4853 my $newdrive = $drive;
4854 $newdrive->{format} = $format;
4855 $newdrive->{file} = $newvolid;
4856 my $drivestr = print_drive($newdrive);
4857 $nbd->{$opt}->{drivestr} = $drivestr;
4858 $nbd->{$opt}->{volid} = $newvolid;
4864 # see vm_start_nolock for parameters, additionally:
4866 # storagemap = parsed storage map for allocating NBD disks
4868 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4870 return PVE::QemuConfig->lock_config($vmid, sub {
4871 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4873 die "you can't start a vm
if it
's a template\n"
4874 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4876 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended
');
4877 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup
');
4879 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4881 if ($has_backup_lock && $running) {
4882 # a backup is currently running, attempt to start the guest in the
4883 # existing QEMU instance
4884 return vm_resume($vmid);
4887 PVE::QemuConfig->check_lock($conf)
4888 if !($params->{skiplock} || $has_suspended_lock);
4890 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4892 die "VM $vmid already running\n" if $running;
4894 if (my $storagemap = $migrate_opts->{storagemap}) {
4895 my $replicated = $migrate_opts->{replicated_volumes};
4896 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4897 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4899 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4900 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4904 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4910 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
4911 # skiplock => 0/1, skip checking for config lock
4912 # skiptemplate => 0/1, skip checking whether VM is template
4913 # forcemachine => to force Qemu machine (rollback/migration)
4914 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
4915 # timeout => in seconds
4916 # paused => start VM in paused state (backup)
4917 # resume => resume from hibernation
4919 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4920 # migratedfrom => source node
4921 # spice_ticket => used for spice migration, passed via tunnel/stdin
4922 # network => CIDR of migration network
4923 # type => secure/insecure - tunnel over encrypted connection or plain-text
4924 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
4925 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4926 sub vm_start_nolock {
4927 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4929 my $statefile = $params->{statefile};
4930 my $resume = $params->{resume};
4932 my $migratedfrom = $migrate_opts->{migratedfrom};
4933 my $migration_type = $migrate_opts->{type};
4937 # clean up leftover reboot request files
4938 eval { clear_reboot_request($vmid); };
4941 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4942 vmconfig_apply_pending($vmid, $conf, $storecfg);
4943 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4946 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4948 my $defaults = load_defaults();
4950 # set environment variable useful inside network script
4951 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4953 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
4955 my $forcemachine = $params->{forcemachine};
4956 my $forcecpu = $params->{forcecpu};
4958 # enforce machine and CPU type on suspended vm to ensure HW compatibility
4959 $forcemachine = $conf->{runningmachine};
4960 $forcecpu = $conf->{runningcpu};
4961 print "Resuming suspended VM\n";
4964 my ($cmd, $vollist, $spice_port) =
4965 config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4968 my $get_migration_ip = sub {
4969 my ($nodename) = @_;
4971 return $migration_ip if defined($migration_ip);
4973 my $cidr = $migrate_opts->{network};
4975 if (!defined($cidr)) {
4976 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
4977 $cidr = $dc_conf->{migration}->{network};
4980 if (defined($cidr)) {
4981 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4983 die "could not get IP: no address configured on local " .
4984 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4986 die "could not get IP: multiple addresses configured on local " .
4987 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4989 $migration_ip = @$ips[0];
4992 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4993 if !defined($migration_ip);
4995 return $migration_ip;
5000 if ($statefile eq 'tcp
') {
5001 my $localip = "localhost";
5002 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5003 my $nodename = nodename();
5005 if (!defined($migration_type)) {
5006 if (defined($datacenterconf->{migration}->{type})) {
5007 $migration_type = $datacenterconf->{migration}->{type};
5009 $migration_type = 'secure
';
5013 if ($migration_type eq 'insecure
') {
5014 $localip = $get_migration_ip->($nodename);
5015 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5018 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5019 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5020 $migrate_uri = "tcp:${localip}:${migrate_port}";
5021 push @$cmd, '-incoming
', $migrate_uri;
5024 } elsif ($statefile eq 'unix
') {
5025 # should be default for secure migrations as a ssh TCP forward
5026 # tunnel is not deterministic reliable ready and fails regurarly
5027 # to set up in time, so use UNIX socket forwards
5028 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5029 unlink $socket_addr;
5031 $migrate_uri = "unix:$socket_addr";
5033 push @$cmd, '-incoming
', $migrate_uri;
5036 } elsif (-e $statefile) {
5037 push @$cmd, '-loadstate
', $statefile;
5039 my $statepath = PVE::Storage::path($storecfg, $statefile);
5040 push @$vollist, $statefile;
5041 push @$cmd, '-loadstate
', $statepath;
5043 } elsif ($params->{paused}) {
5048 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
5049 my $d = parse_hostpci($conf->{"hostpci$i"});
5051 my $pcidevices = $d->{pciid};
5052 foreach my $pcidevice (@$pcidevices) {
5053 my $pciid = $pcidevice->{id};
5055 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5056 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5057 die "no pci device info for device '$pciid'\n" if !$info;
5060 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5061 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5063 die "can't unbind
/bind PCI group to VFIO
'$pciid'\n"
5064 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5065 die "can
't reset PCI device '$pciid'\n"
5066 if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5071 PVE::Storage::activate_volumes($storecfg, $vollist);
5074 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5075 outfunc => sub {}, errfunc => sub {});
5077 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5078 # timeout should be more than enough here...
5079 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5081 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5082 : $defaults->{cpuunits};
5084 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5086 timeout => $statefile ? undef : $start_timeout,
5091 # when migrating, prefix QEMU output so other side can pick up any
5092 # errors that might occur and show the user
5093 if ($migratedfrom) {
5094 $run_params{quiet} = 1;
5095 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5099 Slice => 'qemu
.slice
',
5103 if (PVE::CGroup::cgroup_mode() == 2) {
5104 $properties{CPUWeight} = $cpuunits;
5106 $properties{CPUShares} = $cpuunits;
5109 if (my $cpulimit = $conf->{cpulimit}) {
5110 $properties{CPUQuota} = int($cpulimit * 100);
5112 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5114 my $run_qemu = sub {
5115 PVE::Tools::run_fork sub {
5116 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5118 my $exitcode = run_command($cmd, %run_params);
5119 die "QEMU exited with code $exitcode\n" if $exitcode;
5123 if ($conf->{hugepages}) {
5126 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5127 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5129 PVE::QemuServer::Memory::hugepages_mount();
5130 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5132 eval { $run_qemu->() };
5134 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5135 if !$conf->{keephugepages};
5139 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5140 if !$conf->{keephugepages};
5142 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5145 eval { $run_qemu->() };
5149 # deactivate volumes if start fails
5150 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5151 die "start failed: $err";
5154 print "migration listens on $migrate_uri\n" if $migrate_uri;
5155 $res->{migrate_uri} = $migrate_uri;
5157 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5158 eval { mon_cmd($vmid, "cont"); };
5162 #start nbd server for storage migration
5163 if (my $nbd = $migrate_opts->{nbd}) {
5164 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5166 my $migrate_storage_uri;
5167 # nbd_protocol_version > 0 for unix socket support
5168 if ($nbd_protocol_version > 0 && $migration_type eq 'secure
') {
5169 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5170 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix
', data => { path => $socket_path } } );
5171 $migrate_storage_uri = "nbd:unix:$socket_path";
5173 my $nodename = nodename();
5174 my $localip = $get_migration_ip->($nodename);
5175 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5176 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5178 mon_cmd($vmid, "nbd-server-start", addr => {
5181 host => "${localip}",
5182 port => "${storage_migrate_port}",
5185 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5186 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5189 $res->{migrate_storage_uri} = $migrate_storage_uri;
5191 foreach my $opt (sort keys %$nbd) {
5192 my $drivestr = $nbd->{$opt}->{drivestr};
5193 my $volid = $nbd->{$opt}->{volid};
5194 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5195 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5196 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5197 print "re-using replicated volume: $opt - $volid\n"
5198 if $nbd->{$opt}->{replicated};
5200 $res->{drives}->{$opt} = $nbd->{$opt};
5201 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5205 if ($migratedfrom) {
5207 set_migration_caps($vmid);
5212 print "spice listens on port $spice_port\n";
5213 $res->{spice_port} = $spice_port;
5214 if ($migrate_opts->{spice_ticket}) {
5215 mon_cmd($vmid, "set_password", protocol => 'spice
', password =>
5216 $migrate_opts->{spice_ticket});
5217 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5222 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5223 if !$statefile && $conf->{balloon};
5225 foreach my $opt (keys %$conf) {
5226 next if $opt !~ m/^net\d+$/;
5227 my $nicconf = parse_net($conf->{$opt});
5228 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5232 mon_cmd($vmid, 'qom-set
',
5233 path => "machine/peripheral/balloon0",
5234 property => "guest-stats-polling-interval",
5235 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5238 print "Resumed VM, removing state\n";
5239 if (my $vmstate = $conf->{vmstate}) {
5240 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5241 PVE::Storage::vdisk_free($storecfg, $vmstate);
5243 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5244 PVE
::QemuConfig-
>write_config($vmid, $conf);
5247 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5252 sub vm_commandline
{
5253 my ($storecfg, $vmid, $snapname) = @_;
5255 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5260 my $snapshot = $conf->{snapshots
}->{$snapname};
5261 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5263 # check for machine or CPU overrides in snapshot
5264 $forcemachine = $snapshot->{runningmachine
};
5265 $forcecpu = $snapshot->{runningcpu
};
5267 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5272 my $defaults = load_defaults
();
5274 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults,
5275 $forcemachine, $forcecpu);
5277 return PVE
::Tools
::cmd2string
($cmd);
5281 my ($vmid, $skiplock) = @_;
5283 PVE
::QemuConfig-
>lock_config($vmid, sub {
5285 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5287 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5289 mon_cmd
($vmid, "system_reset");
5293 sub get_vm_volumes
{
5297 foreach_volid
($conf, sub {
5298 my ($volid, $attr) = @_;
5300 return if $volid =~ m
|^/|;
5302 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5305 push @$vollist, $volid;
5311 sub vm_stop_cleanup
{
5312 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5317 my $vollist = get_vm_volumes
($conf);
5318 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5321 foreach my $ext (qw(mon qmp pid vnc qga)) {
5322 unlink "/var/run/qemu-server/${vmid}.$ext";
5325 if ($conf->{ivshmem
}) {
5326 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5327 # just delete it for now, VMs which have this already open do not
5328 # are affected, but new VMs will get a separated one. If this
5329 # becomes an issue we either add some sort of ref-counting or just
5330 # add a "don't delete on stop" flag to the ivshmem format.
5331 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5334 foreach my $key (keys %$conf) {
5335 next if $key !~ m/^hostpci(\d+)$/;
5336 my $hostpciindex = $1;
5337 my $d = parse_hostpci
($conf->{$key});
5338 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5340 foreach my $pci (@{$d->{pciid
}}) {
5341 my $pciid = $pci->{id
};
5342 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5346 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5348 warn $@ if $@; # avoid errors - just warn
5351 # call only in locked context
5353 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5355 my $pid = check_running
($vmid, $nocheck);
5360 $conf = PVE
::QemuConfig-
>load_config($vmid);
5361 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5362 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5363 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5364 $timeout = $opts->{down
} if $opts->{down
};
5366 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5371 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
5372 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5374 mon_cmd
($vmid, "system_powerdown");
5377 mon_cmd
($vmid, "quit");
5383 $timeout = 60 if !defined($timeout);
5386 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5391 if ($count >= $timeout) {
5393 warn "VM still running - terminating now with SIGTERM\n";
5396 die "VM quit/powerdown failed - got timeout\n";
5399 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5403 if (!check_running
($vmid, $nocheck)) {
5404 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5408 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5411 die "VM quit/powerdown failed\n";
5419 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5424 if ($count >= $timeout) {
5425 warn "VM still running - terminating now with SIGKILL\n";
5430 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5433 # Note: use $nocheck to skip tests if VM configuration file exists.
5434 # We need that when migration VMs to other nodes (files already moved)
5435 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5437 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5439 $force = 1 if !defined($force) && !$shutdown;
5442 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5443 kill 15, $pid if $pid;
5444 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5445 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5449 PVE
::QemuConfig-
>lock_config($vmid, sub {
5450 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5455 my ($vmid, $timeout) = @_;
5457 PVE
::QemuConfig-
>lock_config($vmid, sub {
5460 # only reboot if running, as qmeventd starts it again on a stop event
5461 return if !check_running
($vmid);
5463 create_reboot_request
($vmid);
5465 my $storecfg = PVE
::Storage
::config
();
5466 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5470 # avoid that the next normal shutdown will be confused for a reboot
5471 clear_reboot_request
($vmid);
5477 # note: if using the statestorage parameter, the caller has to check privileges
5479 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5486 PVE
::QemuConfig-
>lock_config($vmid, sub {
5488 $conf = PVE
::QemuConfig-
>load_config($vmid);
5490 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5491 PVE
::QemuConfig-
>check_lock($conf)
5492 if !($skiplock || $is_backing_up);
5494 die "cannot suspend to disk during backup\n"
5495 if $is_backing_up && $includestate;
5497 if ($includestate) {
5498 $conf->{lock} = 'suspending';
5499 my $date = strftime
("%Y-%m-%d", localtime(time()));
5500 $storecfg = PVE
::Storage
::config
();
5501 if (!$statestorage) {
5502 $statestorage = find_vmstate_storage
($conf, $storecfg);
5503 # check permissions for the storage
5504 my $rpcenv = PVE
::RPCEnvironment
::get
();
5505 if ($rpcenv->{type
} ne 'cli') {
5506 my $authuser = $rpcenv->get_user();
5507 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5512 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
5513 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5514 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5515 PVE
::QemuConfig-
>write_config($vmid, $conf);
5517 mon_cmd
($vmid, "stop");
5521 if ($includestate) {
5523 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5526 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5528 my $state = mon_cmd
($vmid, "query-savevm");
5529 if (!$state->{status
}) {
5530 die "savevm not active\n";
5531 } elsif ($state->{status
} eq 'active') {
5534 } elsif ($state->{status
} eq 'completed') {
5535 print "State saved, quitting\n";
5537 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5538 die "query-savevm failed with error '$state->{error}'\n"
5540 die "query-savevm returned status '$state->{status}'\n";
5546 PVE
::QemuConfig-
>lock_config($vmid, sub {
5547 $conf = PVE
::QemuConfig-
>load_config($vmid);
5549 # cleanup, but leave suspending lock, to indicate something went wrong
5551 mon_cmd
($vmid, "savevm-end");
5552 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5553 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5554 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5555 PVE
::QemuConfig-
>write_config($vmid, $conf);
5561 die "lock changed unexpectedly\n"
5562 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5564 mon_cmd
($vmid, "quit");
5565 $conf->{lock} = 'suspended';
5566 PVE
::QemuConfig-
>write_config($vmid, $conf);
5572 my ($vmid, $skiplock, $nocheck) = @_;
5574 PVE
::QemuConfig-
>lock_config($vmid, sub {
5575 my $res = mon_cmd
($vmid, 'query-status');
5576 my $resume_cmd = 'cont';
5579 if ($res->{status
}) {
5580 return if $res->{status
} eq 'running'; # job done, go home
5581 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
5582 $reset = 1 if $res->{status
} eq 'shutdown';
5587 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5589 PVE
::QemuConfig-
>check_lock($conf)
5590 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5594 # required if a VM shuts down during a backup and we get a resume
5595 # request before the backup finishes for example
5596 mon_cmd
($vmid, "system_reset");
5598 mon_cmd
($vmid, $resume_cmd);
5603 my ($vmid, $skiplock, $key) = @_;
5605 PVE
::QemuConfig-
>lock_config($vmid, sub {
5607 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5609 # there is no qmp command, so we use the human monitor command
5610 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5611 die $res if $res ne '';
5615 # vzdump restore implementaion
5617 sub tar_archive_read_firstfile
{
5618 my $archive = shift;
5620 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5622 # try to detect archive type first
5623 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5624 die "unable to open file '$archive'\n";
5625 my $firstfile = <$fh>;
5629 die "ERROR: archive contaions no data\n" if !$firstfile;
5635 sub tar_restore_cleanup
{
5636 my ($storecfg, $statfile) = @_;
5638 print STDERR
"starting cleanup\n";
5640 if (my $fd = IO
::File-
>new($statfile, "r")) {
5641 while (defined(my $line = <$fd>)) {
5642 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5645 if ($volid =~ m
|^/|) {
5646 unlink $volid || die 'unlink failed\n';
5648 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5650 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5652 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5654 print STDERR
"unable to parse line in statfile - $line";
5661 sub restore_file_archive
{
5662 my ($archive, $vmid, $user, $opts) = @_;
5664 return restore_vma_archive
($archive, $vmid, $user, $opts)
5667 my $info = PVE
::Storage
::archive_info
($archive);
5668 my $format = $opts->{format
} // $info->{format
};
5669 my $comp = $info->{compression
};
5671 # try to detect archive format
5672 if ($format eq 'tar') {
5673 return restore_tar_archive
($archive, $vmid, $user, $opts);
5675 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5679 # hepler to remove disks that will not be used after restore
5680 my $restore_cleanup_oldconf = sub {
5681 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5683 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
5684 my ($ds, $drive) = @_;
5686 return if drive_is_cdrom
($drive, 1);
5688 my $volid = $drive->{file
};
5689 return if !$volid || $volid =~ m
|^/|;
5691 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
5692 return if !$path || !$owner || ($owner != $vmid);
5694 # Note: only delete disk we want to restore
5695 # other volumes will become unused
5696 if ($virtdev_hash->{$ds}) {
5697 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
5704 # delete vmstate files, after the restore we have no snapshots anymore
5705 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5706 my $snap = $oldconf->{snapshots
}->{$snapname};
5707 if ($snap->{vmstate
}) {
5708 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
5716 # Helper to parse vzdump backup device hints
5718 # $rpcenv: Environment, used to ckeck storage permissions
5719 # $user: User ID, to check storage permissions
5720 # $storecfg: Storage configuration
5721 # $fh: the file handle for reading the configuration
5722 # $devinfo: should contain device sizes for all backu-up'ed devices
5723 # $options: backup options (pool, default storage)
5725 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5726 my $parse_backup_hints = sub {
5727 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5729 my $virtdev_hash = {};
5731 while (defined(my $line = <$fh>)) {
5732 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5733 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5734 die "archive does not contain data for drive '$virtdev'\n"
5735 if !$devinfo->{$devname};
5737 if (defined($options->{storage
})) {
5738 $storeid = $options->{storage
} || 'local';
5739 } elsif (!$storeid) {
5742 $format = 'raw' if !$format;
5743 $devinfo->{$devname}->{devname
} = $devname;
5744 $devinfo->{$devname}->{virtdev
} = $virtdev;
5745 $devinfo->{$devname}->{format
} = $format;
5746 $devinfo->{$devname}->{storeid
} = $storeid;
5748 # check permission on storage
5749 my $pool = $options->{pool
}; # todo: do we need that?
5750 if ($user ne 'root@pam') {
5751 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5754 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5755 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5757 my $drive = parse_drive
($virtdev, $2);
5758 if (drive_is_cloudinit
($drive)) {
5759 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
5760 $storeid = $options->{storage
} if defined ($options->{storage
});
5761 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5762 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
5764 $virtdev_hash->{$virtdev} = {
5766 storeid
=> $storeid,
5767 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
5774 return $virtdev_hash;
5777 # Helper to allocate and activate all volumes required for a restore
5779 # $storecfg: Storage configuration
5780 # $virtdev_hash: as returned by parse_backup_hints()
5782 # Returns: { $virtdev => $volid }
5783 my $restore_allocate_devices = sub {
5784 my ($storecfg, $virtdev_hash, $vmid) = @_;
5787 foreach my $virtdev (sort keys %$virtdev_hash) {
5788 my $d = $virtdev_hash->{$virtdev};
5789 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5790 my $storeid = $d->{storeid
};
5791 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5793 # test if requested format is supported
5794 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5795 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
5796 $d->{format
} = $defFormat if !$supported;
5799 if ($d->{is_cloudinit
}) {
5800 $name = "vm-$vmid-cloudinit";
5801 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5802 if ($scfg->{path
}) {
5803 $name .= ".$d->{format}";
5807 my $volid = PVE
::Storage
::vdisk_alloc
(
5808 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
5810 print STDERR
"new volume ID is '$volid'\n";
5811 $d->{volid
} = $volid;
5813 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
5815 $map->{$virtdev} = $volid;
5821 my $restore_update_config_line = sub {
5822 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5824 return if $line =~ m/^\#qmdump\#/;
5825 return if $line =~ m/^\#vzdump\#/;
5826 return if $line =~ m/^lock:/;
5827 return if $line =~ m/^unused\d+:/;
5828 return if $line =~ m/^parent:/;
5830 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5831 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5832 # try to convert old 1.X settings
5833 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5834 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5835 my ($model, $macaddr) = split(/\=/, $devconfig);
5836 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5839 bridge
=> "vmbr$ind",
5840 macaddr
=> $macaddr,
5842 my $netstr = print_net
($net);
5844 print $outfd "net$cookie->{netcount}: $netstr\n";
5845 $cookie->{netcount
}++;
5847 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5848 my ($id, $netstr) = ($1, $2);
5849 my $net = parse_net
($netstr);
5850 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5851 $netstr = print_net
($net);
5852 print $outfd "$id: $netstr\n";
5853 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5856 my $di = parse_drive
($virtdev, $value);
5857 if (defined($di->{backup
}) && !$di->{backup
}) {
5858 print $outfd "#$line";
5859 } elsif ($map->{$virtdev}) {
5860 delete $di->{format
}; # format can change on restore
5861 $di->{file
} = $map->{$virtdev};
5862 $value = print_drive
($di);
5863 print $outfd "$virtdev: $value\n";
5867 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5869 if ($vmgenid ne '0') {
5870 # always generate a new vmgenid if there was a valid one setup
5871 $vmgenid = generate_uuid
();
5873 print $outfd "vmgenid: $vmgenid\n";
5874 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5875 my ($uuid, $uuid_str);
5876 UUID
::generate
($uuid);
5877 UUID
::unparse
($uuid, $uuid_str);
5878 my $smbios1 = parse_smbios1
($2);
5879 $smbios1->{uuid
} = $uuid_str;
5880 print $outfd $1.print_smbios1
($smbios1)."\n";
5886 my $restore_deactivate_volumes = sub {
5887 my ($storecfg, $devinfo) = @_;
5890 foreach my $devname (keys %$devinfo) {
5891 my $volid = $devinfo->{$devname}->{volid
};
5892 push @$vollist, $volid if $volid;
5895 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5898 my $restore_destroy_volumes = sub {
5899 my ($storecfg, $devinfo) = @_;
5901 foreach my $devname (keys %$devinfo) {
5902 my $volid = $devinfo->{$devname}->{volid
};
5905 if ($volid =~ m
|^/|) {
5906 unlink $volid || die 'unlink failed\n';
5908 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5910 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5912 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5917 my ($cfg, $vmid) = @_;
5919 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5921 my $volid_hash = {};
5922 foreach my $storeid (keys %$info) {
5923 foreach my $item (@{$info->{$storeid}}) {
5924 next if !($item->{volid
} && $item->{size
});
5925 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5926 $volid_hash->{$item->{volid
}} = $item;
5933 sub update_disk_config
{
5934 my ($vmid, $conf, $volid_hash) = @_;
5937 my $prefix = "VM $vmid";
5939 # used and unused disks
5940 my $referenced = {};
5942 # Note: it is allowed to define multiple storages with same path (alias), so
5943 # we need to check both 'volid' and real 'path' (two different volid can point
5944 # to the same path).
5946 my $referencedpath = {};
5949 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5950 my ($opt, $drive) = @_;
5952 my $volid = $drive->{file
};
5954 my $volume = $volid_hash->{$volid};
5956 # mark volid as "in-use" for next step
5957 $referenced->{$volid} = 1;
5958 if ($volume && (my $path = $volume->{path
})) {
5959 $referencedpath->{$path} = 1;
5962 return if drive_is_cdrom
($drive);
5965 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
5966 if (defined($updated)) {
5968 $conf->{$opt} = print_drive
($updated);
5969 print "$prefix ($opt): $msg\n";
5973 # remove 'unusedX' entry if volume is used
5974 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
5975 my ($opt, $drive) = @_;
5977 my $volid = $drive->{file
};
5981 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
5982 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5983 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5985 delete $conf->{$opt};
5988 $referenced->{$volid} = 1;
5989 $referencedpath->{$path} = 1 if $path;
5992 foreach my $volid (sort keys %$volid_hash) {
5993 next if $volid =~ m/vm-$vmid-state-/;
5994 next if $referenced->{$volid};
5995 my $path = $volid_hash->{$volid}->{path
};
5996 next if !$path; # just to be sure
5997 next if $referencedpath->{$path};
5999 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6000 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6001 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6008 my ($vmid, $nolock, $dryrun) = @_;
6010 my $cfg = PVE
::Storage
::config
();
6012 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6013 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6014 foreach my $stor (keys %{$cfg->{ids
}}) {
6015 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
6018 print "rescan volumes...\n";
6019 my $volid_hash = scan_volids
($cfg, $vmid);
6021 my $updatefn = sub {
6024 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6026 PVE
::QemuConfig-
>check_lock($conf);
6029 foreach my $volid (keys %$volid_hash) {
6030 my $info = $volid_hash->{$volid};
6031 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6034 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6036 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6039 if (defined($vmid)) {
6043 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6046 my $vmlist = config_list
();
6047 foreach my $vmid (keys %$vmlist) {
6051 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6057 sub restore_proxmox_backup_archive
{
6058 my ($archive, $vmid, $user, $options) = @_;
6060 my $storecfg = PVE
::Storage
::config
();
6062 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6063 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6065 my $fingerprint = $scfg->{fingerprint
};
6066 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6068 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6070 # This is only used for `pbs-restore`!
6071 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6072 local $ENV{PBS_PASSWORD
} = $password;
6073 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6075 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6076 PVE
::Storage
::parse_volname
($storecfg, $archive);
6078 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6080 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6082 my $tmpdir = "/var/tmp/vzdumptmp$$";
6086 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6087 my $tmpfn = "$conffile.$$.tmp";
6088 # disable interrupts (always do cleanups)
6092 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6094 # Note: $oldconf is undef if VM does not exists
6095 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6096 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6098 my $rpcenv = PVE
::RPCEnvironment
::get
();
6107 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6109 my $cfgfn = "$tmpdir/qemu-server.conf";
6110 my $firewall_config_fn = "$tmpdir/fw.conf";
6111 my $index_fn = "$tmpdir/index.json";
6113 my $cmd = "restore";
6115 my $param = [$pbs_backup_name, "index.json", $index_fn];
6116 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6117 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6118 $index = decode_json
($index);
6120 # print Dumper($index);
6121 foreach my $info (@{$index->{files
}}) {
6122 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6124 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6125 $devinfo->{$devname}->{size
} = $1;
6127 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6132 my $is_qemu_server_backup = scalar(
6133 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6135 if (!$is_qemu_server_backup) {
6136 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6138 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6140 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6141 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6143 if ($has_firewall_config) {
6144 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6145 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6147 my $pve_firewall_dir = '/etc/pve/firewall';
6148 mkdir $pve_firewall_dir; # make sure the dir exists
6149 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6152 my $fh = IO
::File-
>new($cfgfn, "r") ||
6153 die "unable to read qemu-server.conf - $!\n";
6155 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6157 # fixme: rate limit?
6159 # create empty/temp config
6160 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6162 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6165 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6167 foreach my $virtdev (sort keys %$virtdev_hash) {
6168 my $d = $virtdev_hash->{$virtdev};
6169 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6171 my $volid = $d->{volid
};
6173 my $path = PVE
::Storage
::path
($storecfg, $volid);
6175 # This is the ONLY user of the PBS_ env vars set on top of this function!
6176 my $pbs_restore_cmd = [
6177 '/usr/bin/pbs-restore',
6178 '--repository', $repo,
6180 "$d->{devname}.img.fidx",
6185 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6186 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6188 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6189 push @$pbs_restore_cmd, '--skip-zero';
6192 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6193 print "restore proxmox backup image: $dbg_cmdstring\n";
6194 run_command
($pbs_restore_cmd);
6197 $fh->seek(0, 0) || die "seek failed - $!\n";
6199 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6201 my $cookie = { netcount
=> 0 };
6202 while (defined(my $line = <$fh>)) {
6203 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique
});
6211 $restore_deactivate_volumes->($storecfg, $devinfo);
6217 $restore_destroy_volumes->($storecfg, $devinfo);
6221 rename($tmpfn, $conffile) ||
6222 die "unable to commit configuration file '$conffile'\n";
6224 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6226 eval { rescan
($vmid, 1); };
6230 sub restore_vma_archive
{
6231 my ($archive, $vmid, $user, $opts, $comp) = @_;
6233 my $readfrom = $archive;
6235 my $cfg = PVE
::Storage
::config
();
6237 my $bwlimit = $opts->{bwlimit
};
6239 my $dbg_cmdstring = '';
6240 my $add_pipe = sub {
6242 push @$commands, $cmd;
6243 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6244 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6249 if ($archive eq '-') {
6252 # If we use a backup from a PVE defined storage we also consider that
6253 # storage's rate limit:
6254 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6255 if (defined($volid)) {
6256 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6257 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6259 print STDERR
"applying read rate limit: $readlimit\n";
6260 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6261 $add_pipe->($cstream);
6267 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6268 my $cmd = $info->{decompressor
};
6269 push @$cmd, $readfrom;
6273 my $tmpdir = "/var/tmp/vzdumptmp$$";
6276 # disable interrupts (always do cleanups)
6280 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6282 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6283 POSIX
::mkfifo
($mapfifo, 0600);
6285 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6287 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6294 my $rpcenv = PVE
::RPCEnvironment
::get
();
6296 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6297 my $tmpfn = "$conffile.$$.tmp";
6299 # Note: $oldconf is undef if VM does not exist
6300 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6301 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6305 my $print_devmap = sub {
6306 my $cfgfn = "$tmpdir/qemu-server.conf";
6308 # we can read the config - that is already extracted
6309 my $fh = IO
::File-
>new($cfgfn, "r") ||
6310 die "unable to read qemu-server.conf - $!\n";
6312 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6314 my $pve_firewall_dir = '/etc/pve/firewall';
6315 mkdir $pve_firewall_dir; # make sure the dir exists
6316 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6319 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6321 foreach my $key (keys %storage_limits) {
6322 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6324 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6325 $storage_limits{$key} = $limit * 1024;
6328 foreach my $devname (keys %$devinfo) {
6329 die "found no device mapping information for device '$devname'\n"
6330 if !$devinfo->{$devname}->{virtdev
};
6333 # create empty/temp config
6335 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6336 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6340 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6342 # print restore information to $fifofh
6343 foreach my $virtdev (sort keys %$virtdev_hash) {
6344 my $d = $virtdev_hash->{$virtdev};
6345 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6347 my $storeid = $d->{storeid
};
6348 my $volid = $d->{volid
};
6351 if (my $limit = $storage_limits{$storeid}) {
6352 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6355 my $write_zeros = 1;
6356 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6360 my $path = PVE
::Storage
::path
($cfg, $volid);
6362 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6364 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6367 $fh->seek(0, 0) || die "seek failed - $!\n";
6369 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6371 my $cookie = { netcount
=> 0 };
6372 while (defined(my $line = <$fh>)) {
6373 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6386 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6387 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6389 $oldtimeout = alarm($timeout);
6396 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6397 my ($dev_id, $size, $devname) = ($1, $2, $3);
6398 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6399 } elsif ($line =~ m/^CTIME: /) {
6400 # we correctly received the vma config, so we can disable
6401 # the timeout now for disk allocation (set to 10 minutes, so
6402 # that we always timeout if something goes wrong)
6405 print $fifofh "done\n";
6406 my $tmp = $oldtimeout || 0;
6407 $oldtimeout = undef;
6414 print "restore vma archive: $dbg_cmdstring\n";
6415 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6419 alarm($oldtimeout) if $oldtimeout;
6421 $restore_deactivate_volumes->($cfg, $devinfo);
6423 close($fifofh) if $fifofh;
6429 $restore_destroy_volumes->($cfg, $devinfo);
6433 rename($tmpfn, $conffile) ||
6434 die "unable to commit configuration file '$conffile'\n";
6436 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6438 eval { rescan
($vmid, 1); };
6442 sub restore_tar_archive
{
6443 my ($archive, $vmid, $user, $opts) = @_;
6445 if ($archive ne '-') {
6446 my $firstfile = tar_archive_read_firstfile
($archive);
6447 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6448 if $firstfile ne 'qemu-server.conf';
6451 my $storecfg = PVE
::Storage
::config
();
6453 # avoid zombie disks when restoring over an existing VM -> cleanup first
6454 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6455 # skiplock=1 because qmrestore has set the 'create' lock itself already
6456 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6457 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6459 my $tocmd = "/usr/lib/qemu-server/qmextract";
6461 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6462 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6463 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6464 $tocmd .= ' --info' if $opts->{info
};
6466 # tar option "xf" does not autodetect compression when read from STDIN,
6467 # so we pipe to zcat
6468 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6469 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6471 my $tmpdir = "/var/tmp/vzdumptmp$$";
6474 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6475 local $ENV{VZDUMP_VMID
} = $vmid;
6476 local $ENV{VZDUMP_USER
} = $user;
6478 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6479 my $tmpfn = "$conffile.$$.tmp";
6481 # disable interrupts (always do cleanups)
6485 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6493 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6495 if ($archive eq '-') {
6496 print "extracting archive from STDIN\n";
6497 run_command
($cmd, input
=> "<&STDIN");
6499 print "extracting archive '$archive'\n";
6503 return if $opts->{info
};
6507 my $statfile = "$tmpdir/qmrestore.stat";
6508 if (my $fd = IO
::File-
>new($statfile, "r")) {
6509 while (defined (my $line = <$fd>)) {
6510 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6511 $map->{$1} = $2 if $1;
6513 print STDERR
"unable to parse line in statfile - $line\n";
6519 my $confsrc = "$tmpdir/qemu-server.conf";
6521 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6523 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6525 my $cookie = { netcount
=> 0 };
6526 while (defined (my $line = <$srcfd>)) {
6527 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6535 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6541 rename $tmpfn, $conffile ||
6542 die "unable to commit configuration file '$conffile'\n";
6544 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6546 eval { rescan
($vmid, 1); };
6550 sub foreach_storage_used_by_vm
{
6551 my ($conf, $func) = @_;
6555 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6556 my ($ds, $drive) = @_;
6557 return if drive_is_cdrom
($drive);
6559 my $volid = $drive->{file
};
6561 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6562 $sidhash->{$sid} = $sid if $sid;
6565 foreach my $sid (sort keys %$sidhash) {
6570 my $qemu_snap_storage = {
6573 sub do_snapshots_with_qemu
{
6574 my ($storecfg, $volid) = @_;
6576 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6577 my $scfg = $storecfg->{ids
}->{$storage_name};
6578 die "could not find storage '$storage_name'\n" if !defined($scfg);
6580 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6584 if ($volid =~ m/\.(qcow2|qed)$/){
6591 sub qga_check_running
{
6592 my ($vmid, $nowarn) = @_;
6594 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6596 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6602 sub template_create
{
6603 my ($vmid, $conf, $disk) = @_;
6605 my $storecfg = PVE
::Storage
::config
();
6607 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6608 my ($ds, $drive) = @_;
6610 return if drive_is_cdrom
($drive);
6611 return if $disk && $ds ne $disk;
6613 my $volid = $drive->{file
};
6614 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6616 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6617 $drive->{file
} = $voliddst;
6618 $conf->{$ds} = print_drive
($drive);
6619 PVE
::QemuConfig-
>write_config($vmid, $conf);
6623 sub convert_iscsi_path
{
6626 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6631 my $initiator_name = get_initiator_name
();
6633 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6634 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6637 die "cannot convert iscsi path '$path', unkown format\n";
6640 sub qemu_img_convert
{
6641 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6643 my $storecfg = PVE
::Storage
::config
();
6644 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6645 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6647 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6651 my $src_is_iscsi = 0;
6655 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6656 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6657 $src_format = qemu_img_format
($src_scfg, $src_volname);
6658 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6659 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6660 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6661 } elsif (-f
$src_volid) {
6662 $src_path = $src_volid;
6663 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6668 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6670 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6671 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6672 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6673 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6676 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6677 push @$cmd, '-l', "snapshot.name=$snapname"
6678 if $snapname && $src_format && $src_format eq "qcow2";
6679 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6680 push @$cmd, '-T', $cachemode if defined($cachemode);
6682 if ($src_is_iscsi) {
6683 push @$cmd, '--image-opts';
6684 $src_path = convert_iscsi_path
($src_path);
6685 } elsif ($src_format) {
6686 push @$cmd, '-f', $src_format;
6689 if ($dst_is_iscsi) {
6690 push @$cmd, '--target-image-opts';
6691 $dst_path = convert_iscsi_path
($dst_path);
6693 push @$cmd, '-O', $dst_format;
6696 push @$cmd, $src_path;
6698 if (!$dst_is_iscsi && $is_zero_initialized) {
6699 push @$cmd, "zeroinit:$dst_path";
6701 push @$cmd, $dst_path;
6706 if($line =~ m/\((\S+)\/100\
%\)/){
6708 my $transferred = int($size * $percent / 100);
6709 my $remaining = $size - $transferred;
6711 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6716 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6718 die "copy failed: $err" if $err;
6721 sub qemu_img_format
{
6722 my ($scfg, $volname) = @_;
6724 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6731 sub qemu_drive_mirror
{
6732 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6734 $jobs = {} if !$jobs;
6738 $jobs->{"drive-$drive"} = {};
6740 if ($dst_volid =~ /^nbd:/) {
6741 $qemu_target = $dst_volid;
6744 my $storecfg = PVE
::Storage
::config
();
6745 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6747 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6749 $format = qemu_img_format
($dst_scfg, $dst_volname);
6751 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6753 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6756 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6757 $opts->{format
} = $format if $format;
6759 if (defined($src_bitmap)) {
6760 $opts->{sync
} = 'incremental';
6761 $opts->{bitmap
} = $src_bitmap;
6762 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6765 if (defined($bwlimit)) {
6766 $opts->{speed
} = $bwlimit * 1024;
6767 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6769 print "drive mirror is starting for drive-$drive\n";
6772 # if a job already runs for this device we get an error, catch it for cleanup
6773 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
6775 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6777 die "mirroring error: $err\n";
6780 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
6783 # $completion can be either
6784 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6785 # 'cancel': wait until all jobs are ready, block-job-cancel them
6786 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6787 sub qemu_drive_mirror_monitor
{
6788 my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6790 $completion //= 'complete';
6793 my $err_complete = 0;
6796 die "storage migration timed out\n" if $err_complete > 300;
6798 my $stats = mon_cmd
($vmid, "query-block-jobs");
6800 my $running_mirror_jobs = {};
6801 foreach my $stat (@$stats) {
6802 next if $stat->{type
} ne 'mirror';
6803 $running_mirror_jobs->{$stat->{device
}} = $stat;
6806 my $readycounter = 0;
6808 foreach my $job (keys %$jobs) {
6810 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6811 print "$job : finished\n";
6812 delete $jobs->{$job};
6816 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6818 my $busy = $running_mirror_jobs->{$job}->{busy
};
6819 my $ready = $running_mirror_jobs->{$job}->{ready
};
6820 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6821 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6822 my $remaining = $total - $transferred;
6823 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6825 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6828 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6831 last if scalar(keys %$jobs) == 0;
6833 if ($readycounter == scalar(keys %$jobs)) {
6834 print "all mirroring jobs are ready \n";
6835 last if $completion eq 'skip'; #do the complete later
6837 if ($vmiddst && $vmiddst != $vmid) {
6838 my $agent_running = $qga && qga_check_running
($vmid);
6839 if ($agent_running) {
6840 print "freeze filesystem\n";
6841 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6843 print "suspend vm\n";
6844 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6847 # if we clone a disk for a new target vm, we don't switch the disk
6848 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6850 if ($agent_running) {
6851 print "unfreeze filesystem\n";
6852 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6854 print "resume vm\n";
6855 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6861 foreach my $job (keys %$jobs) {
6862 # try to switch the disk if source and destination are on the same guest
6863 print "$job: Completing block job...\n";
6866 if ($completion eq 'complete') {
6867 $op = 'block-job-complete';
6868 } elsif ($completion eq 'cancel') {
6869 $op = 'block-job-cancel';
6871 die "invalid completion value: $completion\n";
6873 eval { mon_cmd
($vmid, $op, device
=> $job) };
6874 if ($@ =~ m/cannot be completed/) {
6875 print "$job: Block job cannot be completed, try again.\n";
6878 print "$job: Completed successfully.\n";
6879 $jobs->{$job}->{complete
} = 1;
6890 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6891 die "mirroring error: $err";
6896 sub qemu_blockjobs_cancel
{
6897 my ($vmid, $jobs) = @_;
6899 foreach my $job (keys %$jobs) {
6900 print "$job: Cancelling block job\n";
6901 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6902 $jobs->{$job}->{cancel
} = 1;
6906 my $stats = mon_cmd
($vmid, "query-block-jobs");
6908 my $running_jobs = {};
6909 foreach my $stat (@$stats) {
6910 $running_jobs->{$stat->{device
}} = $stat;
6913 foreach my $job (keys %$jobs) {
6915 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6916 print "$job: Done.\n";
6917 delete $jobs->{$job};
6921 last if scalar(keys %$jobs) == 0;
6928 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6929 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6934 print "create linked clone of drive $drivename ($drive->{file})\n";
6935 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6936 push @$newvollist, $newvolid;
6939 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6940 $storeid = $storage if $storage;
6942 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6944 print "create full clone of drive $drivename ($drive->{file})\n";
6947 if (drive_is_cloudinit
($drive)) {
6948 $name = "vm-$newvmid-cloudinit";
6949 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6950 if ($scfg->{path
}) {
6951 $name .= ".$dst_format";
6954 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
6955 } elsif ($drivename eq 'efidisk0') {
6956 $size = get_efivars_size
($conf);
6958 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
6960 $newvolid = PVE
::Storage
::vdisk_alloc
(
6961 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
6963 push @$newvollist, $newvolid;
6965 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
6967 if (drive_is_cloudinit
($drive)) {
6971 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6972 if (!$running || $snapname) {
6973 # TODO: handle bwlimits
6974 if ($drivename eq 'efidisk0') {
6975 # the relevant data on the efidisk may be smaller than the source
6976 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6977 # that is given by the OVMF_VARS.fd
6978 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
6979 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
6980 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
6981 "if=$src_path", "of=$dst_path"]);
6983 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
6987 my $kvmver = get_running_qemu_version
($vmid);
6988 if (!min_version
($kvmver, 2, 7)) {
6989 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6990 if $drive->{iothread
};
6993 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
6994 $completion, $qga, $bwlimit);
6999 my ($size) = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
7002 $disk->{format
} = undef;
7003 $disk->{file
} = $newvolid;
7004 $disk->{size
} = $size if defined($size);
7009 sub get_running_qemu_version
{
7011 my $res = mon_cmd
($vmid, "query-version");
7012 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7015 sub qemu_use_old_bios_files
{
7016 my ($machine_type) = @_;
7018 return if !$machine_type;
7020 my $use_old_bios_files = undef;
7022 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7024 $use_old_bios_files = 1;
7026 my $version = extract_version
($machine_type, kvm_user_version
());
7027 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7028 # load new efi bios files on migration. So this hack is required to allow
7029 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7030 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7031 $use_old_bios_files = !min_version
($version, 2, 4);
7034 return ($use_old_bios_files, $machine_type);
7037 sub get_efivars_size
{
7039 my $arch = get_vm_arch
($conf);
7040 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7041 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
7042 return -s
$ovmf_vars;
7045 sub update_efidisk_size
{
7048 return if !defined($conf->{efidisk0
});
7050 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
7051 $disk->{size
} = get_efivars_size
($conf);
7052 $conf->{efidisk0
} = print_drive
($disk);
7057 sub create_efidisk
($$$$$) {
7058 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7060 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7061 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7063 my $vars_size_b = -s
$ovmf_vars;
7064 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7065 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7066 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7068 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7069 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7071 return ($volid, $size/1024);
7074 sub vm_iothreads_list
{
7077 my $res = mon_cmd
($vmid, 'query-iothreads');
7080 foreach my $iothread (@$res) {
7081 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7088 my ($conf, $drive) = @_;
7092 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7094 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7100 my $controller = int($drive->{index} / $maxdev);
7101 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
7105 return ($maxdev, $controller, $controller_prefix);
7108 sub windows_version
{
7111 return 0 if !$ostype;
7115 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7117 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7119 } elsif ($ostype =~ m/^win(\d+)$/) {
7126 sub resolve_dst_disk_format
{
7127 my ($storecfg, $storeid, $src_volname, $format) = @_;
7128 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7131 # if no target format is specified, use the source disk format as hint
7133 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7134 $format = qemu_img_format
($scfg, $src_volname);
7140 # test if requested format is supported - else use default
7141 my $supported = grep { $_ eq $format } @$validFormats;
7142 $format = $defFormat if !$supported;
7146 # NOTE: if this logic changes, please update docs & possibly gui logic
7147 sub find_vmstate_storage
{
7148 my ($conf, $storecfg) = @_;
7150 # first, return storage from conf if set
7151 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7153 my ($target, $shared, $local);
7155 foreach_storage_used_by_vm
($conf, sub {
7157 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7158 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7159 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7162 # second, use shared storage where VM has at least one disk
7163 # third, use local storage where VM has at least one disk
7164 # fall back to local storage
7165 $target = $shared // $local // 'local';
7171 my ($uuid, $uuid_str);
7172 UUID
::generate
($uuid);
7173 UUID
::unparse
($uuid, $uuid_str);
7177 sub generate_smbios1_uuid
{
7178 return "uuid=".generate_uuid
();
7184 mon_cmd
($vmid, 'nbd-server-stop');
7187 sub create_reboot_request
{
7189 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7190 or die "failed to create reboot trigger file: $!\n";
7194 sub clear_reboot_request
{
7196 my $path = "/run/qemu-server/$vmid.reboot";
7199 $res = unlink($path);
7200 die "could not remove reboot request for $vmid: $!"
7201 if !$res && $! != POSIX
::ENOENT
;
7206 sub bootorder_from_legacy
{
7207 my ($conf, $bootcfg) = @_;
7209 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
7210 my $bootindex_hash = {};
7212 foreach my $o (split(//, $boot)) {
7213 $bootindex_hash->{$o} = $i*100;
7219 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7220 my ($ds, $drive) = @_;
7222 if (drive_is_cdrom
($drive, 1)) {
7223 if ($bootindex_hash->{d
}) {
7224 $bootorder->{$ds} = $bootindex_hash->{d
};
7225 $bootindex_hash->{d
} += 1;
7227 } elsif ($bootindex_hash->{c
}) {
7228 $bootorder->{$ds} = $bootindex_hash->{c
}
7229 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
7230 $bootindex_hash->{c
} += 1;
7234 if ($bootindex_hash->{n
}) {
7235 for (my $i = 0; $i < $MAX_NETS; $i++) {
7236 my $netname = "net$i";
7237 next if !$conf->{$netname};
7238 $bootorder->{$netname} = $bootindex_hash->{n
};
7239 $bootindex_hash->{n
} += 1;
7246 # Generate default device list for 'boot: order=' property. Matches legacy
7247 # default boot order, but with explicit device names. This is important, since
7248 # the fallback for when neither 'order' nor the old format is specified relies
7249 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7250 sub get_default_bootdevices
{
7256 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
7257 push @ret, $first if $first;
7260 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
7261 push @ret, $first if $first;
7264 for (my $i = 0; $i < $MAX_NETS; $i++) {
7265 my $netname = "net$i";
7266 next if !$conf->{$netname};
7267 push @ret, $netname;
7274 sub device_bootorder
{
7277 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
7279 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
7282 if (!defined($boot) || $boot->{legacy
}) {
7283 $bootorder = bootorder_from_legacy
($conf, $boot);
7284 } elsif ($boot->{order
}) {
7285 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7286 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
7287 $bootorder->{$dev} = $i++;
7294 # bash completion helper
7296 sub complete_backup_archives
{
7297 my ($cmdname, $pname, $cvalue) = @_;
7299 my $cfg = PVE
::Storage
::config
();
7303 if ($cvalue =~ m/^([^:]+):/) {
7307 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7310 foreach my $id (keys %$data) {
7311 foreach my $item (@{$data->{$id}}) {
7312 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7313 push @$res, $item->{volid
} if defined($item->{volid
});
7320 my $complete_vmid_full = sub {
7323 my $idlist = vmstatus
();
7327 foreach my $id (keys %$idlist) {
7328 my $d = $idlist->{$id};
7329 if (defined($running)) {
7330 next if $d->{template
};
7331 next if $running && $d->{status
} ne 'running';
7332 next if !$running && $d->{status
} eq 'running';
7341 return &$complete_vmid_full();
7344 sub complete_vmid_stopped
{
7345 return &$complete_vmid_full(0);
7348 sub complete_vmid_running
{
7349 return &$complete_vmid_full(1);
7352 sub complete_storage
{
7354 my $cfg = PVE
::Storage
::config
();
7355 my $ids = $cfg->{ids
};
7358 foreach my $sid (keys %$ids) {
7359 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7360 next if !$ids->{$sid}->{content
}->{images
};
7367 sub complete_migration_storage
{
7368 my ($cmd, $param, $current_value, $all_args) = @_;
7370 my $targetnode = @$all_args[1];
7372 my $cfg = PVE
::Storage
::config
();
7373 my $ids = $cfg->{ids
};
7376 foreach my $sid (keys %$ids) {
7377 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7378 next if !$ids->{$sid}->{content
}->{images
};