1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
31 use PVE
::DataCenterConfig
;
32 use PVE
::Exception
qw(raise raise_param_exc);
33 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
35 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
38 use PVE
::RPCEnvironment
;
42 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
46 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
47 use PVE
::QemuServer
::Cloudinit
;
48 use PVE
::QemuServer
::CGroup
;
49 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
50 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
51 use PVE
::QemuServer
::Machine
;
52 use PVE
::QemuServer
::Memory
;
53 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
54 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
55 use PVE
::QemuServer
::USB
qw(parse_usb_device);
59 require PVE
::Network
::SDN
::Zones
;
63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
66 "$EDK2_FW_BASE/OVMF_CODE.fd",
67 "$EDK2_FW_BASE/OVMF_VARS.fd"
70 "$EDK2_FW_BASE/AAVMF_CODE.fd",
71 "$EDK2_FW_BASE/AAVMF_VARS.fd"
75 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
77 # Note about locking: we use flock on the config file protect
78 # against concurent actions.
79 # Aditionaly, we have a 'lock' setting in the config file. This
80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
81 # allowed when such lock is set. But you can ignore this kind of
82 # lock with the --skiplock flag.
84 cfs_register_file
('/qemu-server/',
88 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
89 description
=> "Some command save/restore state from this location.",
95 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
96 description
=> "Specifies the Qemu machine type.",
98 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
105 my ($map, $source) = @_;
107 return $source if !defined($map);
109 return $map->{entries
}->{$source}
110 if $map->{entries
} && defined($map->{entries
}->{$source});
112 return $map->{default} if $map->{default};
114 # identity (fallback)
118 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
119 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
121 format
=> 'storagepair-list',
125 #no warnings 'redefine';
129 $nodename_cache //= PVE
::INotify
::nodename
();
130 return $nodename_cache;
137 enum
=> [qw(i6300esb ib700)],
138 description
=> "Watchdog type to emulate.",
139 default => 'i6300esb',
144 enum
=> [qw(reset shutdown poweroff pause debug none)],
145 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
149 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
153 description
=> "Enable/disable Qemu GuestAgent.",
158 fstrim_cloned_disks
=> {
159 description
=> "Run fstrim after moving a disk or migrating the VM.",
165 description
=> "Select the agent type",
169 enum
=> [qw(virtio isa)],
175 description
=> "Select the VGA type.",
180 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
183 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
195 description
=> "The size of the file in MB.",
199 pattern
=> '[a-zA-Z0-9\-]+',
201 format_description
=> 'string',
202 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
209 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
210 description
=> "Configure an audio device."
214 enum
=> ['spice', 'none'],
217 description
=> "Driver backend for the audio device."
221 my $spice_enhancements_fmt = {
226 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
230 enum
=> ['off', 'all', 'filter'],
233 description
=> "Enable video streaming. Uses compression for detected video streams."
240 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
242 description
=> "The file on the host to gather entropy from. In most"
243 . " cases /dev/urandom should be preferred over /dev/random"
244 . " to avoid entropy-starvation issues on the host. Using"
245 . " urandom does *not* decrease security in any meaningful"
246 . " way, as it's still seeded from real entropy, and the"
247 . " bytes provided will most likely be mixed with real"
248 . " entropy on the guest as well. /dev/hwrng can be used"
249 . " to pass through a hardware RNG from the host.",
253 description
=> "Maximum bytes of entropy injected into the guest every"
254 . " 'period' milliseconds. Prefer a lower value when using"
255 . " /dev/random as source. Use 0 to disable limiting"
256 . " (potentially dangerous!).",
259 # default is 1 KiB/s, provides enough entropy to the guest to avoid
260 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
261 # of overwhelming the host, provided we're reading from /dev/urandom
266 description
=> "Every 'period' milliseconds the entropy-injection quota"
267 . " is reset, allowing the guest to retrieve another"
268 . " 'max_bytes' of entropy.",
278 description
=> "Specifies whether a VM will be started during system bootup.",
284 description
=> "Automatic restart after crash (currently ignored).",
289 type
=> 'string', format
=> 'pve-hotplug-features',
290 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
291 default => 'network,disk,usb',
296 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
302 description
=> "Lock/unlock the VM.",
303 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
308 description
=> "Limit of CPU usage.",
309 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
317 description
=> "CPU weight for a VM.",
318 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
326 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
333 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
339 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
347 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
348 "It should not be necessary to set it.",
349 enum
=> PVE
::Tools
::kvmkeymaplist
(),
354 type
=> 'string', format
=> 'dns-name',
355 description
=> "Set a name for the VM. Only used on the configuration web interface.",
360 description
=> "SCSI controller model",
361 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
367 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
372 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
373 description
=> "Specify guest operating system.",
374 verbose_description
=> <<EODESC,
375 Specify guest operating system. This is used to enable special
376 optimization/features for specific operating systems:
379 other;; unspecified OS
380 wxp;; Microsoft Windows XP
381 w2k;; Microsoft Windows 2000
382 w2k3;; Microsoft Windows 2003
383 w2k8;; Microsoft Windows 2008
384 wvista;; Microsoft Windows Vista
385 win7;; Microsoft Windows 7
386 win8;; Microsoft Windows 8/2012/2012r2
387 win10;; Microsoft Windows 10/2016/2019
388 l24;; Linux 2.4 Kernel
389 l26;; Linux 2.6 - 5.X Kernel
390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
395 type
=> 'string', format
=> 'pve-qm-boot',
396 description
=> "Specify guest boot order. Use with 'order=', usage with"
397 . " no key or 'legacy=' is deprecated.",
401 type
=> 'string', format
=> 'pve-qm-bootdisk',
402 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
403 pattern
=> '(ide|sata|scsi|virtio)\d+',
408 description
=> "The number of CPUs. Please use option -sockets instead.",
415 description
=> "The number of CPU sockets.",
422 description
=> "The number of cores per socket.",
429 description
=> "Enable/disable NUMA.",
435 description
=> "Enable/disable hugepages memory.",
436 enum
=> [qw(any 2 1024)],
442 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
443 ." after VM shutdown and can be used for subsequent starts.",
448 description
=> "Number of hotplugged vcpus.",
455 description
=> "Enable/disable ACPI.",
460 description
=> "Enable/disable Qemu GuestAgent and its properties.",
462 format
=> $agent_fmt,
467 description
=> "Enable/disable KVM hardware virtualization.",
473 description
=> "Enable/disable time drift fix.",
479 description
=> "Set the real time clock to local time. This is enabled by default if ostype"
480 ." indicates a Microsoft OS.",
485 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
489 type
=> 'string', format
=> $vga_fmt,
490 description
=> "Configure the VGA hardware.",
491 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
492 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
493 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
494 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
495 ." display server. For win* OS you can select how many independent displays you want,"
496 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
497 ." using a serial device as terminal.",
501 type
=> 'string', format
=> 'pve-qm-watchdog',
502 description
=> "Create a virtual hardware watchdog device.",
503 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
504 ." action), the watchdog must be periodically polled by an agent inside the guest or"
505 ." else the watchdog will reset the guest (or execute the respective action specified)",
510 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
511 description
=> "Set the initial date of the real time clock. Valid format for date are:"
512 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
513 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
516 startup
=> get_standard_option
('pve-startup-order'),
520 description
=> "Enable/disable Template.",
526 description
=> "Arbitrary arguments passed to kvm.",
527 verbose_description
=> <<EODESCR,
528 Arbitrary arguments passed to kvm, for example:
530 args: -no-reboot -no-hpet
532 NOTE: this option is for experts only.
539 description
=> "Enable/disable the USB tablet device.",
540 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
541 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
542 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
543 ." may consider disabling this to save some context switches. This is turned off by"
544 ." default if you use spice (`qm set <vmid> --vga qxl`).",
549 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
553 migrate_downtime
=> {
556 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
562 type
=> 'string', format
=> 'pve-qm-ide',
563 typetext
=> '<volume>',
564 description
=> "This is an alias for option -ide2",
568 description
=> "Emulated CPU type.",
570 format
=> 'pve-vm-cpu-conf',
572 parent
=> get_standard_option
('pve-snapshot-name', {
574 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
578 description
=> "Timestamp for snapshots.",
584 type
=> 'string', format
=> 'pve-volume-id',
585 description
=> "Reference to a volume which stores the VM state. This is used internally"
588 vmstatestorage
=> get_standard_option
('pve-storage-id', {
589 description
=> "Default storage for VM state volumes/files.",
592 runningmachine
=> get_standard_option
('pve-qemu-machine', {
593 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
597 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
598 ." internally for snapshots.",
601 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
602 format_description
=> 'QEMU -cpu parameter'
604 machine
=> get_standard_option
('pve-qemu-machine'),
606 description
=> "Virtual processor architecture. Defaults to the host.",
609 enum
=> [qw(x86_64 aarch64)],
612 description
=> "Specify SMBIOS type 1 fields.",
613 type
=> 'string', format
=> 'pve-qm-smbios1',
620 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
621 ." remove disk operations.",
627 enum
=> [ qw(seabios ovmf) ],
628 description
=> "Select BIOS implementation.",
629 default => 'seabios',
633 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
634 format_description
=> 'UUID',
635 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
636 ." to disable explicitly.",
637 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
638 ." value identifier to the guest OS. This allows to notify the guest operating system"
639 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
640 ." execution or creation from a template). The guest operating system notices the"
641 ." change, and is then able to react as appropriate by marking its copies of"
642 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
643 ."Note that auto-creation only works when done through API/CLI create or update methods"
644 .", but not when manually editing the config file.",
645 default => "1 (autogenerated)",
650 format
=> 'pve-volume-id',
652 description
=> "Script that will be executed during various steps in the vms lifetime.",
656 format
=> $ivshmem_fmt,
657 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
663 format
=> $audio_fmt,
664 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
667 spice_enhancements
=> {
669 format
=> $spice_enhancements_fmt,
670 description
=> "Configure additional enhancements for SPICE.",
674 type
=> 'string', format
=> 'pve-tag-list',
675 description
=> 'Tags of the VM. This is only meta information.',
681 description
=> "Configure a VirtIO-based Random Number Generator.",
690 description
=> 'Specify a custom file containing all meta data passed to the VM via"
691 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
692 format
=> 'pve-volume-id',
693 format_description
=> 'volume',
698 description
=> 'Specify a custom file containing all network data passed to the VM via'
700 format
=> 'pve-volume-id',
701 format_description
=> 'volume',
706 description
=> 'Specify a custom file containing all user data passed to the VM via'
708 format
=> 'pve-volume-id',
709 format_description
=> 'volume',
712 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
714 my $confdesc_cloudinit = {
718 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
719 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
720 .' and `configdrive2` for windows.',
721 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
726 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
727 ." image's configured default user.",
732 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
733 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
734 .' support hashed passwords.',
739 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
741 format
=> 'pve-qm-cicustom',
746 description
=> "cloud-init: Sets DNS search domains for a container. Create will'
747 .' automatically use the setting from the host if neither searchdomain nor nameserver'
752 type
=> 'string', format
=> 'address-list',
753 description
=> "cloud-init: Sets DNS server IP address for a container. Create will'
754 .' automatically use the setting from the host if neither searchdomain nor nameserver'
760 format
=> 'urlencoded',
761 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
765 # what about other qemu settings ?
767 #machine => 'string',
780 ##soundhw => 'string',
782 while (my ($k, $v) = each %$confdesc) {
783 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
786 my $MAX_USB_DEVICES = 5;
788 my $MAX_SERIAL_PORTS = 4;
789 my $MAX_PARALLEL_PORTS = 3;
795 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
796 description
=> "CPUs accessing this NUMA node.",
797 format_description
=> "id[-id];...",
801 description
=> "Amount of memory this NUMA node provides.",
806 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
807 description
=> "Host NUMA nodes to use.",
808 format_description
=> "id[-id];...",
813 enum
=> [qw(preferred bind interleave)],
814 description
=> "NUMA allocation policy.",
818 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
821 type
=> 'string', format
=> $numa_fmt,
822 description
=> "NUMA topology.",
824 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
826 for (my $i = 0; $i < $MAX_NUMA; $i++) {
827 $confdesc->{"numa$i"} = $numadesc;
830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
831 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
832 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
835 my $net_fmt_bridge_descr = <<__EOD__;
836 Bridge to attach the network device to. The Proxmox VE standard bridge
839 If you do not specify a bridge, we create a kvm user (NATed) network
840 device, which provides DHCP and DNS services. The following addresses
847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
851 macaddr
=> get_standard_option
('mac-addr', {
852 description
=> "MAC address. That address must be unique withing your network. This is"
853 ." automatically generated if not specified.",
857 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
858 ." very low CPU overhead. If your guest does not support this driver, it is usually"
859 ." best to use 'e1000'.",
860 enum
=> $nic_model_list,
863 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
866 description
=> $net_fmt_bridge_descr,
867 format_description
=> 'bridge',
868 pattern
=> '[-_.\w\d]+',
873 minimum
=> 0, maximum
=> 16,
874 description
=> 'Number of packet queues to be used on the device.',
880 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
885 minimum
=> 1, maximum
=> 4094,
886 description
=> 'VLAN tag to apply to packets on this interface.',
891 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
892 description
=> 'VLAN trunks to pass through this interface.',
893 format_description
=> 'vlanid[;vlanid...]',
898 description
=> 'Whether this interface should be protected by the firewall.',
903 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
908 minimum
=> 1, maximum
=> 65520,
909 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
916 type
=> 'string', format
=> $net_fmt,
917 description
=> "Specify network devices.",
920 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
925 format
=> 'pve-ipv4-config',
926 format_description
=> 'IPv4Format/CIDR',
927 description
=> 'IPv4 address in CIDR format.',
934 format_description
=> 'GatewayIPv4',
935 description
=> 'Default gateway for IPv4 traffic.',
941 format
=> 'pve-ipv6-config',
942 format_description
=> 'IPv6Format/CIDR',
943 description
=> 'IPv6 address in CIDR format.',
950 format_description
=> 'GatewayIPv6',
951 description
=> 'Default gateway for IPv6 traffic.',
956 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
959 type
=> 'string', format
=> 'pve-qm-ipconfig',
960 description
=> <<'EODESCR',
961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
966 gateway should be provided.
967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
973 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
975 for (my $i = 0; $i < $MAX_NETS; $i++) {
976 $confdesc->{"net$i"} = $netdesc;
977 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
980 foreach my $key (keys %$confdesc_cloudinit) {
981 $confdesc->{$key} = $confdesc_cloudinit->{$key};
984 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
985 sub verify_volume_id_or_qm_path
{
986 my ($volid, $noerr) = @_;
988 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
992 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
993 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1004 type
=> 'string', format
=> 'pve-qm-usb-device',
1005 format_description
=> 'HOSTUSBDEVICE|spice',
1006 description
=> <<EODESCR,
1007 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1009 'bus-port(.port)*' (decimal numbers) or
1010 'vendor_id:product_id' (hexadeciaml numbers) or
1013 You can use the 'lsusb -t' command to list existing usb devices.
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1016 machines - use with special care.
1018 The value 'spice' can be used to add a usb redirection devices for spice.
1024 description
=> "Specifies whether if given host option is a USB3 device or port.",
1031 type
=> 'string', format
=> $usb_fmt,
1032 description
=> "Configure an USB device (n is 0 to 4).",
1034 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1039 pattern
=> '(/dev/.+|socket)',
1040 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1041 verbose_description
=> <<EODESCR,
1042 Create a serial device inside the VM (n is 0 to 3), and pass through a
1043 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1044 host side (use 'qm terminal' to open a terminal connection).
1046 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1047 use with special care.
1049 CAUTION: Experimental! User reported problems with this option.
1056 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1057 description
=> "Map host parallel devices (n is 0 to 2).",
1058 verbose_description
=> <<EODESCR,
1059 Map host parallel devices (n is 0 to 2).
1061 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1062 machines - use with special care.
1064 CAUTION: Experimental! User reported problems with this option.
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1069 $confdesc->{"parallel$i"} = $paralleldesc;
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1073 $confdesc->{"serial$i"} = $serialdesc;
1076 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1077 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1080 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1081 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1084 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1085 $confdesc->{"usb$i"} = $usbdesc;
1093 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1094 . " Deprecated, use 'order=' instead.",
1095 pattern
=> '[acdn]{1,4}',
1096 format_description
=> "[acdn]{1,4}",
1098 # note: this is also the fallback if boot: is not given at all
1104 format
=> 'pve-qm-bootdev-list',
1105 format_description
=> "device[;device...]",
1106 description
=> <<EODESC,
1107 The guest will attempt to boot from devices in the order they appear here.
1109 Disks, optical drives and passed-through storage USB devices will be directly
1110 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1111 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1113 Note that only devices in this list will be marked as bootable and thus loaded
1114 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1115 (e.g. software-raid), you need to specify all of them here.
1117 Overrides the deprecated 'legacy=[acdn]*' value when given.
1121 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1123 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1124 sub verify_bootdev
{
1125 my ($dev, $noerr) = @_;
1127 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && $dev !~ m/^efidisk/;
1131 return 0 if $dev !~ m/^$base\d+$/;
1132 return 0 if !$confdesc->{$dev};
1136 return $dev if $check->("net");
1137 return $dev if $check->("usb");
1138 return $dev if $check->("hostpci");
1141 die "invalid boot device '$dev'\n";
1144 sub print_bootorder
{
1146 return "" if !@$devs;
1147 my $data = { order
=> join(';', @$devs) };
1148 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1151 my $kvm_api_version = 0;
1154 return $kvm_api_version if $kvm_api_version;
1156 open my $fh, '<', '/dev/kvm' or return;
1158 # 0xae00 => KVM_GET_API_VERSION
1159 $kvm_api_version = ioctl($fh, 0xae00, 0);
1162 return $kvm_api_version;
1165 my $kvm_user_version = {};
1168 sub kvm_user_version
{
1171 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1172 my $st = stat($binary);
1174 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1175 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1176 $cachedmtime == $st->mtime;
1178 $kvm_user_version->{$binary} = 'unknown';
1179 $kvm_mtime->{$binary} = $st->mtime;
1183 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1184 $kvm_user_version->{$binary} = $2;
1188 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1191 return $kvm_user_version->{$binary};
1194 my sub extract_version
{
1195 my ($machine_type, $version) = @_;
1196 $version = kvm_user_version
() if !defined($version);
1197 PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1200 sub kernel_has_vhost_net
{
1201 return -c
'/dev/vhost-net';
1206 return defined($confdesc->{$key});
1210 sub get_cdrom_path
{
1212 return $cdrom_path if $cdrom_path;
1214 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1215 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1216 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1220 my ($storecfg, $vmid, $cdrom) = @_;
1222 if ($cdrom eq 'cdrom') {
1223 return get_cdrom_path
();
1224 } elsif ($cdrom eq 'none') {
1226 } elsif ($cdrom =~ m
|^/|) {
1229 return PVE
::Storage
::path
($storecfg, $cdrom);
1233 # try to convert old style file names to volume IDs
1234 sub filename_to_volume_id
{
1235 my ($vmid, $file, $media) = @_;
1237 if (!($file eq 'none' || $file eq 'cdrom' ||
1238 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1240 return if $file =~ m
|/|;
1242 if ($media && $media eq 'cdrom') {
1243 $file = "local:iso/$file";
1245 $file = "local:$vmid/$file";
1252 sub verify_media_type
{
1253 my ($opt, $vtype, $media) = @_;
1258 if ($media eq 'disk') {
1260 } elsif ($media eq 'cdrom') {
1263 die "internal error";
1266 return if ($vtype eq $etype);
1268 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1271 sub cleanup_drive_path
{
1272 my ($opt, $storecfg, $drive) = @_;
1274 # try to convert filesystem paths to volume IDs
1276 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1277 ($drive->{file
} !~ m
|^/dev/.+|) &&
1278 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1279 ($drive->{file
} !~ m/^\d+$/)) {
1280 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1281 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1283 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1284 verify_media_type
($opt, $vtype, $drive->{media
});
1285 $drive->{file
} = $volid;
1288 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1291 sub parse_hotplug_features
{
1296 return $res if $data eq '0';
1298 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1300 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1301 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1304 die "invalid hotplug feature '$feature'\n";
1310 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1311 sub pve_verify_hotplug_features
{
1312 my ($value, $noerr) = @_;
1314 return $value if parse_hotplug_features
($value);
1318 die "unable to parse hotplug option\n";
1322 my($fh, $noerr) = @_;
1325 my $SG_GET_VERSION_NUM = 0x2282;
1327 my $versionbuf = "\x00" x
8;
1328 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1330 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1333 my $version = unpack("I", $versionbuf);
1334 if ($version < 30000) {
1335 die "scsi generic interface too old\n" if !$noerr;
1339 my $buf = "\x00" x
36;
1340 my $sensebuf = "\x00" x
8;
1341 my $cmd = pack("C x3 C x1", 0x12, 36);
1343 # see /usr/include/scsi/sg.h
1344 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1346 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1347 length($sensebuf), 0, length($buf), $buf,
1348 $cmd, $sensebuf, 6000);
1350 $ret = ioctl($fh, $SG_IO, $packet);
1352 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1356 my @res = unpack($sg_io_hdr_t, $packet);
1357 if ($res[17] || $res[18]) {
1358 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363 (my $byte0, my $byte1, $res->{vendor
},
1364 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1366 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1367 $res->{type
} = $byte0 & 31;
1375 my $fh = IO
::File-
>new("+<$path") || return;
1376 my $res = scsi_inquiry
($fh, 1);
1382 sub print_tabletdevice_full
{
1383 my ($conf, $arch) = @_;
1385 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1387 # we use uhci for old VMs because tablet driver was buggy in older qemu
1389 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1395 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1398 sub print_keyboarddevice_full
{
1399 my ($conf, $arch, $machine) = @_;
1401 return if $arch ne 'aarch64';
1403 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1406 sub print_drivedevice_full
{
1407 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1412 my $drive_id = "$drive->{interface}$drive->{index}";
1413 if ($drive->{interface
} eq 'virtio') {
1414 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1415 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1416 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1417 } elsif ($drive->{interface
} eq 'scsi') {
1419 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1420 my $unit = $drive->{index} % $maxdev;
1421 my $devicetype = 'hd';
1423 if (drive_is_cdrom
($drive)) {
1426 if ($drive->{file
} =~ m
|^/|) {
1427 $path = $drive->{file
};
1428 if (my $info = path_is_scsi
($path)) {
1429 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1430 $devicetype = 'block';
1431 } elsif ($info->{type
} == 1) { # tape
1432 $devicetype = 'generic';
1436 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1439 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1440 my $version = extract_version
($machine_type, kvm_user_version
());
1441 if ($path =~ m/^iscsi\:\/\
// &&
1442 !min_version
($version, 4, 1)) {
1443 $devicetype = 'generic';
1447 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1448 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1450 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1451 .",lun=$drive->{index}";
1453 $device .= ",drive=drive-$drive_id,id=$drive_id";
1455 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1456 $device .= ",rotation_rate=1";
1458 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1460 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1461 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1462 my $controller = int($drive->{index} / $maxdev);
1463 my $unit = $drive->{index} % $maxdev;
1464 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1466 $device = "ide-$devicetype";
1467 if ($drive->{interface
} eq 'ide') {
1468 $device .= ",bus=ide.$controller,unit=$unit";
1470 $device .= ",bus=ahci$controller.$unit";
1472 $device .= ",drive=drive-$drive_id,id=$drive_id";
1474 if ($devicetype eq 'hd') {
1475 if (my $model = $drive->{model
}) {
1476 $model = URI
::Escape
::uri_unescape
($model);
1477 $device .= ",model=$model";
1479 if ($drive->{ssd
}) {
1480 $device .= ",rotation_rate=1";
1483 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1484 } elsif ($drive->{interface
} eq 'usb') {
1486 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1488 die "unsupported interface type";
1491 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1493 if (my $serial = $drive->{serial
}) {
1494 $serial = URI
::Escape
::uri_unescape
($serial);
1495 $device .= ",serial=$serial";
1502 sub get_initiator_name
{
1505 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1506 while (defined(my $line = <$fh>)) {
1507 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1516 sub print_drive_commandline_full
{
1517 my ($storecfg, $vmid, $drive) = @_;
1520 my $volid = $drive->{file
};
1523 if (drive_is_cdrom
($drive)) {
1524 $path = get_iso_path
($storecfg, $vmid, $volid);
1526 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1528 $path = PVE
::Storage
::path
($storecfg, $volid);
1529 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1530 $format = qemu_img_format
($scfg, $volname);
1538 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1539 foreach my $o (@qemu_drive_options) {
1540 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1543 # snapshot only accepts on|off
1544 if (defined($drive->{snapshot
})) {
1545 my $v = $drive->{snapshot
} ?
'on' : 'off';
1546 $opts .= ",snapshot=$v";
1549 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1550 my ($dir, $qmpname) = @$type;
1551 if (my $v = $drive->{"mbps$dir"}) {
1552 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1554 if (my $v = $drive->{"mbps${dir}_max"}) {
1555 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1557 if (my $v = $drive->{"bps${dir}_max_length"}) {
1558 $opts .= ",throttling.bps$qmpname-max-length=$v";
1560 if (my $v = $drive->{"iops${dir}"}) {
1561 $opts .= ",throttling.iops$qmpname=$v";
1563 if (my $v = $drive->{"iops${dir}_max"}) {
1564 $opts .= ",throttling.iops$qmpname-max=$v";
1566 if (my $v = $drive->{"iops${dir}_max_length"}) {
1567 $opts .= ",throttling.iops$qmpname-max-length=$v";
1571 $opts .= ",format=$format" if $format && !$drive->{format
};
1573 my $cache_direct = 0;
1575 if (my $cache = $drive->{cache
}) {
1576 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1577 } elsif (!drive_is_cdrom
($drive)) {
1578 $opts .= ",cache=none";
1582 # aio native works only with O_DIRECT
1583 if (!$drive->{aio
}) {
1585 $opts .= ",aio=native";
1587 $opts .= ",aio=threads";
1591 if (!drive_is_cdrom
($drive)) {
1593 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1594 $detectzeroes = 'off';
1595 } elsif ($drive->{discard
}) {
1596 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1598 # This used to be our default with discard not being specified:
1599 $detectzeroes = 'on';
1601 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1604 my $pathinfo = $path ?
"file=$path," : '';
1606 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1609 sub print_netdevice_full
{
1610 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1612 my $device = $net->{model
};
1613 if ($net->{model
} eq 'virtio') {
1614 $device = 'virtio-net-pci';
1617 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1618 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1619 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1620 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1621 # and out of each queue plus one config interrupt and control vector queue
1622 my $vectors = $net->{queues
} * 2 + 2;
1623 $tmpstr .= ",vectors=$vectors,mq=on";
1625 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1627 if (my $mtu = $net->{mtu
}) {
1628 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1629 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1632 } elsif ($mtu < 576) {
1633 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1634 } elsif ($mtu > $bridge_mtu) {
1635 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1637 $tmpstr .= ",host_mtu=$mtu";
1639 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1643 if ($use_old_bios_files) {
1645 if ($device eq 'virtio-net-pci') {
1646 $romfile = 'pxe-virtio.rom';
1647 } elsif ($device eq 'e1000') {
1648 $romfile = 'pxe-e1000.rom';
1649 } elsif ($device eq 'ne2k') {
1650 $romfile = 'pxe-ne2k_pci.rom';
1651 } elsif ($device eq 'pcnet') {
1652 $romfile = 'pxe-pcnet.rom';
1653 } elsif ($device eq 'rtl8139') {
1654 $romfile = 'pxe-rtl8139.rom';
1656 $tmpstr .= ",romfile=$romfile" if $romfile;
1662 sub print_netdev_full
{
1663 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1666 if ($netid =~ m/^net(\d+)$/) {
1670 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1672 my $ifname = "tap${vmid}i$i";
1674 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1675 die "interface name '$ifname' is too long (max 15 character)\n"
1676 if length($ifname) >= 16;
1678 my $vhostparam = '';
1679 if (is_native
($arch)) {
1680 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1683 my $vmname = $conf->{name
} || "vm$vmid";
1686 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1688 if ($net->{bridge
}) {
1689 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1690 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1692 $netdev = "type=user,id=$netid,hostname=$vmname";
1695 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1701 'cirrus' => 'cirrus-vga',
1703 'vmware' => 'vmware-svga',
1704 'virtio' => 'virtio-vga',
1707 sub print_vga_device
{
1708 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1710 my $type = $vga_map->{$vga->{type
}};
1711 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1712 $type = 'virtio-gpu';
1714 my $vgamem_mb = $vga->{memory
};
1716 my $max_outputs = '';
1718 $type = $id ?
'qxl' : 'qxl-vga';
1720 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1721 # set max outputs so linux can have up to 4 qxl displays with one device
1722 if (min_version
($machine_version, 4, 1)) {
1723 $max_outputs = ",max_outputs=4";
1728 die "no devicetype for $vga->{type}\n" if !$type;
1732 if ($vga->{type
} eq 'virtio') {
1733 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1734 $memory = ",max_hostmem=$bytes";
1736 # from https://www.spice-space.org/multiple-monitors.html
1737 $memory = ",vgamem_mb=$vga->{memory}";
1738 my $ram = $vgamem_mb * 4;
1739 my $vram = $vgamem_mb * 2;
1740 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1742 $memory = ",vgamem_mb=$vga->{memory}";
1744 } elsif ($qxlnum && $id) {
1745 $memory = ",ram_size=67108864,vram_size=33554432";
1749 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1750 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1753 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1754 my $vgaid = "vga" . ($id // '');
1757 if ($q35 && $vgaid eq 'vga') {
1758 # the first display uses pcie.0 bus on q35 machines
1759 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1761 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1764 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1767 sub parse_number_sets
{
1770 foreach my $part (split(/;/, $set)) {
1771 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1772 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1773 push @$res, [ $1, $2 ];
1775 die "invalid range: $part\n";
1784 my $res = parse_property_string
($numa_fmt, $data);
1785 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1786 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1790 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1794 my $res = eval { parse_property_string
($net_fmt, $data) };
1799 if (!defined($res->{macaddr
})) {
1800 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1801 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1806 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1807 sub parse_ipconfig
{
1810 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1816 if ($res->{gw
} && !$res->{ip
}) {
1817 warn 'gateway specified without specifying an IP address';
1820 if ($res->{gw6
} && !$res->{ip6
}) {
1821 warn 'IPv6 gateway specified without specifying an IPv6 address';
1824 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1825 warn 'gateway specified together with DHCP';
1828 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1830 warn "IPv6 gateway specified together with $res->{ip6} address";
1834 if (!$res->{ip
} && !$res->{ip6
}) {
1835 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1844 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1847 sub add_random_macs
{
1848 my ($settings) = @_;
1850 foreach my $opt (keys %$settings) {
1851 next if $opt !~ m/^net(\d+)$/;
1852 my $net = parse_net
($settings->{$opt});
1854 $settings->{$opt} = print_net
($net);
1858 sub vm_is_volid_owner
{
1859 my ($storecfg, $vmid, $volid) = @_;
1861 if ($volid !~ m
|^/|) {
1863 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1864 if ($owner && ($owner == $vmid)) {
1872 sub vmconfig_register_unused_drive
{
1873 my ($storecfg, $vmid, $conf, $drive) = @_;
1875 if (drive_is_cloudinit
($drive)) {
1876 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1878 } elsif (!drive_is_cdrom
($drive)) {
1879 my $volid = $drive->{file
};
1880 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1881 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1886 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1890 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1891 format_description
=> 'UUID',
1892 description
=> "Set SMBIOS1 UUID.",
1897 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1898 format_description
=> 'Base64 encoded string',
1899 description
=> "Set SMBIOS1 version.",
1904 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1905 format_description
=> 'Base64 encoded string',
1906 description
=> "Set SMBIOS1 serial number.",
1911 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1912 format_description
=> 'Base64 encoded string',
1913 description
=> "Set SMBIOS1 manufacturer.",
1918 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1919 format_description
=> 'Base64 encoded string',
1920 description
=> "Set SMBIOS1 product ID.",
1925 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1926 format_description
=> 'Base64 encoded string',
1927 description
=> "Set SMBIOS1 SKU string.",
1932 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1933 format_description
=> 'Base64 encoded string',
1934 description
=> "Set SMBIOS1 family string.",
1939 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
1947 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
1954 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
1957 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
1959 sub parse_watchdog
{
1964 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
1969 sub parse_guest_agent
{
1972 return {} if !defined($conf->{agent
});
1974 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
1977 # if the agent is disabled ignore the other potentially set properties
1978 return {} if !$res->{enabled
};
1983 my ($conf, $key) = @_;
1984 return undef if !defined($conf->{agent
});
1986 my $agent = parse_guest_agent
($conf);
1987 return $agent->{$key};
1993 return {} if !$value;
1994 my $res = eval { parse_property_string
($vga_fmt, $value) };
2004 my $res = eval { parse_property_string
($rng_fmt, $value) };
2009 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2010 sub verify_usb_device
{
2011 my ($value, $noerr) = @_;
2013 return $value if parse_usb_device
($value);
2017 die "unable to parse usb device\n";
2020 # add JSON properties for create and set function
2021 sub json_config_properties
{
2024 foreach my $opt (keys %$confdesc) {
2025 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2026 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2027 $prop->{$opt} = $confdesc->{$opt};
2033 # return copy of $confdesc_cloudinit to generate documentation
2034 sub cloudinit_config_properties
{
2036 return dclone
($confdesc_cloudinit);
2040 my ($key, $value) = @_;
2042 die "unknown setting '$key'\n" if !$confdesc->{$key};
2044 my $type = $confdesc->{$key}->{type
};
2046 if (!defined($value)) {
2047 die "got undefined value\n";
2050 if ($value =~ m/[\n\r]/) {
2051 die "property contains a line feed\n";
2054 if ($type eq 'boolean') {
2055 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2056 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2057 die "type check ('boolean') failed - got '$value'\n";
2058 } elsif ($type eq 'integer') {
2059 return int($1) if $value =~ m/^(\d+)$/;
2060 die "type check ('integer') failed - got '$value'\n";
2061 } elsif ($type eq 'number') {
2062 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2063 die "type check ('number') failed - got '$value'\n";
2064 } elsif ($type eq 'string') {
2065 if (my $fmt = $confdesc->{$key}->{format
}) {
2066 PVE
::JSONSchema
::check_format
($fmt, $value);
2069 $value =~ s/^\"(.*)\"$/$1/;
2072 die "internal error"
2077 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2079 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2081 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2083 if ($conf->{template
}) {
2084 # check if any base image is still used by a linked clone
2085 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2086 my ($ds, $drive) = @_;
2087 return if drive_is_cdrom
($drive);
2089 my $volid = $drive->{file
};
2090 return if !$volid || $volid =~ m
|^/|;
2092 die "base volume '$volid' is still in use by linked cloned\n"
2093 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2098 # only remove disks owned by this VM (referenced in the config)
2099 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2100 my ($ds, $drive) = @_;
2101 return if drive_is_cdrom
($drive, 1);
2103 my $volid = $drive->{file
};
2104 return if !$volid || $volid =~ m
|^/|;
2106 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2107 return if !$path || !$owner || ($owner != $vmid);
2109 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2110 warn "Could not remove disk '$volid', check manually: $@" if $@;
2113 if ($purge_unreferenced) { # also remove unreferenced disk
2114 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2115 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2116 my ($volid, $sid, $volname, $d) = @_;
2117 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2122 if (defined $replacement_conf) {
2123 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2125 PVE
::QemuConfig-
>destroy_config($vmid);
2129 sub parse_vm_config
{
2130 my ($filename, $raw) = @_;
2132 return if !defined($raw);
2135 digest
=> Digest
::SHA
::sha1_hex
($raw),
2140 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2141 || die "got strange filename '$filename'";
2149 my @lines = split(/\n/, $raw);
2150 foreach my $line (@lines) {
2151 next if $line =~ m/^\s*$/;
2153 if ($line =~ m/^\[PENDING\]\s*$/i) {
2154 $section = 'pending';
2155 if (defined($descr)) {
2157 $conf->{description
} = $descr;
2160 $conf = $res->{$section} = {};
2163 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2165 if (defined($descr)) {
2167 $conf->{description
} = $descr;
2170 $conf = $res->{snapshots
}->{$section} = {};
2174 if ($line =~ m/^\#(.*)\s*$/) {
2175 $descr = '' if !defined($descr);
2176 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2180 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2181 $descr = '' if !defined($descr);
2182 $descr .= PVE
::Tools
::decode_text
($2);
2183 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2184 $conf->{snapstate
} = $1;
2185 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2188 $conf->{$key} = $value;
2189 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2191 if ($section eq 'pending') {
2192 $conf->{delete} = $value; # we parse this later
2194 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2196 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2199 eval { $value = check_type
($key, $value); };
2201 warn "vm $vmid - unable to parse value of '$key' - $@";
2203 $key = 'ide2' if $key eq 'cdrom';
2204 my $fmt = $confdesc->{$key}->{format
};
2205 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2206 my $v = parse_drive
($key, $value);
2207 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2208 $v->{file
} = $volid;
2209 $value = print_drive
($v);
2211 warn "vm $vmid - unable to parse value of '$key'\n";
2216 $conf->{$key} = $value;
2219 warn "vm $vmid - unable to parse config: $line\n";
2223 if (defined($descr)) {
2225 $conf->{description
} = $descr;
2227 delete $res->{snapstate
}; # just to be sure
2232 sub write_vm_config
{
2233 my ($filename, $conf) = @_;
2235 delete $conf->{snapstate
}; # just to be sure
2237 if ($conf->{cdrom
}) {
2238 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2239 $conf->{ide2
} = $conf->{cdrom
};
2240 delete $conf->{cdrom
};
2243 # we do not use 'smp' any longer
2244 if ($conf->{sockets
}) {
2245 delete $conf->{smp
};
2246 } elsif ($conf->{smp
}) {
2247 $conf->{sockets
} = $conf->{smp
};
2248 delete $conf->{cores
};
2249 delete $conf->{smp
};
2252 my $used_volids = {};
2254 my $cleanup_config = sub {
2255 my ($cref, $pending, $snapname) = @_;
2257 foreach my $key (keys %$cref) {
2258 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2259 $key eq 'snapstate' || $key eq 'pending';
2260 my $value = $cref->{$key};
2261 if ($key eq 'delete') {
2262 die "propertry 'delete' is only allowed in [PENDING]\n"
2264 # fixme: check syntax?
2267 eval { $value = check_type
($key, $value); };
2268 die "unable to parse value of '$key' - $@" if $@;
2270 $cref->{$key} = $value;
2272 if (!$snapname && is_valid_drivename
($key)) {
2273 my $drive = parse_drive
($key, $value);
2274 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2279 &$cleanup_config($conf);
2281 &$cleanup_config($conf->{pending
}, 1);
2283 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2284 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2285 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2288 # remove 'unusedX' settings if we re-add a volume
2289 foreach my $key (keys %$conf) {
2290 my $value = $conf->{$key};
2291 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2292 delete $conf->{$key};
2296 my $generate_raw_config = sub {
2297 my ($conf, $pending) = @_;
2301 # add description as comment to top of file
2302 if (defined(my $descr = $conf->{description
})) {
2304 foreach my $cl (split(/\n/, $descr)) {
2305 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2308 $raw .= "#\n" if $pending;
2312 foreach my $key (sort keys %$conf) {
2313 next if $key =~ /^(digest|description|pending|snapshots)$/;
2314 $raw .= "$key: $conf->{$key}\n";
2319 my $raw = &$generate_raw_config($conf);
2321 if (scalar(keys %{$conf->{pending
}})){
2322 $raw .= "\n[PENDING]\n";
2323 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2326 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2327 $raw .= "\n[$snapname]\n";
2328 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2338 # we use static defaults from our JSON schema configuration
2339 foreach my $key (keys %$confdesc) {
2340 if (defined(my $default = $confdesc->{$key}->{default})) {
2341 $res->{$key} = $default;
2349 my $vmlist = PVE
::Cluster
::get_vmlist
();
2351 return $res if !$vmlist || !$vmlist->{ids
};
2352 my $ids = $vmlist->{ids
};
2353 my $nodename = nodename
();
2355 foreach my $vmid (keys %$ids) {
2356 my $d = $ids->{$vmid};
2357 next if !$d->{node
} || $d->{node
} ne $nodename;
2358 next if !$d->{type
} || $d->{type
} ne 'qemu';
2359 $res->{$vmid}->{exists} = 1;
2364 # test if VM uses local resources (to prevent migration)
2365 sub check_local_resources
{
2366 my ($conf, $noerr) = @_;
2370 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2371 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2373 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2375 foreach my $k (keys %$conf) {
2376 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2377 # sockets are safe: they will recreated be on the target side post-migrate
2378 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2379 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2382 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2387 # check if used storages are available on all nodes (use by migrate)
2388 sub check_storage_availability
{
2389 my ($storecfg, $conf, $node) = @_;
2391 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2392 my ($ds, $drive) = @_;
2394 my $volid = $drive->{file
};
2397 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2400 # check if storage is available on both nodes
2401 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2402 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2406 # list nodes where all VM images are available (used by has_feature API)
2408 my ($conf, $storecfg) = @_;
2410 my $nodelist = PVE
::Cluster
::get_nodelist
();
2411 my $nodehash = { map { $_ => 1 } @$nodelist };
2412 my $nodename = nodename
();
2414 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2415 my ($ds, $drive) = @_;
2417 my $volid = $drive->{file
};
2420 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2422 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2423 if ($scfg->{disable
}) {
2425 } elsif (my $avail = $scfg->{nodes
}) {
2426 foreach my $node (keys %$nodehash) {
2427 delete $nodehash->{$node} if !$avail->{$node};
2429 } elsif (!$scfg->{shared
}) {
2430 foreach my $node (keys %$nodehash) {
2431 delete $nodehash->{$node} if $node ne $nodename
2440 sub check_local_storage_availability
{
2441 my ($conf, $storecfg) = @_;
2443 my $nodelist = PVE
::Cluster
::get_nodelist
();
2444 my $nodehash = { map { $_ => {} } @$nodelist };
2446 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2447 my ($ds, $drive) = @_;
2449 my $volid = $drive->{file
};
2452 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2454 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2456 if ($scfg->{disable
}) {
2457 foreach my $node (keys %$nodehash) {
2458 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2460 } elsif (my $avail = $scfg->{nodes
}) {
2461 foreach my $node (keys %$nodehash) {
2462 if (!$avail->{$node}) {
2463 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2470 foreach my $node (values %$nodehash) {
2471 if (my $unavail = $node->{unavailable_storages
}) {
2472 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2479 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2481 my ($vmid, $nocheck, $node) = @_;
2483 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2484 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2489 my $vzlist = config_list
();
2491 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2493 while (defined(my $de = $fd->read)) {
2494 next if $de !~ m/^(\d+)\.pid$/;
2496 next if !defined($vzlist->{$vmid});
2497 if (my $pid = check_running
($vmid)) {
2498 $vzlist->{$vmid}->{pid
} = $pid;
2505 our $vmstatus_return_properties = {
2506 vmid
=> get_standard_option
('pve-vmid'),
2508 description
=> "Qemu process status.",
2510 enum
=> ['stopped', 'running'],
2513 description
=> "Maximum memory in bytes.",
2516 renderer
=> 'bytes',
2519 description
=> "Root disk size in bytes.",
2522 renderer
=> 'bytes',
2525 description
=> "VM name.",
2530 description
=> "Qemu QMP agent status.",
2535 description
=> "PID of running qemu process.",
2540 description
=> "Uptime.",
2543 renderer
=> 'duration',
2546 description
=> "Maximum usable CPUs.",
2551 description
=> "The current config lock, if any.",
2556 description
=> "The current configured tags, if any",
2560 'running-machine' => {
2561 description
=> "The currently running machine type (if running).",
2566 description
=> "The currently running QEMU version (if running).",
2572 my $last_proc_pid_stat;
2574 # get VM status information
2575 # This must be fast and should not block ($full == false)
2576 # We only query KVM using QMP if $full == true (this can be slow)
2578 my ($opt_vmid, $full) = @_;
2582 my $storecfg = PVE
::Storage
::config
();
2584 my $list = vzlist
();
2585 my $defaults = load_defaults
();
2587 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2589 my $cpucount = $cpuinfo->{cpus
} || 1;
2591 foreach my $vmid (keys %$list) {
2592 next if $opt_vmid && ($vmid ne $opt_vmid);
2594 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2596 my $d = { vmid
=> $vmid };
2597 $d->{pid
} = $list->{$vmid}->{pid
};
2599 # fixme: better status?
2600 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2602 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2603 if (defined($size)) {
2604 $d->{disk
} = 0; # no info available
2605 $d->{maxdisk
} = $size;
2611 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2612 * ($conf->{cores
} || $defaults->{cores
});
2613 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2614 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2616 $d->{name
} = $conf->{name
} || "VM $vmid";
2617 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2618 : $defaults->{memory
}*(1024*1024);
2620 if ($conf->{balloon
}) {
2621 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2622 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2623 : $defaults->{shares
};
2634 $d->{diskwrite
} = 0;
2636 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
2638 $d->{serial
} = 1 if conf_has_serial
($conf);
2639 $d->{lock} = $conf->{lock} if $conf->{lock};
2640 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2645 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2646 foreach my $dev (keys %$netdev) {
2647 next if $dev !~ m/^tap([1-9]\d*)i/;
2649 my $d = $res->{$vmid};
2652 $d->{netout
} += $netdev->{$dev}->{receive
};
2653 $d->{netin
} += $netdev->{$dev}->{transmit
};
2656 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
2657 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
2662 my $ctime = gettimeofday
;
2664 foreach my $vmid (keys %$list) {
2666 my $d = $res->{$vmid};
2667 my $pid = $d->{pid
};
2670 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2671 next if !$pstat; # not running
2673 my $used = $pstat->{utime} + $pstat->{stime
};
2675 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2677 if ($pstat->{vsize
}) {
2678 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2681 my $old = $last_proc_pid_stat->{$pid};
2683 $last_proc_pid_stat->{$pid} = {
2691 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2693 if ($dtime > 1000) {
2694 my $dutime = $used - $old->{used
};
2696 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2697 $last_proc_pid_stat->{$pid} = {
2703 $d->{cpu
} = $old->{cpu
};
2707 return $res if !$full;
2709 my $qmpclient = PVE
::QMPClient-
>new();
2711 my $ballooncb = sub {
2712 my ($vmid, $resp) = @_;
2714 my $info = $resp->{'return'};
2715 return if !$info->{max_mem
};
2717 my $d = $res->{$vmid};
2719 # use memory assigned to VM
2720 $d->{maxmem
} = $info->{max_mem
};
2721 $d->{balloon
} = $info->{actual
};
2723 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2724 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2725 $d->{freemem
} = $info->{free_mem
};
2728 $d->{ballooninfo
} = $info;
2731 my $blockstatscb = sub {
2732 my ($vmid, $resp) = @_;
2733 my $data = $resp->{'return'} || [];
2734 my $totalrdbytes = 0;
2735 my $totalwrbytes = 0;
2737 for my $blockstat (@$data) {
2738 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2739 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2741 $blockstat->{device
} =~ s/drive-//;
2742 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2744 $res->{$vmid}->{diskread
} = $totalrdbytes;
2745 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2748 my $machinecb = sub {
2749 my ($vmid, $resp) = @_;
2750 my $data = $resp->{'return'} || [];
2752 $res->{$vmid}->{'running-machine'} =
2753 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
2756 my $versioncb = sub {
2757 my ($vmid, $resp) = @_;
2758 my $data = $resp->{'return'} // {};
2759 my $version = 'unknown';
2761 if (my $v = $data->{qemu
}) {
2762 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
2765 $res->{$vmid}->{'running-qemu'} = $version;
2768 my $statuscb = sub {
2769 my ($vmid, $resp) = @_;
2771 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2772 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2773 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2774 # this fails if ballon driver is not loaded, so this must be
2775 # the last commnand (following command are aborted if this fails).
2776 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2778 my $status = 'unknown';
2779 if (!defined($status = $resp->{'return'}->{status
})) {
2780 warn "unable to get VM status\n";
2784 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2787 foreach my $vmid (keys %$list) {
2788 next if $opt_vmid && ($vmid ne $opt_vmid);
2789 next if !$res->{$vmid}->{pid
}; # not running
2790 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2793 $qmpclient->queue_execute(undef, 2);
2795 foreach my $vmid (keys %$list) {
2796 next if $opt_vmid && ($vmid ne $opt_vmid);
2797 next if !$res->{$vmid}->{pid
}; #not running
2799 # we can't use the $qmpclient since it might have already aborted on
2800 # 'query-balloon', but this might also fail for older versions...
2801 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
2802 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2805 foreach my $vmid (keys %$list) {
2806 next if $opt_vmid && ($vmid ne $opt_vmid);
2807 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2813 sub conf_has_serial
{
2816 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2817 if ($conf->{"serial$i"}) {
2825 sub conf_has_audio
{
2826 my ($conf, $id) = @_;
2829 my $audio = $conf->{"audio$id"};
2830 return if !defined($audio);
2832 my $audioproperties = parse_property_string
($audio_fmt, $audio);
2833 my $audiodriver = $audioproperties->{driver
} // 'spice';
2836 dev
=> $audioproperties->{device
},
2837 dev_id
=> "audiodev$id",
2838 backend
=> $audiodriver,
2839 backend_id
=> "$audiodriver-backend${id}",
2844 my ($audio, $audiopciaddr, $machine_version) = @_;
2848 my $id = $audio->{dev_id
};
2850 if (min_version
($machine_version, 4, 2)) {
2851 $audiodev = ",audiodev=$audio->{backend_id}";
2854 if ($audio->{dev
} eq 'AC97') {
2855 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2856 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
2857 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2858 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2859 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2861 die "unkown audio device '$audio->{dev}', implement me!";
2864 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2869 sub vga_conf_has_spice
{
2872 my $vgaconf = parse_vga
($vga);
2873 my $vgatype = $vgaconf->{type
};
2874 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2881 return get_host_arch
() eq $arch;
2886 return $conf->{arch
} // get_host_arch
();
2889 my $default_machines = {
2894 sub get_vm_machine
{
2895 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2897 my $machine = $forcemachine || $conf->{machine
};
2899 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2901 $machine ||= $default_machines->{$arch};
2902 if ($add_pve_version) {
2903 $kvmversion //= kvm_user_version
();
2904 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
2905 $machine .= "+pve$pvever";
2909 if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2910 # for version-pinned machines that do not include a pve-version (e.g.
2911 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2912 $machine .= '+pve0';
2918 sub get_ovmf_files
($) {
2921 my $ovmf = $OVMF->{$arch}
2922 or die "no OVMF images known for architecture '$arch'\n";
2928 aarch64
=> '/usr/bin/qemu-system-aarch64',
2929 x86_64
=> '/usr/bin/qemu-system-x86_64',
2931 sub get_command_for_arch
($) {
2933 return '/usr/bin/kvm' if is_native
($arch);
2935 my $cmd = $Arch2Qemu->{$arch}
2936 or die "don't know how to emulate architecture '$arch'\n";
2940 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2941 # to use in a QEMU command line (-cpu element), first array_intersect the result
2942 # of query_supported_ with query_understood_. This is necessary because:
2944 # a) query_understood_ returns flags the host cannot use and
2945 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2946 # flags, but CPU settings - with most of them being flags. Those settings
2947 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
2949 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2950 # expensive. If you need the value returned from this, you can get it much
2951 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2952 # $accel being 'kvm' or 'tcg'.
2954 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2955 # changes, automatically populating pmxcfs.
2957 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2958 # since kvm and tcg machines support different flags
2960 sub query_supported_cpu_flags
{
2963 $arch //= get_host_arch
();
2964 my $default_machine = $default_machines->{$arch};
2968 # FIXME: Once this is merged, the code below should work for ARM as well:
2969 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2970 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2973 my $kvm_supported = defined(kvm_version
());
2974 my $qemu_cmd = get_command_for_arch
($arch);
2976 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
2978 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2979 my $query_supported_run_qemu = sub {
2985 '-machine', $default_machine,
2987 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2988 '-mon', 'chardev=qmp,mode=control',
2989 '-pidfile', $pidfile,
2994 push @$cmd, '-accel', 'tcg';
2997 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
2998 die "QEMU flag querying VM exited with code " . $rc if $rc;
3001 my $cmd_result = mon_cmd
(
3003 'query-cpu-model-expansion',
3005 model
=> { name
=> 'host' }
3008 my $props = $cmd_result->{model
}->{props
};
3009 foreach my $prop (keys %$props) {
3010 next if $props->{$prop} ne '1';
3011 # QEMU returns some flags multiple times, with '_', '.' or '-'
3012 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3013 # We only keep those with underscores, to match /proc/cpuinfo
3014 $prop =~ s/\.|-/_/g;
3015 $flags->{$prop} = 1;
3020 # force stop with 10 sec timeout and 'nocheck'
3021 # always stop, even if QMP failed
3022 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3026 return [ sort keys %$flags ];
3029 # We need to query QEMU twice, since KVM and TCG have different supported flags
3030 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3031 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3032 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3034 if ($kvm_supported) {
3035 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3036 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3043 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3044 my $understood_cpu_flag_dir = "/usr/share/kvm";
3045 sub query_understood_cpu_flags
{
3046 my $arch = get_host_arch
();
3047 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3049 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3052 my $raw = file_get_contents
($filepath);
3053 $raw =~ s/^\s+|\s+$//g;
3054 my @flags = split(/\s+/, $raw);
3059 sub config_to_command
{
3060 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3063 my $globalFlags = [];
3064 my $machineFlags = [];
3069 my $ostype = $conf->{ostype
};
3070 my $winversion = windows_version
($ostype);
3071 my $kvm = $conf->{kvm
};
3072 my $nodename = nodename
();
3074 my $arch = get_vm_arch
($conf);
3075 my $kvm_binary = get_command_for_arch
($arch);
3076 my $kvmver = kvm_user_version
($kvm_binary);
3078 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3079 $kvmver //= "undefined";
3080 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3083 my $add_pve_version = min_version
($kvmver, 4, 1);
3085 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3086 my $machine_version = extract_version
($machine_type, $kvmver);
3087 $kvm //= 1 if is_native
($arch);
3089 $machine_version =~ m/(\d+)\.(\d+)/;
3090 my ($machine_major, $machine_minor) = ($1, $2);
3092 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3093 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3094 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3095 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3096 ." please upgrade node '$nodename'\n"
3097 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3098 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3099 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3100 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3101 ." node '$nodename'\n";
3104 # if a specific +pve version is required for a feature, use $version_guard
3105 # instead of min_version to allow machines to be run with the minimum
3107 my $required_pve_version = 0;
3108 my $version_guard = sub {
3109 my ($major, $minor, $pve) = @_;
3110 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3111 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3112 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3113 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3117 if ($kvm && !defined kvm_version
()) {
3118 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3119 ." or enable in BIOS.\n";
3122 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3123 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3124 my $use_old_bios_files = undef;
3125 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3127 my $cpuunits = defined($conf->{cpuunits
}) ?
3128 $conf->{cpuunits
} : $defaults->{cpuunits
};
3130 push @$cmd, $kvm_binary;
3132 push @$cmd, '-id', $vmid;
3134 my $vmname = $conf->{name
} || "vm$vmid";
3136 push @$cmd, '-name', $vmname;
3138 push @$cmd, '-no-shutdown';
3142 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3143 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3144 push @$cmd, '-mon', "chardev=qmp,mode=control";
3146 if (min_version
($machine_version, 2, 12)) {
3147 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3148 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3151 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3153 push @$cmd, '-daemonize';
3155 if ($conf->{smbios1
}) {
3156 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3157 if ($smbios_conf->{base64
}) {
3158 # Do not pass base64 flag to qemu
3159 delete $smbios_conf->{base64
};
3160 my $smbios_string = "";
3161 foreach my $key (keys %$smbios_conf) {
3163 if ($key eq "uuid") {
3164 $value = $smbios_conf->{uuid
}
3166 $value = decode_base64
($smbios_conf->{$key});
3168 # qemu accepts any binary data, only commas need escaping by double comma
3170 $smbios_string .= "," . $key . "=" . $value if $value;
3172 push @$cmd, '-smbios', "type=1" . $smbios_string;
3174 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3178 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3179 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3180 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3182 my ($path, $format);
3183 if (my $efidisk = $conf->{efidisk0
}) {
3184 my $d = parse_drive
('efidisk0', $efidisk);
3185 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3186 $format = $d->{format
};
3188 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3189 if (!defined($format)) {
3190 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3191 $format = qemu_img_format
($scfg, $volname);
3195 die "efidisk format must be specified\n"
3196 if !defined($format);
3199 warn "no efidisk configured! Using temporary efivars disk.\n";
3200 $path = "/tmp/$vmid-ovmf.fd";
3201 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3207 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3208 $size_str = ",size=" . (-s
$ovmf_vars);
3211 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3212 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3217 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3218 if (min_version
($machine_version, 4, 0)) {
3219 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3221 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3225 if ($conf->{vmgenid
}) {
3226 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3229 # add usb controllers
3230 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3231 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3232 push @$devices, @usbcontrollers if @usbcontrollers;
3233 my $vga = parse_vga
($conf->{vga
});
3235 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3236 $vga->{type
} = 'qxl' if $qxlnum;
3238 if (!$vga->{type
}) {
3239 if ($arch eq 'aarch64') {
3240 $vga->{type
} = 'virtio';
3241 } elsif (min_version
($machine_version, 2, 9)) {
3242 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3244 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3248 # enable absolute mouse coordinates (needed by vnc)
3250 if (defined($conf->{tablet
})) {
3251 $tablet = $conf->{tablet
};
3253 $tablet = $defaults->{tablet
};
3254 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3255 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3259 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3260 my $kbd = print_keyboarddevice_full
($conf, $arch);
3261 push @$devices, '-device', $kbd if defined($kbd);
3264 my $bootorder = device_bootorder
($conf);
3266 # host pci device passthrough
3267 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3268 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3271 my $usb_dev_features = {};
3272 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3274 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3275 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3276 push @$devices, @usbdevices if @usbdevices;
3279 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3280 if (my $path = $conf->{"serial$i"}) {
3281 if ($path eq 'socket') {
3282 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3283 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3284 # On aarch64, serial0 is the UART device. Qemu only allows
3285 # connecting UART devices via the '-serial' command line, as
3286 # the device has a fixed slot on the hardware...
3287 if ($arch eq 'aarch64' && $i == 0) {
3288 push @$devices, '-serial', "chardev:serial$i";
3290 push @$devices, '-device', "isa-serial,chardev=serial$i";
3293 die "no such serial device\n" if ! -c
$path;
3294 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3295 push @$devices, '-device', "isa-serial,chardev=serial$i";
3301 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3302 if (my $path = $conf->{"parallel$i"}) {
3303 die "no such parallel device\n" if ! -c
$path;
3304 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3305 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3306 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3310 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3311 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3312 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3313 push @$devices, @$audio_devs;
3317 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3318 $sockets = $conf->{sockets
} if $conf->{sockets
};
3320 my $cores = $conf->{cores
} || 1;
3322 my $maxcpus = $sockets * $cores;
3324 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3326 my $allowed_vcpus = $cpuinfo->{cpus
};
3328 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3329 if ($allowed_vcpus < $maxcpus);
3331 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3333 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3334 for (my $i = 2; $i <= $vcpus; $i++) {
3335 my $cpustr = print_cpu_device
($conf,$i);
3336 push @$cmd, '-device', $cpustr;
3341 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3343 push @$cmd, '-nodefaults';
3345 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3347 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3349 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3351 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3352 push @$devices, '-device', print_vga_device
(
3353 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3354 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3355 push @$cmd, '-vnc', "unix:$socket,password";
3357 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3358 push @$cmd, '-nographic';
3362 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3363 my $useLocaltime = $conf->{localtime};
3365 if ($winversion >= 5) { # windows
3366 $useLocaltime = 1 if !defined($conf->{localtime});
3368 # use time drift fix when acpi is enabled
3369 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3370 $tdf = 1 if !defined($conf->{tdf
});
3374 if ($winversion >= 6) {
3375 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3376 push @$cmd, '-no-hpet';
3379 push @$rtcFlags, 'driftfix=slew' if $tdf;
3381 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3382 push @$rtcFlags, "base=$conf->{startdate}";
3383 } elsif ($useLocaltime) {
3384 push @$rtcFlags, 'base=localtime';
3388 push @$cmd, '-cpu', $forcecpu;
3390 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3393 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3395 push @$cmd, '-S' if $conf->{freeze
};
3397 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3399 my $guest_agent = parse_guest_agent
($conf);
3401 if ($guest_agent->{enabled
}) {
3402 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3403 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3405 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3406 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3407 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3408 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3409 } elsif ($guest_agent->{type
} eq 'isa') {
3410 push @$devices, '-device', "isa-serial,chardev=qga0";
3414 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3415 if ($rng && $version_guard->(4, 1, 2)) {
3416 check_rng_source
($rng->{source
});
3418 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3419 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3420 my $limiter_str = "";
3422 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3425 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3426 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3427 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3435 for (my $i = 1; $i < $qxlnum; $i++){
3436 push @$devices, '-device', print_vga_device
(
3437 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3440 # assume other OS works like Linux
3441 my ($ram, $vram) = ("134217728", "67108864");
3442 if ($vga->{memory
}) {
3443 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3444 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3446 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3447 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3451 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3453 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3454 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3455 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3457 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3458 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3459 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3461 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3462 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3464 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3465 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3466 if ($spice_enhancement->{foldersharing
}) {
3467 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3468 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3471 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3472 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3473 if $spice_enhancement->{videostreaming
};
3475 push @$devices, '-spice', "$spice_opts";
3478 # enable balloon by default, unless explicitly disabled
3479 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3480 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3481 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3484 if ($conf->{watchdog
}) {
3485 my $wdopts = parse_watchdog
($conf->{watchdog
});
3486 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3487 my $watchdog = $wdopts->{model
} || 'i6300esb';
3488 push @$devices, '-device', "$watchdog$pciaddr";
3489 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3493 my $scsicontroller = {};
3494 my $ahcicontroller = {};
3495 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3497 # Add iscsi initiator name if available
3498 if (my $initiator = get_initiator_name
()) {
3499 push @$devices, '-iscsi', "initiator-name=$initiator";
3502 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3503 my ($ds, $drive) = @_;
3505 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3506 push @$vollist, $drive->{file
};
3509 # ignore efidisk here, already added in bios/fw handling code above
3510 return if $drive->{interface
} eq 'efidisk';
3512 $use_virtio = 1 if $ds =~ m/^virtio/;
3514 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3516 if ($drive->{interface
} eq 'virtio'){
3517 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3520 if ($drive->{interface
} eq 'scsi') {
3522 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3524 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3525 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3527 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3528 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3531 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3532 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3533 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3534 } elsif ($drive->{iothread
}) {
3535 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3539 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3540 $queues = ",num_queues=$drive->{queues}";
3543 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3544 if !$scsicontroller->{$controller};
3545 $scsicontroller->{$controller}=1;
3548 if ($drive->{interface
} eq 'sata') {
3549 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3550 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3551 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3552 if !$ahcicontroller->{$controller};
3553 $ahcicontroller->{$controller}=1;
3556 my $drive_cmd = print_drive_commandline_full
($storecfg, $vmid, $drive);
3557 $drive_cmd .= ',readonly' if PVE
::QemuConfig-
>is_template($conf);
3559 push @$devices, '-drive',$drive_cmd;
3560 push @$devices, '-device', print_drivedevice_full
(
3561 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3564 for (my $i = 0; $i < $MAX_NETS; $i++) {
3565 my $netname = "net$i";
3567 next if !$conf->{$netname};
3568 my $d = parse_net
($conf->{$netname});
3571 $use_virtio = 1 if $d->{model
} eq 'virtio';
3573 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
3575 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
3576 push @$devices, '-netdev', $netdevfull;
3578 my $netdevicefull = print_netdevice_full
(
3579 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3581 push @$devices, '-device', $netdevicefull;
3584 if ($conf->{ivshmem
}) {
3585 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3589 $bus = print_pcie_addr
("ivshmem");
3591 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3594 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3595 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3597 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3598 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3599 .",size=$ivshmem->{size}M";
3602 # pci.4 is nested in pci.1
3603 $bridges->{1} = 1 if $bridges->{4};
3607 if (min_version
($machine_version, 2, 3)) {
3612 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3616 for my $k (sort {$b cmp $a} keys %$bridges) {
3617 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3620 if ($k == 2 && $legacy_igd) {
3623 $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
3625 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3627 # add after -readconfig pve-q35.cfg
3628 splice @$devices, 2, 0, '-device', $devstr;
3630 unshift @$devices, '-device', $devstr if $k > 0;
3635 push @$machineFlags, 'accel=tcg';
3638 my $machine_type_min = $machine_type;
3639 if ($add_pve_version) {
3640 $machine_type_min =~ s/\+pve\d+$//;
3641 $machine_type_min .= "+pve$required_pve_version";
3643 push @$machineFlags, "type=${machine_type_min}";
3645 push @$cmd, @$devices;
3646 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3647 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3648 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3650 if (my $vmstate = $conf->{vmstate
}) {
3651 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3652 push @$vollist, $vmstate;
3653 push @$cmd, '-loadstate', $statepath;
3654 print "activating and using '$vmstate' as vmstate\n";
3658 if ($conf->{args
}) {
3659 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3663 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3666 sub check_rng_source
{
3669 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3670 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3673 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3674 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
3675 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3676 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3677 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3678 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3686 my $res = mon_cmd
($vmid, 'query-spice');
3688 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3691 sub vm_devices_list
{
3694 my $res = mon_cmd
($vmid, 'query-pci');
3695 my $devices_to_check = [];
3697 foreach my $pcibus (@$res) {
3698 push @$devices_to_check, @{$pcibus->{devices
}},
3701 while (@$devices_to_check) {
3703 for my $d (@$devices_to_check) {
3704 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3705 next if !$d->{'pci_bridge'};
3707 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3708 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3710 $devices_to_check = $to_check;
3713 my $resblock = mon_cmd
($vmid, 'query-block');
3714 foreach my $block (@$resblock) {
3715 if($block->{device
} =~ m/^drive-(\S+)/){
3720 my $resmice = mon_cmd
($vmid, 'query-mice');
3721 foreach my $mice (@$resmice) {
3722 if ($mice->{name
} eq 'QEMU HID Tablet') {
3723 $devices->{tablet
} = 1;
3728 # for usb devices there is no query-usb
3729 # but we can iterate over the entries in
3730 # qom-list path=/machine/peripheral
3731 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3732 foreach my $per (@$resperipheral) {
3733 if ($per->{name
} =~ m/^usb\d+$/) {
3734 $devices->{$per->{name
}} = 1;
3742 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3744 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3746 my $devices_list = vm_devices_list
($vmid);
3747 return 1 if defined($devices_list->{$deviceid});
3749 # add PCI bridge if we need it for the device
3750 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3752 if ($deviceid eq 'tablet') {
3754 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
3756 } elsif ($deviceid eq 'keyboard') {
3758 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
3760 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3762 die "usb hotplug currently not reliable\n";
3763 # since we can't reliably hot unplug all added usb devices and usb
3764 # passthrough breaks live migration we disable usb hotplugging for now
3765 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3767 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3769 qemu_iothread_add
($vmid, $deviceid, $device);
3771 qemu_driveadd
($storecfg, $vmid, $device);
3772 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3774 qemu_deviceadd
($vmid, $devicefull);
3775 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3777 eval { qemu_drivedel
($vmid, $deviceid); };
3782 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3785 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3786 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3787 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3789 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3791 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3792 qemu_iothread_add
($vmid, $deviceid, $device);
3793 $devicefull .= ",iothread=iothread-$deviceid";
3796 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3797 $devicefull .= ",num_queues=$device->{queues}";
3800 qemu_deviceadd
($vmid, $devicefull);
3801 qemu_deviceaddverify
($vmid, $deviceid);
3803 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3805 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3806 qemu_driveadd
($storecfg, $vmid, $device);
3808 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3809 eval { qemu_deviceadd
($vmid, $devicefull); };
3811 eval { qemu_drivedel
($vmid, $deviceid); };
3816 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3818 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
3820 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
3821 my $use_old_bios_files = undef;
3822 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3824 my $netdevicefull = print_netdevice_full
(
3825 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3826 qemu_deviceadd
($vmid, $netdevicefull);
3828 qemu_deviceaddverify
($vmid, $deviceid);
3829 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
3832 eval { qemu_netdevdel
($vmid, $deviceid); };
3837 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3840 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3841 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3843 qemu_deviceadd
($vmid, $devicefull);
3844 qemu_deviceaddverify
($vmid, $deviceid);
3847 die "can't hotplug device '$deviceid'\n";
3853 # fixme: this should raise exceptions on error!
3854 sub vm_deviceunplug
{
3855 my ($vmid, $conf, $deviceid) = @_;
3857 my $devices_list = vm_devices_list
($vmid);
3858 return 1 if !defined($devices_list->{$deviceid});
3860 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
3861 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3863 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3865 qemu_devicedel
($vmid, $deviceid);
3867 } elsif ($deviceid =~ m/^usb\d+$/) {
3869 die "usb hotplug currently not reliable\n";
3870 # when unplugging usb devices this way, there may be remaining usb
3871 # controllers/hubs so we disable it for now
3872 #qemu_devicedel($vmid, $deviceid);
3873 #qemu_devicedelverify($vmid, $deviceid);
3875 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3877 qemu_devicedel
($vmid, $deviceid);
3878 qemu_devicedelverify
($vmid, $deviceid);
3879 qemu_drivedel
($vmid, $deviceid);
3880 qemu_iothread_del
($conf, $vmid, $deviceid);
3882 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3884 qemu_devicedel
($vmid, $deviceid);
3885 qemu_devicedelverify
($vmid, $deviceid);
3886 qemu_iothread_del
($conf, $vmid, $deviceid);
3888 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3890 qemu_devicedel
($vmid, $deviceid);
3891 qemu_drivedel
($vmid, $deviceid);
3892 qemu_deletescsihw
($conf, $vmid, $deviceid);
3894 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3896 qemu_devicedel
($vmid, $deviceid);
3897 qemu_devicedelverify
($vmid, $deviceid);
3898 qemu_netdevdel
($vmid, $deviceid);
3901 die "can't unplug device '$deviceid'\n";
3907 sub qemu_deviceadd
{
3908 my ($vmid, $devicefull) = @_;
3910 $devicefull = "driver=".$devicefull;
3911 my %options = split(/[=,]/, $devicefull);
3913 mon_cmd
($vmid, "device_add" , %options);
3916 sub qemu_devicedel
{
3917 my ($vmid, $deviceid) = @_;
3919 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
3922 sub qemu_iothread_add
{
3923 my($vmid, $deviceid, $device) = @_;
3925 if ($device->{iothread
}) {
3926 my $iothreads = vm_iothreads_list
($vmid);
3927 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3931 sub qemu_iothread_del
{
3932 my($conf, $vmid, $deviceid) = @_;
3934 my $confid = $deviceid;
3935 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3936 $confid = 'scsi' . $1;
3938 my $device = parse_drive
($confid, $conf->{$confid});
3939 if ($device->{iothread
}) {
3940 my $iothreads = vm_iothreads_list
($vmid);
3941 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3945 sub qemu_objectadd
{
3946 my($vmid, $objectid, $qomtype) = @_;
3948 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
3953 sub qemu_objectdel
{
3954 my($vmid, $objectid) = @_;
3956 mon_cmd
($vmid, "object-del", id
=> $objectid);
3962 my ($storecfg, $vmid, $device) = @_;
3964 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device);
3965 $drive =~ s/\\/\\\\/g;
3966 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
3968 # If the command succeeds qemu prints: "OK
"
3969 return 1 if $ret =~ m/OK/s;
3971 die "adding drive failed
: $ret\n";
3975 my($vmid, $deviceid) = @_;
3977 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
3980 return 1 if $ret eq "";
3982 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3983 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3985 die "deleting drive
$deviceid failed
: $ret\n";
3988 sub qemu_deviceaddverify {
3989 my ($vmid, $deviceid) = @_;
3991 for (my $i = 0; $i <= 5; $i++) {
3992 my $devices_list = vm_devices_list($vmid);
3993 return 1 if defined($devices_list->{$deviceid});
3997 die "error on hotplug device
'$deviceid'\n";
4001 sub qemu_devicedelverify {
4002 my ($vmid, $deviceid) = @_;
4004 # need to verify that the device is correctly removed as device_del
4005 # is async and empty return is not reliable
4007 for (my $i = 0; $i <= 5; $i++) {
4008 my $devices_list = vm_devices_list($vmid);
4009 return 1 if !defined($devices_list->{$deviceid});
4013 die "error on hot-unplugging device
'$deviceid'\n";
4016 sub qemu_findorcreatescsihw {
4017 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4019 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4021 my $scsihwid="$controller_prefix$controller";
4022 my $devices_list = vm_devices_list($vmid);
4024 if(!defined($devices_list->{$scsihwid})) {
4025 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4031 sub qemu_deletescsihw {
4032 my ($conf, $vmid, $opt) = @_;
4034 my $device = parse_drive($opt, $conf->{$opt});
4036 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4037 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4041 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4043 my $devices_list = vm_devices_list($vmid);
4044 foreach my $opt (keys %{$devices_list}) {
4045 if (is_valid_drivename($opt)) {
4046 my $drive = parse_drive($opt, $conf->{$opt});
4047 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4053 my $scsihwid="scsihw
$controller";
4055 vm_deviceunplug($vmid, $conf, $scsihwid);
4060 sub qemu_add_pci_bridge {
4061 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4067 print_pci_addr($device, $bridges, $arch, $machine_type);
4069 while (my ($k, $v) = each %$bridges) {
4072 return 1 if !defined($bridgeid) || $bridgeid < 1;
4074 my $bridge = "pci
.$bridgeid";
4075 my $devices_list = vm_devices_list($vmid);
4077 if (!defined($devices_list->{$bridge})) {
4078 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4084 sub qemu_set_link_status {
4085 my ($vmid, $device, $up) = @_;
4087 mon_cmd($vmid, "set_link
", name => $device,
4088 up => $up ? JSON::true : JSON::false);
4091 sub qemu_netdevadd {
4092 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4094 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4095 my %options = split(/[=,]/, $netdev);
4097 if (defined(my $vhost = $options{vhost})) {
4098 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4101 if (defined(my $queues = $options{queues})) {
4102 $options{queues} = $queues + 0;
4105 mon_cmd($vmid, "netdev_add
", %options);
4109 sub qemu_netdevdel {
4110 my ($vmid, $deviceid) = @_;
4112 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4115 sub qemu_usb_hotplug {
4116 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4120 # remove the old one first
4121 vm_deviceunplug($vmid, $conf, $deviceid);
4123 # check if xhci controller is necessary and available
4124 if ($device->{usb3}) {
4126 my $devicelist = vm_devices_list($vmid);
4128 if (!$devicelist->{xhci}) {
4129 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4130 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4133 my $d = parse_usb_device($device->{host});
4134 $d->{usb3} = $device->{usb3};
4137 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4140 sub qemu_cpu_hotplug {
4141 my ($vmid, $conf, $vcpus) = @_;
4143 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4146 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4147 $sockets = $conf->{sockets} if $conf->{sockets};
4148 my $cores = $conf->{cores} || 1;
4149 my $maxcpus = $sockets * $cores;
4151 $vcpus = $maxcpus if !$vcpus;
4153 die "you can
't add more vcpus than maxcpus\n"
4154 if $vcpus > $maxcpus;
4156 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4158 if ($vcpus < $currentvcpus) {
4160 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4162 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4163 qemu_devicedel($vmid, "cpu$i");
4165 my $currentrunningvcpus = undef;
4167 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4168 last if scalar(@{$currentrunningvcpus}) == $i-1;
4169 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4173 #update conf after each succesfull cpu unplug
4174 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4175 PVE::QemuConfig->write_config($vmid, $conf);
4178 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4184 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4185 die "vcpus in running vm does not match its configuration\n"
4186 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4188 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4190 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4191 my $cpustr = print_cpu_device($conf, $i);
4192 qemu_deviceadd($vmid, $cpustr);
4195 my $currentrunningvcpus = undef;
4197 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4198 last if scalar(@{$currentrunningvcpus}) == $i;
4199 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4203 #update conf after each succesfull cpu hotplug
4204 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4205 PVE::QemuConfig->write_config($vmid, $conf);
4209 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4210 mon_cmd($vmid, "cpu-add", id => int($i));
4215 sub qemu_block_set_io_throttle {
4216 my ($vmid, $deviceid,
4217 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4218 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4219 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4220 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4222 return if !check_running($vmid) ;
4224 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4226 bps_rd => int($bps_rd),
4227 bps_wr => int($bps_wr),
4229 iops_rd => int($iops_rd),
4230 iops_wr => int($iops_wr),
4231 bps_max => int($bps_max),
4232 bps_rd_max => int($bps_rd_max),
4233 bps_wr_max => int($bps_wr_max),
4234 iops_max => int($iops_max),
4235 iops_rd_max => int($iops_rd_max),
4236 iops_wr_max => int($iops_wr_max),
4237 bps_max_length => int($bps_max_length),
4238 bps_rd_max_length => int($bps_rd_max_length),
4239 bps_wr_max_length => int($bps_wr_max_length),
4240 iops_max_length => int($iops_max_length),
4241 iops_rd_max_length => int($iops_rd_max_length),
4242 iops_wr_max_length => int($iops_wr_max_length),
4247 sub qemu_block_resize {
4248 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4250 my $running = check_running($vmid);
4252 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4254 return if !$running;
4256 my $padding = (1024 - $size % 1024) % 1024;
4257 $size = $size + $padding;
4259 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4263 sub qemu_volume_snapshot {
4264 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4266 my $running = check_running($vmid);
4268 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4269 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4271 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4275 sub qemu_volume_snapshot_delete {
4276 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4278 my $running = check_running($vmid);
4283 my $conf = PVE::QemuConfig->load_config($vmid);
4284 PVE::QemuConfig->foreach_volume($conf, sub {
4285 my ($ds, $drive) = @_;
4286 $running = 1 if $drive->{file} eq $volid;
4290 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4291 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4293 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4297 sub set_migration_caps {
4300 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4305 "auto-converge" => 1,
4307 "x-rdma-pin-all" => 0,
4310 "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration
'} ? 1 : 0,
4313 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4315 for my $supported_capability (@$supported_capabilities) {
4317 capability => $supported_capability->{capability},
4318 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4322 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4326 my ($conf, $func, @param) = @_;
4330 my $test_volid = sub {
4331 my ($key, $drive, $snapname) = @_;
4333 my $volid = $drive->{file};
4336 $volhash->{$volid}->{cdrom} //= 1;
4337 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4339 my $replicate = $drive->{replicate} // 1;
4340 $volhash->{$volid}->{replicate} //= 0;
4341 $volhash->{$volid}->{replicate} = 1 if $replicate;
4343 $volhash->{$volid}->{shared} //= 0;
4344 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4346 $volhash->{$volid}->{referenced_in_config} //= 0;
4347 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4349 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4350 if defined($snapname);
4352 my $size = $drive->{size};
4353 $volhash->{$volid}->{size} //= $size if $size;
4355 $volhash->{$volid}->{is_vmstate} //= 0;
4356 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4358 $volhash->{$volid}->{is_unused} //= 0;
4359 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4362 my $include_opts = {
4363 extra_keys => ['vmstate
'],
4364 include_unused => 1,
4367 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4368 foreach my $snapname (keys %{$conf->{snapshots}}) {
4369 my $snap = $conf->{snapshots}->{$snapname};
4370 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4373 foreach my $volid (keys %$volhash) {
4374 &$func($volid, $volhash->{$volid}, @param);
4378 my $fast_plug_option = {
4386 'vmstatestorage
' => 1,
4391 # hotplug changes in [PENDING]
4392 # $selection hash can be used to only apply specified options, for
4393 # example: { cores => 1 } (only apply changed 'cores
')
4394 # $errors ref is used to return error messages
4395 sub vmconfig_hotplug_pending {
4396 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4398 my $defaults = load_defaults();
4399 my $arch = get_vm_arch($conf);
4400 my $machine_type = get_vm_machine($conf, undef, $arch);
4402 # commit values which do not have any impact on running VM first
4403 # Note: those option cannot raise errors, we we do not care about
4404 # $selection and always apply them.
4406 my $add_error = sub {
4407 my ($opt, $msg) = @_;
4408 $errors->{$opt} = "hotplug problem - $msg";
4412 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4413 if ($fast_plug_option->{$opt}) {
4414 $conf->{$opt} = $conf->{pending}->{$opt};
4415 delete $conf->{pending}->{$opt};
4421 PVE::QemuConfig->write_config($vmid, $conf);
4424 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4426 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4427 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4428 foreach my $opt (sort keys %$pending_delete_hash) {
4429 next if $selection && !$selection->{$opt};
4430 my $force = $pending_delete_hash->{$opt}->{force};
4432 if ($opt eq 'hotplug
') {
4433 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4434 } elsif ($opt eq 'tablet
') {
4435 die "skip\n" if !$hotplug_features->{usb};
4436 if ($defaults->{tablet}) {
4437 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4438 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4439 if $arch eq 'aarch64
';
4441 vm_deviceunplug($vmid, $conf, 'tablet
');
4442 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4444 } elsif ($opt =~ m/^usb\d+/) {
4446 # since we cannot reliably hot unplug usb devices we are disabling it
4447 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4448 #vm_deviceunplug($vmid, $conf, $opt);
4449 } elsif ($opt eq 'vcpus
') {
4450 die "skip\n" if !$hotplug_features->{cpu};
4451 qemu_cpu_hotplug($vmid, $conf, undef);
4452 } elsif ($opt eq 'balloon
') {
4453 # enable balloon device is not hotpluggable
4454 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4455 # here we reset the ballooning value to memory
4456 my $balloon = $conf->{memory} || $defaults->{memory};
4457 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4458 } elsif ($fast_plug_option->{$opt}) {
4460 } elsif ($opt =~ m/^net(\d+)$/) {
4461 die "skip\n" if !$hotplug_features->{network};
4462 vm_deviceunplug($vmid, $conf, $opt);
4463 } elsif (is_valid_drivename($opt)) {
4464 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4465 vm_deviceunplug($vmid, $conf, $opt);
4466 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4467 } elsif ($opt =~ m/^memory$/) {
4468 die "skip\n" if !$hotplug_features->{memory};
4469 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4470 } elsif ($opt eq 'cpuunits
') {
4471 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4472 } elsif ($opt eq 'cpulimit
') {
4473 $cgroup->change_cpu_quota(-1, 100000);
4479 &$add_error($opt, $err) if $err ne "skip\n";
4481 delete $conf->{$opt};
4482 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4486 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4487 $apply_pending_cloudinit = sub {
4488 return if $apply_pending_cloudinit_done; # once is enough
4489 $apply_pending_cloudinit_done = 1; # once is enough
4491 my ($key, $value) = @_;
4493 my @cloudinit_opts = keys %$confdesc_cloudinit;
4494 foreach my $opt (keys %{$conf->{pending}}) {
4495 next if !grep { $_ eq $opt } @cloudinit_opts;
4496 $conf->{$opt} = delete $conf->{pending}->{$opt};
4499 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4500 foreach my $opt (sort keys %$pending_delete_hash) {
4501 next if !grep { $_ eq $opt } @cloudinit_opts;
4502 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4503 delete $conf->{$opt};
4506 my $new_conf = { %$conf };
4507 $new_conf->{$key} = $value;
4508 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4511 foreach my $opt (keys %{$conf->{pending}}) {
4512 next if $selection && !$selection->{$opt};
4513 my $value = $conf->{pending}->{$opt};
4515 if ($opt eq 'hotplug
') {
4516 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4517 } elsif ($opt eq 'tablet
') {
4518 die "skip\n" if !$hotplug_features->{usb};
4520 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4521 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4522 if $arch eq 'aarch64
';
4523 } elsif ($value == 0) {
4524 vm_deviceunplug($vmid, $conf, 'tablet
');
4525 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4527 } elsif ($opt =~ m/^usb\d+$/) {
4529 # since we cannot reliably hot unplug usb devices we disable it for now
4530 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4531 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4532 #die "skip\n" if !$d;
4533 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4534 } elsif ($opt eq 'vcpus
') {
4535 die "skip\n" if !$hotplug_features->{cpu};
4536 qemu_cpu_hotplug($vmid, $conf, $value);
4537 } elsif ($opt eq 'balloon
') {
4538 # enable/disable balloning device is not hotpluggable
4539 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4540 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4541 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4543 # allow manual ballooning if shares is set to zero
4544 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4545 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4546 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4548 } elsif ($opt =~ m/^net(\d+)$/) {
4549 # some changes can be done without hotplug
4550 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4551 $vmid, $opt, $value, $arch, $machine_type);
4552 } elsif (is_valid_drivename($opt)) {
4553 die "skip\n" if $opt eq 'efidisk0
';
4554 # some changes can be done without hotplug
4555 my $drive = parse_drive($opt, $value);
4556 if (drive_is_cloudinit($drive)) {
4557 &$apply_pending_cloudinit($opt, $value);
4559 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4560 $vmid, $opt, $value, $arch, $machine_type);
4561 } elsif ($opt =~ m/^memory$/) { #dimms
4562 die "skip\n" if !$hotplug_features->{memory};
4563 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4564 } elsif ($opt eq 'cpuunits
') {
4565 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4566 } elsif ($opt eq 'cpulimit
') {
4567 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4568 $cgroup->change_cpu_quota($cpulimit, 100000);
4570 die "skip\n"; # skip non-hot-pluggable options
4574 &$add_error($opt, $err) if $err ne "skip\n";
4576 $conf->{$opt} = $value;
4577 delete $conf->{pending}->{$opt};
4581 PVE::QemuConfig->write_config($vmid, $conf);
4584 sub try_deallocate_drive {
4585 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4587 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4588 my $volid = $drive->{file};
4589 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4590 my $sid = PVE::Storage::parse_volume_id($volid);
4591 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4593 # check if the disk is really unused
4594 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4595 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4596 PVE::Storage::vdisk_free($storecfg, $volid);
4599 # If vm is not owner of this disk remove from config
4607 sub vmconfig_delete_or_detach_drive {
4608 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4610 my $drive = parse_drive($opt, $conf->{$opt});
4612 my $rpcenv = PVE::RPCEnvironment::get();
4613 my $authuser = $rpcenv->get_user();
4616 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4617 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4619 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4625 sub vmconfig_apply_pending {
4626 my ($vmid, $conf, $storecfg, $errors) = @_;
4628 my $add_apply_error = sub {
4629 my ($opt, $msg) = @_;
4630 my $err_msg = "unable to apply pending change $opt : $msg";
4631 $errors->{$opt} = $err_msg;
4637 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4638 foreach my $opt (sort keys %$pending_delete_hash) {
4639 my $force = $pending_delete_hash->{$opt}->{force};
4641 if ($opt =~ m/^unused/) {
4642 die "internal error";
4643 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4644 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4648 $add_apply_error->($opt, $err);
4650 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4651 delete $conf->{$opt};
4655 PVE::QemuConfig->cleanup_pending($conf);
4657 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4658 next if $opt eq 'delete'; # just to be sure
4660 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4661 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4665 $add_apply_error->($opt, $err);
4667 $conf->{$opt} = delete $conf->{pending}->{$opt};
4671 # write all changes at once to avoid unnecessary i/o
4672 PVE::QemuConfig->write_config($vmid, $conf);
4675 sub vmconfig_update_net {
4676 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4678 my $newnet = parse_net($value);
4680 if ($conf->{$opt}) {
4681 my $oldnet = parse_net($conf->{$opt});
4683 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4684 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4685 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4686 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4688 # for non online change, we try to hot-unplug
4689 die "skip\n" if !$hotplug;
4690 vm_deviceunplug($vmid, $conf, $opt);
4693 die "internal error" if $opt !~ m/net(\d+)/;
4694 my $iface = "tap${vmid}i$1";
4696 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4697 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4698 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4699 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4700 PVE::Network::tap_unplug($iface);
4703 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4705 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4707 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4708 # Rate can be applied on its own but any change above needs to
4709 # include the rate in tap_plug since OVS resets everything.
4710 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4713 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4714 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4722 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4728 sub vmconfig_update_disk {
4729 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4731 my $drive = parse_drive($opt, $value);
4733 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4734 my $media = $drive->{media} || 'disk
';
4735 my $oldmedia = $old_drive->{media} || 'disk
';
4736 die "unable to change media type\n" if $media ne $oldmedia;
4738 if (!drive_is_cdrom($old_drive)) {
4740 if ($drive->{file} ne $old_drive->{file}) {
4742 die "skip\n" if !$hotplug;
4744 # unplug and register as unused
4745 vm_deviceunplug($vmid, $conf, $opt);
4746 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4749 # update existing disk
4751 # skip non hotpluggable value
4752 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4753 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4754 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4755 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4756 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4761 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4762 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4763 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4764 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4765 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4766 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4767 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4768 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4769 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4770 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4771 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4772 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4773 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4774 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4775 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4776 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4777 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4778 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4780 qemu_block_set_io_throttle(
4782 ($drive->{mbps} || 0)*1024*1024,
4783 ($drive->{mbps_rd} || 0)*1024*1024,
4784 ($drive->{mbps_wr} || 0)*1024*1024,
4785 $drive->{iops} || 0,
4786 $drive->{iops_rd} || 0,
4787 $drive->{iops_wr} || 0,
4788 ($drive->{mbps_max} || 0)*1024*1024,
4789 ($drive->{mbps_rd_max} || 0)*1024*1024,
4790 ($drive->{mbps_wr_max} || 0)*1024*1024,
4791 $drive->{iops_max} || 0,
4792 $drive->{iops_rd_max} || 0,
4793 $drive->{iops_wr_max} || 0,
4794 $drive->{bps_max_length} || 1,
4795 $drive->{bps_rd_max_length} || 1,
4796 $drive->{bps_wr_max_length} || 1,
4797 $drive->{iops_max_length} || 1,
4798 $drive->{iops_rd_max_length} || 1,
4799 $drive->{iops_wr_max_length} || 1,
4809 if ($drive->{file} eq 'none
') {
4810 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4811 if (drive_is_cloudinit($old_drive)) {
4812 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4815 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4817 # force eject if locked
4818 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4821 mon_cmd($vmid, "blockdev-change-medium",
4822 id => "$opt", filename => "$path");
4830 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4832 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4833 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4836 # called in locked context by incoming migration
4837 sub vm_migrate_get_nbd_disks {
4838 my ($storecfg, $conf, $replicated_volumes) = @_;
4840 my $local_volumes = {};
4841 PVE::QemuConfig->foreach_volume($conf, sub {
4842 my ($ds, $drive) = @_;
4844 return if drive_is_cdrom($drive);
4846 my $volid = $drive->{file};
4850 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4852 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4853 return if $scfg->{shared};
4855 # replicated disks re-use existing state via bitmap
4856 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4857 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4859 return $local_volumes;
4862 # called in locked context by incoming migration
4863 sub vm_migrate_alloc_nbd_disks {
4864 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4869 foreach my $opt (sort keys %$source_volumes) {
4870 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4872 if ($use_existing) {
4873 $nbd->{$opt}->{drivestr} = print_drive($drive);
4874 $nbd->{$opt}->{volid} = $volid;
4875 $nbd->{$opt}->{replicated} = 1;
4879 # If a remote storage is specified and the format of the original
4880 # volume is not available there, fall back to the default format.
4881 # Otherwise use the same format as the original.
4882 if (!$storagemap->{identity}) {
4883 $storeid = map_storage($storagemap, $storeid);
4884 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4885 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4886 my $fileFormat = qemu_img_format($scfg, $volname);
4887 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4889 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4890 $format = qemu_img_format($scfg, $volname);
4893 my $size = $drive->{size} / 1024;
4894 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4895 my $newdrive = $drive;
4896 $newdrive->{format} = $format;
4897 $newdrive->{file} = $newvolid;
4898 my $drivestr = print_drive($newdrive);
4899 $nbd->{$opt}->{drivestr} = $drivestr;
4900 $nbd->{$opt}->{volid} = $newvolid;
4906 # see vm_start_nolock for parameters, additionally:
4908 # storagemap = parsed storage map for allocating NBD disks
4910 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4912 return PVE::QemuConfig->lock_config($vmid, sub {
4913 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4915 die "you can't start a vm
if it
's a template\n"
4916 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4918 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended
');
4919 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup
');
4921 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4923 if ($has_backup_lock && $running) {
4924 # a backup is currently running, attempt to start the guest in the
4925 # existing QEMU instance
4926 return vm_resume($vmid);
4929 PVE::QemuConfig->check_lock($conf)
4930 if !($params->{skiplock} || $has_suspended_lock);
4932 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4934 die "VM $vmid already running\n" if $running;
4936 if (my $storagemap = $migrate_opts->{storagemap}) {
4937 my $replicated = $migrate_opts->{replicated_volumes};
4938 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4939 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4941 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4942 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4946 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4952 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
4953 # skiplock => 0/1, skip checking for config lock
4954 # skiptemplate => 0/1, skip checking whether VM is template
4955 # forcemachine => to force Qemu machine (rollback/migration)
4956 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
4957 # timeout => in seconds
4958 # paused => start VM in paused state (backup)
4959 # resume => resume from hibernation
4961 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4962 # migratedfrom => source node
4963 # spice_ticket => used for spice migration, passed via tunnel/stdin
4964 # network => CIDR of migration network
4965 # type => secure/insecure - tunnel over encrypted connection or plain-text
4966 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
4967 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4968 sub vm_start_nolock {
4969 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4971 my $statefile = $params->{statefile};
4972 my $resume = $params->{resume};
4974 my $migratedfrom = $migrate_opts->{migratedfrom};
4975 my $migration_type = $migrate_opts->{type};
4979 # clean up leftover reboot request files
4980 eval { clear_reboot_request($vmid); };
4983 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4984 vmconfig_apply_pending($vmid, $conf, $storecfg);
4985 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4988 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4990 my $defaults = load_defaults();
4992 # set environment variable useful inside network script
4993 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4995 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
4997 my $forcemachine = $params->{forcemachine};
4998 my $forcecpu = $params->{forcecpu};
5000 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5001 $forcemachine = $conf->{runningmachine};
5002 $forcecpu = $conf->{runningcpu};
5003 print "Resuming suspended VM\n";
5006 my ($cmd, $vollist, $spice_port) =
5007 config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5010 my $get_migration_ip = sub {
5011 my ($nodename) = @_;
5013 return $migration_ip if defined($migration_ip);
5015 my $cidr = $migrate_opts->{network};
5017 if (!defined($cidr)) {
5018 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5019 $cidr = $dc_conf->{migration}->{network};
5022 if (defined($cidr)) {
5023 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5025 die "could not get IP: no address configured on local " .
5026 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5028 die "could not get IP: multiple addresses configured on local " .
5029 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5031 $migration_ip = @$ips[0];
5034 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5035 if !defined($migration_ip);
5037 return $migration_ip;
5042 if ($statefile eq 'tcp
') {
5043 my $localip = "localhost";
5044 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5045 my $nodename = nodename();
5047 if (!defined($migration_type)) {
5048 if (defined($datacenterconf->{migration}->{type})) {
5049 $migration_type = $datacenterconf->{migration}->{type};
5051 $migration_type = 'secure
';
5055 if ($migration_type eq 'insecure
') {
5056 $localip = $get_migration_ip->($nodename);
5057 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5060 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5061 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5062 $migrate_uri = "tcp:${localip}:${migrate_port}";
5063 push @$cmd, '-incoming
', $migrate_uri;
5066 } elsif ($statefile eq 'unix
') {
5067 # should be default for secure migrations as a ssh TCP forward
5068 # tunnel is not deterministic reliable ready and fails regurarly
5069 # to set up in time, so use UNIX socket forwards
5070 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5071 unlink $socket_addr;
5073 $migrate_uri = "unix:$socket_addr";
5075 push @$cmd, '-incoming
', $migrate_uri;
5078 } elsif (-e $statefile) {
5079 push @$cmd, '-loadstate
', $statefile;
5081 my $statepath = PVE::Storage::path($storecfg, $statefile);
5082 push @$vollist, $statefile;
5083 push @$cmd, '-loadstate
', $statepath;
5085 } elsif ($params->{paused}) {
5090 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
5091 my $d = parse_hostpci($conf->{"hostpci$i"});
5093 my $pcidevices = $d->{pciid};
5094 foreach my $pcidevice (@$pcidevices) {
5095 my $pciid = $pcidevice->{id};
5097 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5098 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5099 die "no pci device info for device '$pciid'\n" if !$info;
5102 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5103 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5105 die "can't unbind
/bind PCI group to VFIO
'$pciid'\n"
5106 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5107 die "can
't reset PCI device '$pciid'\n"
5108 if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5113 PVE::Storage::activate_volumes($storecfg, $vollist);
5116 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5117 outfunc => sub {}, errfunc => sub {});
5119 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5120 # timeout should be more than enough here...
5121 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5123 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5124 : $defaults->{cpuunits};
5126 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5128 timeout => $statefile ? undef : $start_timeout,
5133 # when migrating, prefix QEMU output so other side can pick up any
5134 # errors that might occur and show the user
5135 if ($migratedfrom) {
5136 $run_params{quiet} = 1;
5137 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5141 Slice => 'qemu
.slice
',
5145 if (PVE::CGroup::cgroup_mode() == 2) {
5146 $properties{CPUWeight} = $cpuunits;
5148 $properties{CPUShares} = $cpuunits;
5151 if (my $cpulimit = $conf->{cpulimit}) {
5152 $properties{CPUQuota} = int($cpulimit * 100);
5154 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5156 my $run_qemu = sub {
5157 PVE::Tools::run_fork sub {
5158 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5160 my $exitcode = run_command($cmd, %run_params);
5161 die "QEMU exited with code $exitcode\n" if $exitcode;
5165 if ($conf->{hugepages}) {
5168 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5169 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5171 PVE::QemuServer::Memory::hugepages_mount();
5172 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5174 eval { $run_qemu->() };
5176 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5177 if !$conf->{keephugepages};
5181 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5182 if !$conf->{keephugepages};
5184 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5187 eval { $run_qemu->() };
5191 # deactivate volumes if start fails
5192 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5193 die "start failed: $err";
5196 print "migration listens on $migrate_uri\n" if $migrate_uri;
5197 $res->{migrate_uri} = $migrate_uri;
5199 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5200 eval { mon_cmd($vmid, "cont"); };
5204 #start nbd server for storage migration
5205 if (my $nbd = $migrate_opts->{nbd}) {
5206 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5208 my $migrate_storage_uri;
5209 # nbd_protocol_version > 0 for unix socket support
5210 if ($nbd_protocol_version > 0 && $migration_type eq 'secure
') {
5211 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5212 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix
', data => { path => $socket_path } } );
5213 $migrate_storage_uri = "nbd:unix:$socket_path";
5215 my $nodename = nodename();
5216 my $localip = $get_migration_ip->($nodename);
5217 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5218 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5220 mon_cmd($vmid, "nbd-server-start", addr => {
5223 host => "${localip}",
5224 port => "${storage_migrate_port}",
5227 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5228 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5231 $res->{migrate_storage_uri} = $migrate_storage_uri;
5233 foreach my $opt (sort keys %$nbd) {
5234 my $drivestr = $nbd->{$opt}->{drivestr};
5235 my $volid = $nbd->{$opt}->{volid};
5236 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5237 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5238 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5239 print "re-using replicated volume: $opt - $volid\n"
5240 if $nbd->{$opt}->{replicated};
5242 $res->{drives}->{$opt} = $nbd->{$opt};
5243 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5247 if ($migratedfrom) {
5249 set_migration_caps($vmid);
5254 print "spice listens on port $spice_port\n";
5255 $res->{spice_port} = $spice_port;
5256 if ($migrate_opts->{spice_ticket}) {
5257 mon_cmd($vmid, "set_password", protocol => 'spice
', password =>
5258 $migrate_opts->{spice_ticket});
5259 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5264 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5265 if !$statefile && $conf->{balloon};
5267 foreach my $opt (keys %$conf) {
5268 next if $opt !~ m/^net\d+$/;
5269 my $nicconf = parse_net($conf->{$opt});
5270 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5274 mon_cmd($vmid, 'qom-set
',
5275 path => "machine/peripheral/balloon0",
5276 property => "guest-stats-polling-interval",
5277 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5280 print "Resumed VM, removing state\n";
5281 if (my $vmstate = $conf->{vmstate}) {
5282 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5283 PVE::Storage::vdisk_free($storecfg, $vmstate);
5285 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5286 PVE
::QemuConfig-
>write_config($vmid, $conf);
5289 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5294 sub vm_commandline
{
5295 my ($storecfg, $vmid, $snapname) = @_;
5297 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5302 my $snapshot = $conf->{snapshots
}->{$snapname};
5303 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5305 # check for machine or CPU overrides in snapshot
5306 $forcemachine = $snapshot->{runningmachine
};
5307 $forcecpu = $snapshot->{runningcpu
};
5309 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5314 my $defaults = load_defaults
();
5316 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults,
5317 $forcemachine, $forcecpu);
5319 return PVE
::Tools
::cmd2string
($cmd);
5323 my ($vmid, $skiplock) = @_;
5325 PVE
::QemuConfig-
>lock_config($vmid, sub {
5327 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5329 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5331 mon_cmd
($vmid, "system_reset");
5335 sub get_vm_volumes
{
5339 foreach_volid
($conf, sub {
5340 my ($volid, $attr) = @_;
5342 return if $volid =~ m
|^/|;
5344 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5347 push @$vollist, $volid;
5353 sub vm_stop_cleanup
{
5354 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5359 my $vollist = get_vm_volumes
($conf);
5360 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5363 foreach my $ext (qw(mon qmp pid vnc qga)) {
5364 unlink "/var/run/qemu-server/${vmid}.$ext";
5367 if ($conf->{ivshmem
}) {
5368 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5369 # just delete it for now, VMs which have this already open do not
5370 # are affected, but new VMs will get a separated one. If this
5371 # becomes an issue we either add some sort of ref-counting or just
5372 # add a "don't delete on stop" flag to the ivshmem format.
5373 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5376 foreach my $key (keys %$conf) {
5377 next if $key !~ m/^hostpci(\d+)$/;
5378 my $hostpciindex = $1;
5379 my $d = parse_hostpci
($conf->{$key});
5380 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5382 foreach my $pci (@{$d->{pciid
}}) {
5383 my $pciid = $pci->{id
};
5384 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5388 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5390 warn $@ if $@; # avoid errors - just warn
5393 # call only in locked context
5395 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5397 my $pid = check_running
($vmid, $nocheck);
5402 $conf = PVE
::QemuConfig-
>load_config($vmid);
5403 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5404 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5405 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5406 $timeout = $opts->{down
} if $opts->{down
};
5408 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5413 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
5414 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5416 mon_cmd
($vmid, "system_powerdown");
5419 mon_cmd
($vmid, "quit");
5425 $timeout = 60 if !defined($timeout);
5428 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5433 if ($count >= $timeout) {
5435 warn "VM still running - terminating now with SIGTERM\n";
5438 die "VM quit/powerdown failed - got timeout\n";
5441 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5445 if (!check_running
($vmid, $nocheck)) {
5446 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5450 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5453 die "VM quit/powerdown failed\n";
5461 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5466 if ($count >= $timeout) {
5467 warn "VM still running - terminating now with SIGKILL\n";
5472 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5475 # Note: use $nocheck to skip tests if VM configuration file exists.
5476 # We need that when migration VMs to other nodes (files already moved)
5477 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5479 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5481 $force = 1 if !defined($force) && !$shutdown;
5484 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5485 kill 15, $pid if $pid;
5486 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5487 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5491 PVE
::QemuConfig-
>lock_config($vmid, sub {
5492 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5497 my ($vmid, $timeout) = @_;
5499 PVE
::QemuConfig-
>lock_config($vmid, sub {
5502 # only reboot if running, as qmeventd starts it again on a stop event
5503 return if !check_running
($vmid);
5505 create_reboot_request
($vmid);
5507 my $storecfg = PVE
::Storage
::config
();
5508 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5512 # avoid that the next normal shutdown will be confused for a reboot
5513 clear_reboot_request
($vmid);
5519 # note: if using the statestorage parameter, the caller has to check privileges
5521 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5528 PVE
::QemuConfig-
>lock_config($vmid, sub {
5530 $conf = PVE
::QemuConfig-
>load_config($vmid);
5532 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5533 PVE
::QemuConfig-
>check_lock($conf)
5534 if !($skiplock || $is_backing_up);
5536 die "cannot suspend to disk during backup\n"
5537 if $is_backing_up && $includestate;
5539 if ($includestate) {
5540 $conf->{lock} = 'suspending';
5541 my $date = strftime
("%Y-%m-%d", localtime(time()));
5542 $storecfg = PVE
::Storage
::config
();
5543 if (!$statestorage) {
5544 $statestorage = find_vmstate_storage
($conf, $storecfg);
5545 # check permissions for the storage
5546 my $rpcenv = PVE
::RPCEnvironment
::get
();
5547 if ($rpcenv->{type
} ne 'cli') {
5548 my $authuser = $rpcenv->get_user();
5549 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5554 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
5555 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5556 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5557 PVE
::QemuConfig-
>write_config($vmid, $conf);
5559 mon_cmd
($vmid, "stop");
5563 if ($includestate) {
5565 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5568 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5570 my $state = mon_cmd
($vmid, "query-savevm");
5571 if (!$state->{status
}) {
5572 die "savevm not active\n";
5573 } elsif ($state->{status
} eq 'active') {
5576 } elsif ($state->{status
} eq 'completed') {
5577 print "State saved, quitting\n";
5579 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5580 die "query-savevm failed with error '$state->{error}'\n"
5582 die "query-savevm returned status '$state->{status}'\n";
5588 PVE
::QemuConfig-
>lock_config($vmid, sub {
5589 $conf = PVE
::QemuConfig-
>load_config($vmid);
5591 # cleanup, but leave suspending lock, to indicate something went wrong
5593 mon_cmd
($vmid, "savevm-end");
5594 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5595 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5596 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5597 PVE
::QemuConfig-
>write_config($vmid, $conf);
5603 die "lock changed unexpectedly\n"
5604 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5606 mon_cmd
($vmid, "quit");
5607 $conf->{lock} = 'suspended';
5608 PVE
::QemuConfig-
>write_config($vmid, $conf);
5614 my ($vmid, $skiplock, $nocheck) = @_;
5616 PVE
::QemuConfig-
>lock_config($vmid, sub {
5617 my $res = mon_cmd
($vmid, 'query-status');
5618 my $resume_cmd = 'cont';
5621 if ($res->{status
}) {
5622 return if $res->{status
} eq 'running'; # job done, go home
5623 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
5624 $reset = 1 if $res->{status
} eq 'shutdown';
5629 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5631 PVE
::QemuConfig-
>check_lock($conf)
5632 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5636 # required if a VM shuts down during a backup and we get a resume
5637 # request before the backup finishes for example
5638 mon_cmd
($vmid, "system_reset");
5640 mon_cmd
($vmid, $resume_cmd);
5645 my ($vmid, $skiplock, $key) = @_;
5647 PVE
::QemuConfig-
>lock_config($vmid, sub {
5649 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5651 # there is no qmp command, so we use the human monitor command
5652 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5653 die $res if $res ne '';
5657 # vzdump restore implementaion
5659 sub tar_archive_read_firstfile
{
5660 my $archive = shift;
5662 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5664 # try to detect archive type first
5665 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5666 die "unable to open file '$archive'\n";
5667 my $firstfile = <$fh>;
5671 die "ERROR: archive contaions no data\n" if !$firstfile;
5677 sub tar_restore_cleanup
{
5678 my ($storecfg, $statfile) = @_;
5680 print STDERR
"starting cleanup\n";
5682 if (my $fd = IO
::File-
>new($statfile, "r")) {
5683 while (defined(my $line = <$fd>)) {
5684 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5687 if ($volid =~ m
|^/|) {
5688 unlink $volid || die 'unlink failed\n';
5690 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5692 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5694 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5696 print STDERR
"unable to parse line in statfile - $line";
5703 sub restore_file_archive
{
5704 my ($archive, $vmid, $user, $opts) = @_;
5706 return restore_vma_archive
($archive, $vmid, $user, $opts)
5709 my $info = PVE
::Storage
::archive_info
($archive);
5710 my $format = $opts->{format
} // $info->{format
};
5711 my $comp = $info->{compression
};
5713 # try to detect archive format
5714 if ($format eq 'tar') {
5715 return restore_tar_archive
($archive, $vmid, $user, $opts);
5717 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5721 # hepler to remove disks that will not be used after restore
5722 my $restore_cleanup_oldconf = sub {
5723 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5725 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
5726 my ($ds, $drive) = @_;
5728 return if drive_is_cdrom
($drive, 1);
5730 my $volid = $drive->{file
};
5731 return if !$volid || $volid =~ m
|^/|;
5733 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
5734 return if !$path || !$owner || ($owner != $vmid);
5736 # Note: only delete disk we want to restore
5737 # other volumes will become unused
5738 if ($virtdev_hash->{$ds}) {
5739 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
5746 # delete vmstate files, after the restore we have no snapshots anymore
5747 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5748 my $snap = $oldconf->{snapshots
}->{$snapname};
5749 if ($snap->{vmstate
}) {
5750 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
5758 # Helper to parse vzdump backup device hints
5760 # $rpcenv: Environment, used to ckeck storage permissions
5761 # $user: User ID, to check storage permissions
5762 # $storecfg: Storage configuration
5763 # $fh: the file handle for reading the configuration
5764 # $devinfo: should contain device sizes for all backu-up'ed devices
5765 # $options: backup options (pool, default storage)
5767 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5768 my $parse_backup_hints = sub {
5769 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5771 my $virtdev_hash = {};
5773 while (defined(my $line = <$fh>)) {
5774 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5775 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5776 die "archive does not contain data for drive '$virtdev'\n"
5777 if !$devinfo->{$devname};
5779 if (defined($options->{storage
})) {
5780 $storeid = $options->{storage
} || 'local';
5781 } elsif (!$storeid) {
5784 $format = 'raw' if !$format;
5785 $devinfo->{$devname}->{devname
} = $devname;
5786 $devinfo->{$devname}->{virtdev
} = $virtdev;
5787 $devinfo->{$devname}->{format
} = $format;
5788 $devinfo->{$devname}->{storeid
} = $storeid;
5790 # check permission on storage
5791 my $pool = $options->{pool
}; # todo: do we need that?
5792 if ($user ne 'root@pam') {
5793 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5796 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5797 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5799 my $drive = parse_drive
($virtdev, $2);
5800 if (drive_is_cloudinit
($drive)) {
5801 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
5802 $storeid = $options->{storage
} if defined ($options->{storage
});
5803 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5804 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
5806 $virtdev_hash->{$virtdev} = {
5808 storeid
=> $storeid,
5809 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
5816 return $virtdev_hash;
5819 # Helper to allocate and activate all volumes required for a restore
5821 # $storecfg: Storage configuration
5822 # $virtdev_hash: as returned by parse_backup_hints()
5824 # Returns: { $virtdev => $volid }
5825 my $restore_allocate_devices = sub {
5826 my ($storecfg, $virtdev_hash, $vmid) = @_;
5829 foreach my $virtdev (sort keys %$virtdev_hash) {
5830 my $d = $virtdev_hash->{$virtdev};
5831 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5832 my $storeid = $d->{storeid
};
5833 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5835 # test if requested format is supported
5836 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5837 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
5838 $d->{format
} = $defFormat if !$supported;
5841 if ($d->{is_cloudinit
}) {
5842 $name = "vm-$vmid-cloudinit";
5843 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5844 if ($scfg->{path
}) {
5845 $name .= ".$d->{format}";
5849 my $volid = PVE
::Storage
::vdisk_alloc
(
5850 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
5852 print STDERR
"new volume ID is '$volid'\n";
5853 $d->{volid
} = $volid;
5855 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
5857 $map->{$virtdev} = $volid;
5863 my $restore_update_config_line = sub {
5864 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5866 return if $line =~ m/^\#qmdump\#/;
5867 return if $line =~ m/^\#vzdump\#/;
5868 return if $line =~ m/^lock:/;
5869 return if $line =~ m/^unused\d+:/;
5870 return if $line =~ m/^parent:/;
5872 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5873 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5874 # try to convert old 1.X settings
5875 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5876 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5877 my ($model, $macaddr) = split(/\=/, $devconfig);
5878 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5881 bridge
=> "vmbr$ind",
5882 macaddr
=> $macaddr,
5884 my $netstr = print_net
($net);
5886 print $outfd "net$cookie->{netcount}: $netstr\n";
5887 $cookie->{netcount
}++;
5889 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5890 my ($id, $netstr) = ($1, $2);
5891 my $net = parse_net
($netstr);
5892 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5893 $netstr = print_net
($net);
5894 print $outfd "$id: $netstr\n";
5895 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5898 my $di = parse_drive
($virtdev, $value);
5899 if (defined($di->{backup
}) && !$di->{backup
}) {
5900 print $outfd "#$line";
5901 } elsif ($map->{$virtdev}) {
5902 delete $di->{format
}; # format can change on restore
5903 $di->{file
} = $map->{$virtdev};
5904 $value = print_drive
($di);
5905 print $outfd "$virtdev: $value\n";
5909 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5911 if ($vmgenid ne '0') {
5912 # always generate a new vmgenid if there was a valid one setup
5913 $vmgenid = generate_uuid
();
5915 print $outfd "vmgenid: $vmgenid\n";
5916 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5917 my ($uuid, $uuid_str);
5918 UUID
::generate
($uuid);
5919 UUID
::unparse
($uuid, $uuid_str);
5920 my $smbios1 = parse_smbios1
($2);
5921 $smbios1->{uuid
} = $uuid_str;
5922 print $outfd $1.print_smbios1
($smbios1)."\n";
5928 my $restore_deactivate_volumes = sub {
5929 my ($storecfg, $devinfo) = @_;
5932 foreach my $devname (keys %$devinfo) {
5933 my $volid = $devinfo->{$devname}->{volid
};
5934 push @$vollist, $volid if $volid;
5937 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5940 my $restore_destroy_volumes = sub {
5941 my ($storecfg, $devinfo) = @_;
5943 foreach my $devname (keys %$devinfo) {
5944 my $volid = $devinfo->{$devname}->{volid
};
5947 if ($volid =~ m
|^/|) {
5948 unlink $volid || die 'unlink failed\n';
5950 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5952 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5954 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5959 my ($cfg, $vmid) = @_;
5961 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5963 my $volid_hash = {};
5964 foreach my $storeid (keys %$info) {
5965 foreach my $item (@{$info->{$storeid}}) {
5966 next if !($item->{volid
} && $item->{size
});
5967 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5968 $volid_hash->{$item->{volid
}} = $item;
5975 sub update_disk_config
{
5976 my ($vmid, $conf, $volid_hash) = @_;
5979 my $prefix = "VM $vmid";
5981 # used and unused disks
5982 my $referenced = {};
5984 # Note: it is allowed to define multiple storages with same path (alias), so
5985 # we need to check both 'volid' and real 'path' (two different volid can point
5986 # to the same path).
5988 my $referencedpath = {};
5991 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5992 my ($opt, $drive) = @_;
5994 my $volid = $drive->{file
};
5996 my $volume = $volid_hash->{$volid};
5998 # mark volid as "in-use" for next step
5999 $referenced->{$volid} = 1;
6000 if ($volume && (my $path = $volume->{path
})) {
6001 $referencedpath->{$path} = 1;
6004 return if drive_is_cdrom
($drive);
6007 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6008 if (defined($updated)) {
6010 $conf->{$opt} = print_drive
($updated);
6011 print "$prefix ($opt): $msg\n";
6015 # remove 'unusedX' entry if volume is used
6016 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6017 my ($opt, $drive) = @_;
6019 my $volid = $drive->{file
};
6023 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6024 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6025 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6027 delete $conf->{$opt};
6030 $referenced->{$volid} = 1;
6031 $referencedpath->{$path} = 1 if $path;
6034 foreach my $volid (sort keys %$volid_hash) {
6035 next if $volid =~ m/vm-$vmid-state-/;
6036 next if $referenced->{$volid};
6037 my $path = $volid_hash->{$volid}->{path
};
6038 next if !$path; # just to be sure
6039 next if $referencedpath->{$path};
6041 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6042 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6043 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6050 my ($vmid, $nolock, $dryrun) = @_;
6052 my $cfg = PVE
::Storage
::config
();
6054 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6055 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6056 foreach my $stor (keys %{$cfg->{ids
}}) {
6057 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
6060 print "rescan volumes...\n";
6061 my $volid_hash = scan_volids
($cfg, $vmid);
6063 my $updatefn = sub {
6066 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6068 PVE
::QemuConfig-
>check_lock($conf);
6071 foreach my $volid (keys %$volid_hash) {
6072 my $info = $volid_hash->{$volid};
6073 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6076 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6078 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6081 if (defined($vmid)) {
6085 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6088 my $vmlist = config_list
();
6089 foreach my $vmid (keys %$vmlist) {
6093 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6099 sub restore_proxmox_backup_archive
{
6100 my ($archive, $vmid, $user, $options) = @_;
6102 my $storecfg = PVE
::Storage
::config
();
6104 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6105 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6107 my $fingerprint = $scfg->{fingerprint
};
6108 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6110 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6112 # This is only used for `pbs-restore`!
6113 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6114 local $ENV{PBS_PASSWORD
} = $password;
6115 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6117 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6118 PVE
::Storage
::parse_volname
($storecfg, $archive);
6120 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6122 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6124 my $tmpdir = "/var/tmp/vzdumptmp$$";
6128 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6129 my $tmpfn = "$conffile.$$.tmp";
6130 # disable interrupts (always do cleanups)
6134 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6136 # Note: $oldconf is undef if VM does not exists
6137 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6138 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6140 my $rpcenv = PVE
::RPCEnvironment
::get
();
6149 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6151 my $cfgfn = "$tmpdir/qemu-server.conf";
6152 my $firewall_config_fn = "$tmpdir/fw.conf";
6153 my $index_fn = "$tmpdir/index.json";
6155 my $cmd = "restore";
6157 my $param = [$pbs_backup_name, "index.json", $index_fn];
6158 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6159 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6160 $index = decode_json
($index);
6162 # print Dumper($index);
6163 foreach my $info (@{$index->{files
}}) {
6164 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6166 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6167 $devinfo->{$devname}->{size
} = $1;
6169 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6174 my $is_qemu_server_backup = scalar(
6175 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6177 if (!$is_qemu_server_backup) {
6178 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6180 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6182 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6183 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6185 if ($has_firewall_config) {
6186 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6187 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6189 my $pve_firewall_dir = '/etc/pve/firewall';
6190 mkdir $pve_firewall_dir; # make sure the dir exists
6191 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6194 my $fh = IO
::File-
>new($cfgfn, "r") ||
6195 die "unable to read qemu-server.conf - $!\n";
6197 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6199 # fixme: rate limit?
6201 # create empty/temp config
6202 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6204 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6207 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6209 foreach my $virtdev (sort keys %$virtdev_hash) {
6210 my $d = $virtdev_hash->{$virtdev};
6211 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6213 my $volid = $d->{volid
};
6215 my $path = PVE
::Storage
::path
($storecfg, $volid);
6217 # This is the ONLY user of the PBS_ env vars set on top of this function!
6218 my $pbs_restore_cmd = [
6219 '/usr/bin/pbs-restore',
6220 '--repository', $repo,
6222 "$d->{devname}.img.fidx",
6227 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6228 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6230 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6231 push @$pbs_restore_cmd, '--skip-zero';
6234 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6235 print "restore proxmox backup image: $dbg_cmdstring\n";
6236 run_command
($pbs_restore_cmd);
6239 $fh->seek(0, 0) || die "seek failed - $!\n";
6241 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6243 my $cookie = { netcount
=> 0 };
6244 while (defined(my $line = <$fh>)) {
6245 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique
});
6253 $restore_deactivate_volumes->($storecfg, $devinfo);
6259 $restore_destroy_volumes->($storecfg, $devinfo);
6263 rename($tmpfn, $conffile) ||
6264 die "unable to commit configuration file '$conffile'\n";
6266 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6268 eval { rescan
($vmid, 1); };
6272 sub restore_vma_archive
{
6273 my ($archive, $vmid, $user, $opts, $comp) = @_;
6275 my $readfrom = $archive;
6277 my $cfg = PVE
::Storage
::config
();
6279 my $bwlimit = $opts->{bwlimit
};
6281 my $dbg_cmdstring = '';
6282 my $add_pipe = sub {
6284 push @$commands, $cmd;
6285 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6286 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6291 if ($archive eq '-') {
6294 # If we use a backup from a PVE defined storage we also consider that
6295 # storage's rate limit:
6296 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6297 if (defined($volid)) {
6298 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6299 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6301 print STDERR
"applying read rate limit: $readlimit\n";
6302 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6303 $add_pipe->($cstream);
6309 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6310 my $cmd = $info->{decompressor
};
6311 push @$cmd, $readfrom;
6315 my $tmpdir = "/var/tmp/vzdumptmp$$";
6318 # disable interrupts (always do cleanups)
6322 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6324 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6325 POSIX
::mkfifo
($mapfifo, 0600);
6327 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6329 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6336 my $rpcenv = PVE
::RPCEnvironment
::get
();
6338 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6339 my $tmpfn = "$conffile.$$.tmp";
6341 # Note: $oldconf is undef if VM does not exist
6342 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6343 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6347 my $print_devmap = sub {
6348 my $cfgfn = "$tmpdir/qemu-server.conf";
6350 # we can read the config - that is already extracted
6351 my $fh = IO
::File-
>new($cfgfn, "r") ||
6352 die "unable to read qemu-server.conf - $!\n";
6354 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6356 my $pve_firewall_dir = '/etc/pve/firewall';
6357 mkdir $pve_firewall_dir; # make sure the dir exists
6358 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6361 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6363 foreach my $key (keys %storage_limits) {
6364 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6366 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6367 $storage_limits{$key} = $limit * 1024;
6370 foreach my $devname (keys %$devinfo) {
6371 die "found no device mapping information for device '$devname'\n"
6372 if !$devinfo->{$devname}->{virtdev
};
6375 # create empty/temp config
6377 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6378 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6382 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6384 # print restore information to $fifofh
6385 foreach my $virtdev (sort keys %$virtdev_hash) {
6386 my $d = $virtdev_hash->{$virtdev};
6387 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6389 my $storeid = $d->{storeid
};
6390 my $volid = $d->{volid
};
6393 if (my $limit = $storage_limits{$storeid}) {
6394 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6397 my $write_zeros = 1;
6398 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6402 my $path = PVE
::Storage
::path
($cfg, $volid);
6404 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6406 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6409 $fh->seek(0, 0) || die "seek failed - $!\n";
6411 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6413 my $cookie = { netcount
=> 0 };
6414 while (defined(my $line = <$fh>)) {
6415 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6428 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6429 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6431 $oldtimeout = alarm($timeout);
6438 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6439 my ($dev_id, $size, $devname) = ($1, $2, $3);
6440 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6441 } elsif ($line =~ m/^CTIME: /) {
6442 # we correctly received the vma config, so we can disable
6443 # the timeout now for disk allocation (set to 10 minutes, so
6444 # that we always timeout if something goes wrong)
6447 print $fifofh "done\n";
6448 my $tmp = $oldtimeout || 0;
6449 $oldtimeout = undef;
6456 print "restore vma archive: $dbg_cmdstring\n";
6457 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6461 alarm($oldtimeout) if $oldtimeout;
6463 $restore_deactivate_volumes->($cfg, $devinfo);
6465 close($fifofh) if $fifofh;
6471 $restore_destroy_volumes->($cfg, $devinfo);
6475 rename($tmpfn, $conffile) ||
6476 die "unable to commit configuration file '$conffile'\n";
6478 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6480 eval { rescan
($vmid, 1); };
6484 sub restore_tar_archive
{
6485 my ($archive, $vmid, $user, $opts) = @_;
6487 if ($archive ne '-') {
6488 my $firstfile = tar_archive_read_firstfile
($archive);
6489 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6490 if $firstfile ne 'qemu-server.conf';
6493 my $storecfg = PVE
::Storage
::config
();
6495 # avoid zombie disks when restoring over an existing VM -> cleanup first
6496 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6497 # skiplock=1 because qmrestore has set the 'create' lock itself already
6498 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6499 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6501 my $tocmd = "/usr/lib/qemu-server/qmextract";
6503 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6504 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6505 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6506 $tocmd .= ' --info' if $opts->{info
};
6508 # tar option "xf" does not autodetect compression when read from STDIN,
6509 # so we pipe to zcat
6510 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6511 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6513 my $tmpdir = "/var/tmp/vzdumptmp$$";
6516 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6517 local $ENV{VZDUMP_VMID
} = $vmid;
6518 local $ENV{VZDUMP_USER
} = $user;
6520 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6521 my $tmpfn = "$conffile.$$.tmp";
6523 # disable interrupts (always do cleanups)
6527 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6535 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6537 if ($archive eq '-') {
6538 print "extracting archive from STDIN\n";
6539 run_command
($cmd, input
=> "<&STDIN");
6541 print "extracting archive '$archive'\n";
6545 return if $opts->{info
};
6549 my $statfile = "$tmpdir/qmrestore.stat";
6550 if (my $fd = IO
::File-
>new($statfile, "r")) {
6551 while (defined (my $line = <$fd>)) {
6552 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6553 $map->{$1} = $2 if $1;
6555 print STDERR
"unable to parse line in statfile - $line\n";
6561 my $confsrc = "$tmpdir/qemu-server.conf";
6563 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6565 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6567 my $cookie = { netcount
=> 0 };
6568 while (defined (my $line = <$srcfd>)) {
6569 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6577 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6583 rename $tmpfn, $conffile ||
6584 die "unable to commit configuration file '$conffile'\n";
6586 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6588 eval { rescan
($vmid, 1); };
6592 sub foreach_storage_used_by_vm
{
6593 my ($conf, $func) = @_;
6597 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6598 my ($ds, $drive) = @_;
6599 return if drive_is_cdrom
($drive);
6601 my $volid = $drive->{file
};
6603 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6604 $sidhash->{$sid} = $sid if $sid;
6607 foreach my $sid (sort keys %$sidhash) {
6612 my $qemu_snap_storage = {
6615 sub do_snapshots_with_qemu
{
6616 my ($storecfg, $volid) = @_;
6618 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6619 my $scfg = $storecfg->{ids
}->{$storage_name};
6620 die "could not find storage '$storage_name'\n" if !defined($scfg);
6622 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6626 if ($volid =~ m/\.(qcow2|qed)$/){
6633 sub qga_check_running
{
6634 my ($vmid, $nowarn) = @_;
6636 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6638 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6644 sub template_create
{
6645 my ($vmid, $conf, $disk) = @_;
6647 my $storecfg = PVE
::Storage
::config
();
6649 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6650 my ($ds, $drive) = @_;
6652 return if drive_is_cdrom
($drive);
6653 return if $disk && $ds ne $disk;
6655 my $volid = $drive->{file
};
6656 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6658 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6659 $drive->{file
} = $voliddst;
6660 $conf->{$ds} = print_drive
($drive);
6661 PVE
::QemuConfig-
>write_config($vmid, $conf);
6665 sub convert_iscsi_path
{
6668 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6673 my $initiator_name = get_initiator_name
();
6675 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6676 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6679 die "cannot convert iscsi path '$path', unkown format\n";
6682 sub qemu_img_convert
{
6683 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6685 my $storecfg = PVE
::Storage
::config
();
6686 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6687 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6689 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6693 my $src_is_iscsi = 0;
6697 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6698 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6699 $src_format = qemu_img_format
($src_scfg, $src_volname);
6700 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6701 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6702 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6703 } elsif (-f
$src_volid) {
6704 $src_path = $src_volid;
6705 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6710 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6712 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6713 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6714 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6715 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6718 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6719 push @$cmd, '-l', "snapshot.name=$snapname"
6720 if $snapname && $src_format && $src_format eq "qcow2";
6721 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6722 push @$cmd, '-T', $cachemode if defined($cachemode);
6724 if ($src_is_iscsi) {
6725 push @$cmd, '--image-opts';
6726 $src_path = convert_iscsi_path
($src_path);
6727 } elsif ($src_format) {
6728 push @$cmd, '-f', $src_format;
6731 if ($dst_is_iscsi) {
6732 push @$cmd, '--target-image-opts';
6733 $dst_path = convert_iscsi_path
($dst_path);
6735 push @$cmd, '-O', $dst_format;
6738 push @$cmd, $src_path;
6740 if (!$dst_is_iscsi && $is_zero_initialized) {
6741 push @$cmd, "zeroinit:$dst_path";
6743 push @$cmd, $dst_path;
6748 if($line =~ m/\((\S+)\/100\
%\)/){
6750 my $transferred = int($size * $percent / 100);
6751 my $remaining = $size - $transferred;
6753 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6758 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6760 die "copy failed: $err" if $err;
6763 sub qemu_img_format
{
6764 my ($scfg, $volname) = @_;
6766 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6773 sub qemu_drive_mirror
{
6774 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6776 $jobs = {} if !$jobs;
6780 $jobs->{"drive-$drive"} = {};
6782 if ($dst_volid =~ /^nbd:/) {
6783 $qemu_target = $dst_volid;
6786 my $storecfg = PVE
::Storage
::config
();
6787 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6789 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6791 $format = qemu_img_format
($dst_scfg, $dst_volname);
6793 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6795 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6798 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6799 $opts->{format
} = $format if $format;
6801 if (defined($src_bitmap)) {
6802 $opts->{sync
} = 'incremental';
6803 $opts->{bitmap
} = $src_bitmap;
6804 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6807 if (defined($bwlimit)) {
6808 $opts->{speed
} = $bwlimit * 1024;
6809 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6811 print "drive mirror is starting for drive-$drive\n";
6814 # if a job already runs for this device we get an error, catch it for cleanup
6815 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
6817 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6819 die "mirroring error: $err\n";
6822 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
6825 # $completion can be either
6826 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6827 # 'cancel': wait until all jobs are ready, block-job-cancel them
6828 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6829 sub qemu_drive_mirror_monitor
{
6830 my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6832 $completion //= 'complete';
6835 my $err_complete = 0;
6838 die "storage migration timed out\n" if $err_complete > 300;
6840 my $stats = mon_cmd
($vmid, "query-block-jobs");
6842 my $running_mirror_jobs = {};
6843 foreach my $stat (@$stats) {
6844 next if $stat->{type
} ne 'mirror';
6845 $running_mirror_jobs->{$stat->{device
}} = $stat;
6848 my $readycounter = 0;
6850 foreach my $job (keys %$jobs) {
6852 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6853 print "$job : finished\n";
6854 delete $jobs->{$job};
6858 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6860 my $busy = $running_mirror_jobs->{$job}->{busy
};
6861 my $ready = $running_mirror_jobs->{$job}->{ready
};
6862 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6863 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6864 my $remaining = $total - $transferred;
6865 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6867 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6870 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6873 last if scalar(keys %$jobs) == 0;
6875 if ($readycounter == scalar(keys %$jobs)) {
6876 print "all mirroring jobs are ready \n";
6877 last if $completion eq 'skip'; #do the complete later
6879 if ($vmiddst && $vmiddst != $vmid) {
6880 my $agent_running = $qga && qga_check_running
($vmid);
6881 if ($agent_running) {
6882 print "freeze filesystem\n";
6883 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6885 print "suspend vm\n";
6886 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6889 # if we clone a disk for a new target vm, we don't switch the disk
6890 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6892 if ($agent_running) {
6893 print "unfreeze filesystem\n";
6894 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6896 print "resume vm\n";
6897 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6903 foreach my $job (keys %$jobs) {
6904 # try to switch the disk if source and destination are on the same guest
6905 print "$job: Completing block job...\n";
6908 if ($completion eq 'complete') {
6909 $op = 'block-job-complete';
6910 } elsif ($completion eq 'cancel') {
6911 $op = 'block-job-cancel';
6913 die "invalid completion value: $completion\n";
6915 eval { mon_cmd
($vmid, $op, device
=> $job) };
6916 if ($@ =~ m/cannot be completed/) {
6917 print "$job: Block job cannot be completed, try again.\n";
6920 print "$job: Completed successfully.\n";
6921 $jobs->{$job}->{complete
} = 1;
6932 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6933 die "mirroring error: $err";
6938 sub qemu_blockjobs_cancel
{
6939 my ($vmid, $jobs) = @_;
6941 foreach my $job (keys %$jobs) {
6942 print "$job: Cancelling block job\n";
6943 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6944 $jobs->{$job}->{cancel
} = 1;
6948 my $stats = mon_cmd
($vmid, "query-block-jobs");
6950 my $running_jobs = {};
6951 foreach my $stat (@$stats) {
6952 $running_jobs->{$stat->{device
}} = $stat;
6955 foreach my $job (keys %$jobs) {
6957 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6958 print "$job: Done.\n";
6959 delete $jobs->{$job};
6963 last if scalar(keys %$jobs) == 0;
6970 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6971 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6976 print "create linked clone of drive $drivename ($drive->{file})\n";
6977 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6978 push @$newvollist, $newvolid;
6981 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6982 $storeid = $storage if $storage;
6984 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6986 print "create full clone of drive $drivename ($drive->{file})\n";
6989 if (drive_is_cloudinit
($drive)) {
6990 $name = "vm-$newvmid-cloudinit";
6991 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6992 if ($scfg->{path
}) {
6993 $name .= ".$dst_format";
6996 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
6997 } elsif ($drivename eq 'efidisk0') {
6998 $size = get_efivars_size
($conf);
7000 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
7002 $newvolid = PVE
::Storage
::vdisk_alloc
(
7003 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7005 push @$newvollist, $newvolid;
7007 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
7009 if (drive_is_cloudinit
($drive)) {
7010 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7011 # if this is the case, we have to complete any block-jobs still there from
7012 # previous drive-mirrors
7013 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7014 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
7019 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
7020 if (!$running || $snapname) {
7021 # TODO: handle bwlimits
7022 if ($drivename eq 'efidisk0') {
7023 # the relevant data on the efidisk may be smaller than the source
7024 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7025 # that is given by the OVMF_VARS.fd
7026 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
7027 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
7029 # better for Ceph if block size is not too small, see bug #3324
7032 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7033 "if=$src_path", "of=$dst_path"]);
7035 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
7039 my $kvmver = get_running_qemu_version
($vmid);
7040 if (!min_version
($kvmver, 2, 7)) {
7041 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7042 if $drive->{iothread
};
7045 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7046 $completion, $qga, $bwlimit);
7051 my ($size) = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
7054 $disk->{format
} = undef;
7055 $disk->{file
} = $newvolid;
7056 $disk->{size
} = $size if defined($size);
7061 sub get_running_qemu_version
{
7063 my $res = mon_cmd
($vmid, "query-version");
7064 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7067 sub qemu_use_old_bios_files
{
7068 my ($machine_type) = @_;
7070 return if !$machine_type;
7072 my $use_old_bios_files = undef;
7074 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7076 $use_old_bios_files = 1;
7078 my $version = extract_version
($machine_type, kvm_user_version
());
7079 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7080 # load new efi bios files on migration. So this hack is required to allow
7081 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7082 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7083 $use_old_bios_files = !min_version
($version, 2, 4);
7086 return ($use_old_bios_files, $machine_type);
7089 sub get_efivars_size
{
7091 my $arch = get_vm_arch
($conf);
7092 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7093 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
7094 return -s
$ovmf_vars;
7097 sub update_efidisk_size
{
7100 return if !defined($conf->{efidisk0
});
7102 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
7103 $disk->{size
} = get_efivars_size
($conf);
7104 $conf->{efidisk0
} = print_drive
($disk);
7109 sub create_efidisk
($$$$$) {
7110 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7112 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7113 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7115 my $vars_size_b = -s
$ovmf_vars;
7116 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7117 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7118 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7120 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7121 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7123 return ($volid, $size/1024);
7126 sub vm_iothreads_list
{
7129 my $res = mon_cmd
($vmid, 'query-iothreads');
7132 foreach my $iothread (@$res) {
7133 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7140 my ($conf, $drive) = @_;
7144 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7146 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7152 my $controller = int($drive->{index} / $maxdev);
7153 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
7157 return ($maxdev, $controller, $controller_prefix);
7160 sub windows_version
{
7163 return 0 if !$ostype;
7167 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7169 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7171 } elsif ($ostype =~ m/^win(\d+)$/) {
7178 sub resolve_dst_disk_format
{
7179 my ($storecfg, $storeid, $src_volname, $format) = @_;
7180 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7183 # if no target format is specified, use the source disk format as hint
7185 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7186 $format = qemu_img_format
($scfg, $src_volname);
7192 # test if requested format is supported - else use default
7193 my $supported = grep { $_ eq $format } @$validFormats;
7194 $format = $defFormat if !$supported;
7198 # NOTE: if this logic changes, please update docs & possibly gui logic
7199 sub find_vmstate_storage
{
7200 my ($conf, $storecfg) = @_;
7202 # first, return storage from conf if set
7203 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7205 my ($target, $shared, $local);
7207 foreach_storage_used_by_vm
($conf, sub {
7209 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7210 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7211 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7214 # second, use shared storage where VM has at least one disk
7215 # third, use local storage where VM has at least one disk
7216 # fall back to local storage
7217 $target = $shared // $local // 'local';
7223 my ($uuid, $uuid_str);
7224 UUID
::generate
($uuid);
7225 UUID
::unparse
($uuid, $uuid_str);
7229 sub generate_smbios1_uuid
{
7230 return "uuid=".generate_uuid
();
7236 mon_cmd
($vmid, 'nbd-server-stop');
7239 sub create_reboot_request
{
7241 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7242 or die "failed to create reboot trigger file: $!\n";
7246 sub clear_reboot_request
{
7248 my $path = "/run/qemu-server/$vmid.reboot";
7251 $res = unlink($path);
7252 die "could not remove reboot request for $vmid: $!"
7253 if !$res && $! != POSIX
::ENOENT
;
7258 sub bootorder_from_legacy
{
7259 my ($conf, $bootcfg) = @_;
7261 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
7262 my $bootindex_hash = {};
7264 foreach my $o (split(//, $boot)) {
7265 $bootindex_hash->{$o} = $i*100;
7271 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7272 my ($ds, $drive) = @_;
7274 if (drive_is_cdrom
($drive, 1)) {
7275 if ($bootindex_hash->{d
}) {
7276 $bootorder->{$ds} = $bootindex_hash->{d
};
7277 $bootindex_hash->{d
} += 1;
7279 } elsif ($bootindex_hash->{c
}) {
7280 $bootorder->{$ds} = $bootindex_hash->{c
}
7281 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
7282 $bootindex_hash->{c
} += 1;
7286 if ($bootindex_hash->{n
}) {
7287 for (my $i = 0; $i < $MAX_NETS; $i++) {
7288 my $netname = "net$i";
7289 next if !$conf->{$netname};
7290 $bootorder->{$netname} = $bootindex_hash->{n
};
7291 $bootindex_hash->{n
} += 1;
7298 # Generate default device list for 'boot: order=' property. Matches legacy
7299 # default boot order, but with explicit device names. This is important, since
7300 # the fallback for when neither 'order' nor the old format is specified relies
7301 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7302 sub get_default_bootdevices
{
7308 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
7309 push @ret, $first if $first;
7312 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
7313 push @ret, $first if $first;
7316 for (my $i = 0; $i < $MAX_NETS; $i++) {
7317 my $netname = "net$i";
7318 next if !$conf->{$netname};
7319 push @ret, $netname;
7326 sub device_bootorder
{
7329 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
7331 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
7334 if (!defined($boot) || $boot->{legacy
}) {
7335 $bootorder = bootorder_from_legacy
($conf, $boot);
7336 } elsif ($boot->{order
}) {
7337 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7338 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
7339 $bootorder->{$dev} = $i++;
7346 # bash completion helper
7348 sub complete_backup_archives
{
7349 my ($cmdname, $pname, $cvalue) = @_;
7351 my $cfg = PVE
::Storage
::config
();
7355 if ($cvalue =~ m/^([^:]+):/) {
7359 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7362 foreach my $id (keys %$data) {
7363 foreach my $item (@{$data->{$id}}) {
7364 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7365 push @$res, $item->{volid
} if defined($item->{volid
});
7372 my $complete_vmid_full = sub {
7375 my $idlist = vmstatus
();
7379 foreach my $id (keys %$idlist) {
7380 my $d = $idlist->{$id};
7381 if (defined($running)) {
7382 next if $d->{template
};
7383 next if $running && $d->{status
} ne 'running';
7384 next if !$running && $d->{status
} eq 'running';
7393 return &$complete_vmid_full();
7396 sub complete_vmid_stopped
{
7397 return &$complete_vmid_full(0);
7400 sub complete_vmid_running
{
7401 return &$complete_vmid_full(1);
7404 sub complete_storage
{
7406 my $cfg = PVE
::Storage
::config
();
7407 my $ids = $cfg->{ids
};
7410 foreach my $sid (keys %$ids) {
7411 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7412 next if !$ids->{$sid}->{content
}->{images
};
7419 sub complete_migration_storage
{
7420 my ($cmd, $param, $current_value, $all_args) = @_;
7422 my $targetnode = @$all_args[1];
7424 my $cfg = PVE
::Storage
::config
();
7425 my $ids = $cfg->{ids
};
7428 foreach my $sid (keys %$ids) {
7429 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7430 next if !$ids->{$sid}->{content
}->{images
};
7439 my $qmpstatus = eval {
7440 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
7441 mon_cmd
($vmid, "query-status");
7444 return $qmpstatus && $qmpstatus->{status
} eq "paused";