1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
31 use PVE
::DataCenterConfig
;
32 use PVE
::Exception
qw(raise raise_param_exc);
33 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
35 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
38 use PVE
::RPCEnvironment
;
42 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
46 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
47 use PVE
::QemuServer
::Cloudinit
;
48 use PVE
::QemuServer
::CGroup
;
49 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
50 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
51 use PVE
::QemuServer
::Machine
;
52 use PVE
::QemuServer
::Memory
;
53 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
54 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
55 use PVE
::QemuServer
::USB
qw(parse_usb_device);
59 require PVE
::Network
::SDN
::Zones
;
63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
66 "$EDK2_FW_BASE/OVMF_CODE.fd",
67 "$EDK2_FW_BASE/OVMF_VARS.fd"
70 "$EDK2_FW_BASE/AAVMF_CODE.fd",
71 "$EDK2_FW_BASE/AAVMF_VARS.fd"
75 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
77 # Note about locking: we use flock on the config file protect
78 # against concurent actions.
79 # Aditionaly, we have a 'lock' setting in the config file. This
80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
81 # allowed when such lock is set. But you can ignore this kind of
82 # lock with the --skiplock flag.
84 cfs_register_file
('/qemu-server/',
88 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
89 description
=> "Some command save/restore state from this location.",
95 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
96 description
=> "Specifies the Qemu machine type.",
98 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
105 my ($map, $source) = @_;
107 return $source if !defined($map);
109 return $map->{entries
}->{$source}
110 if $map->{entries
} && defined($map->{entries
}->{$source});
112 return $map->{default} if $map->{default};
114 # identity (fallback)
118 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
119 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
121 format
=> 'storagepair-list',
125 #no warnings 'redefine';
129 $nodename_cache //= PVE
::INotify
::nodename
();
130 return $nodename_cache;
137 enum
=> [qw(i6300esb ib700)],
138 description
=> "Watchdog type to emulate.",
139 default => 'i6300esb',
144 enum
=> [qw(reset shutdown poweroff pause debug none)],
145 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
149 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
153 description
=> "Enable/disable Qemu GuestAgent.",
158 fstrim_cloned_disks
=> {
159 description
=> "Run fstrim after moving a disk or migrating the VM.",
165 description
=> "Select the agent type",
169 enum
=> [qw(virtio isa)],
175 description
=> "Select the VGA type.",
180 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
183 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
195 description
=> "The size of the file in MB.",
199 pattern
=> '[a-zA-Z0-9\-]+',
201 format_description
=> 'string',
202 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
209 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
210 description
=> "Configure an audio device."
214 enum
=> ['spice', 'none'],
217 description
=> "Driver backend for the audio device."
221 my $spice_enhancements_fmt = {
226 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
230 enum
=> ['off', 'all', 'filter'],
233 description
=> "Enable video streaming. Uses compression for detected video streams."
240 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
242 description
=> "The file on the host to gather entropy from. In most"
243 . " cases /dev/urandom should be preferred over /dev/random"
244 . " to avoid entropy-starvation issues on the host. Using"
245 . " urandom does *not* decrease security in any meaningful"
246 . " way, as it's still seeded from real entropy, and the"
247 . " bytes provided will most likely be mixed with real"
248 . " entropy on the guest as well. /dev/hwrng can be used"
249 . " to pass through a hardware RNG from the host.",
253 description
=> "Maximum bytes of entropy injected into the guest every"
254 . " 'period' milliseconds. Prefer a lower value when using"
255 . " /dev/random as source. Use 0 to disable limiting"
256 . " (potentially dangerous!).",
259 # default is 1 KiB/s, provides enough entropy to the guest to avoid
260 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
261 # of overwhelming the host, provided we're reading from /dev/urandom
266 description
=> "Every 'period' milliseconds the entropy-injection quota"
267 . " is reset, allowing the guest to retrieve another"
268 . " 'max_bytes' of entropy.",
278 description
=> "Specifies whether a VM will be started during system bootup.",
284 description
=> "Automatic restart after crash (currently ignored).",
289 type
=> 'string', format
=> 'pve-hotplug-features',
290 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
291 default => 'network,disk,usb',
296 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
302 description
=> "Lock/unlock the VM.",
303 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
308 description
=> "Limit of CPU usage.",
309 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
317 description
=> "CPU weight for a VM.",
318 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
326 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
333 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
339 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
347 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
348 "It should not be necessary to set it.",
349 enum
=> PVE
::Tools
::kvmkeymaplist
(),
354 type
=> 'string', format
=> 'dns-name',
355 description
=> "Set a name for the VM. Only used on the configuration web interface.",
360 description
=> "SCSI controller model",
361 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
367 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
372 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
373 description
=> "Specify guest operating system.",
374 verbose_description
=> <<EODESC,
375 Specify guest operating system. This is used to enable special
376 optimization/features for specific operating systems:
379 other;; unspecified OS
380 wxp;; Microsoft Windows XP
381 w2k;; Microsoft Windows 2000
382 w2k3;; Microsoft Windows 2003
383 w2k8;; Microsoft Windows 2008
384 wvista;; Microsoft Windows Vista
385 win7;; Microsoft Windows 7
386 win8;; Microsoft Windows 8/2012/2012r2
387 win10;; Microsoft Windows 10/2016
388 l24;; Linux 2.4 Kernel
389 l26;; Linux 2.6 - 5.X Kernel
390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
395 type
=> 'string', format
=> 'pve-qm-boot',
396 description
=> "Specify guest boot order. Use with 'order=', usage with"
397 . " no key or 'legacy=' is deprecated.",
401 type
=> 'string', format
=> 'pve-qm-bootdisk',
402 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
403 pattern
=> '(ide|sata|scsi|virtio)\d+',
408 description
=> "The number of CPUs. Please use option -sockets instead.",
415 description
=> "The number of CPU sockets.",
422 description
=> "The number of cores per socket.",
429 description
=> "Enable/disable NUMA.",
435 description
=> "Enable/disable hugepages memory.",
436 enum
=> [qw(any 2 1024)],
442 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
443 ." after VM shutdown and can be used for subsequent starts.",
448 description
=> "Number of hotplugged vcpus.",
455 description
=> "Enable/disable ACPI.",
460 description
=> "Enable/disable Qemu GuestAgent and its properties.",
462 format
=> $agent_fmt,
467 description
=> "Enable/disable KVM hardware virtualization.",
473 description
=> "Enable/disable time drift fix.",
479 description
=> "Set the real time clock to local time. This is enabled by default if ostype"
480 ." indicates a Microsoft OS.",
485 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
489 type
=> 'string', format
=> $vga_fmt,
490 description
=> "Configure the VGA hardware.",
491 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
492 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
493 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
494 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
495 ." display server. For win* OS you can select how many independent displays you want,"
496 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
497 ." using a serial device as terminal.",
501 type
=> 'string', format
=> 'pve-qm-watchdog',
502 description
=> "Create a virtual hardware watchdog device.",
503 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
504 ." action), the watchdog must be periodically polled by an agent inside the guest or"
505 ." else the watchdog will reset the guest (or execute the respective action specified)",
510 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
511 description
=> "Set the initial date of the real time clock. Valid format for date are:"
512 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
513 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
516 startup
=> get_standard_option
('pve-startup-order'),
520 description
=> "Enable/disable Template.",
526 description
=> "Arbitrary arguments passed to kvm.",
527 verbose_description
=> <<EODESCR,
528 Arbitrary arguments passed to kvm, for example:
530 args: -no-reboot -no-hpet
532 NOTE: this option is for experts only.
539 description
=> "Enable/disable the USB tablet device.",
540 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
541 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
542 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
543 ." may consider disabling this to save some context switches. This is turned off by"
544 ." default if you use spice (`qm set <vmid> --vga qxl`).",
549 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
553 migrate_downtime
=> {
556 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
562 type
=> 'string', format
=> 'pve-qm-ide',
563 typetext
=> '<volume>',
564 description
=> "This is an alias for option -ide2",
568 description
=> "Emulated CPU type.",
570 format
=> 'pve-vm-cpu-conf',
572 parent
=> get_standard_option
('pve-snapshot-name', {
574 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
578 description
=> "Timestamp for snapshots.",
584 type
=> 'string', format
=> 'pve-volume-id',
585 description
=> "Reference to a volume which stores the VM state. This is used internally"
588 vmstatestorage
=> get_standard_option
('pve-storage-id', {
589 description
=> "Default storage for VM state volumes/files.",
592 runningmachine
=> get_standard_option
('pve-qemu-machine', {
593 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
597 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
598 ." internally for snapshots.",
601 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
602 format_description
=> 'QEMU -cpu parameter'
604 machine
=> get_standard_option
('pve-qemu-machine'),
606 description
=> "Virtual processor architecture. Defaults to the host.",
609 enum
=> [qw(x86_64 aarch64)],
612 description
=> "Specify SMBIOS type 1 fields.",
613 type
=> 'string', format
=> 'pve-qm-smbios1',
620 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
621 ." remove disk operations.",
627 enum
=> [ qw(seabios ovmf) ],
628 description
=> "Select BIOS implementation.",
629 default => 'seabios',
633 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
634 format_description
=> 'UUID',
635 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
636 ." to disable explicitly.",
637 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
638 ." value identifier to the guest OS. This allows to notify the guest operating system"
639 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
640 ." execution or creation from a template). The guest operating system notices the"
641 ." change, and is then able to react as appropriate by marking its copies of"
642 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
643 ."Note that auto-creation only works when done through API/CLI create or update methods"
644 .", but not when manually editing the config file.",
645 default => "1 (autogenerated)",
650 format
=> 'pve-volume-id',
652 description
=> "Script that will be executed during various steps in the vms lifetime.",
656 format
=> $ivshmem_fmt,
657 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
663 format
=> $audio_fmt,
664 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
667 spice_enhancements
=> {
669 format
=> $spice_enhancements_fmt,
670 description
=> "Configure additional enhancements for SPICE.",
674 type
=> 'string', format
=> 'pve-tag-list',
675 description
=> 'Tags of the VM. This is only meta information.',
681 description
=> "Configure a VirtIO-based Random Number Generator.",
690 description
=> 'Specify a custom file containing all meta data passed to the VM via"
691 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
692 format
=> 'pve-volume-id',
693 format_description
=> 'volume',
698 description
=> 'Specify a custom file containing all network data passed to the VM via'
700 format
=> 'pve-volume-id',
701 format_description
=> 'volume',
706 description
=> 'Specify a custom file containing all user data passed to the VM via'
708 format
=> 'pve-volume-id',
709 format_description
=> 'volume',
712 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
714 my $confdesc_cloudinit = {
718 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
719 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
720 .' and `configdrive2` for windows.',
721 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
726 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
727 ." image's configured default user.",
732 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
733 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
734 .' support hashed passwords.',
739 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
741 format
=> 'pve-qm-cicustom',
746 description
=> "cloud-init: Sets DNS search domains for a container. Create will'
747 .' automatically use the setting from the host if neither searchdomain nor nameserver'
752 type
=> 'string', format
=> 'address-list',
753 description
=> "cloud-init: Sets DNS server IP address for a container. Create will'
754 .' automatically use the setting from the host if neither searchdomain nor nameserver'
760 format
=> 'urlencoded',
761 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
765 # what about other qemu settings ?
767 #machine => 'string',
780 ##soundhw => 'string',
782 while (my ($k, $v) = each %$confdesc) {
783 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
786 my $MAX_USB_DEVICES = 5;
788 my $MAX_SERIAL_PORTS = 4;
789 my $MAX_PARALLEL_PORTS = 3;
795 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
796 description
=> "CPUs accessing this NUMA node.",
797 format_description
=> "id[-id];...",
801 description
=> "Amount of memory this NUMA node provides.",
806 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
807 description
=> "Host NUMA nodes to use.",
808 format_description
=> "id[-id];...",
813 enum
=> [qw(preferred bind interleave)],
814 description
=> "NUMA allocation policy.",
818 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
821 type
=> 'string', format
=> $numa_fmt,
822 description
=> "NUMA topology.",
824 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
826 for (my $i = 0; $i < $MAX_NUMA; $i++) {
827 $confdesc->{"numa$i"} = $numadesc;
830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
831 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
832 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
835 my $net_fmt_bridge_descr = <<__EOD__;
836 Bridge to attach the network device to. The Proxmox VE standard bridge
839 If you do not specify a bridge, we create a kvm user (NATed) network
840 device, which provides DHCP and DNS services. The following addresses
847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
851 macaddr
=> get_standard_option
('mac-addr', {
852 description
=> "MAC address. That address must be unique withing your network. This is"
853 ." automatically generated if not specified.",
857 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
858 ." very low CPU overhead. If your guest does not support this driver, it is usually"
859 ." best to use 'e1000'.",
860 enum
=> $nic_model_list,
863 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
866 description
=> $net_fmt_bridge_descr,
867 format_description
=> 'bridge',
868 pattern
=> '[-_.\w\d]+',
873 minimum
=> 0, maximum
=> 16,
874 description
=> 'Number of packet queues to be used on the device.',
880 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
885 minimum
=> 1, maximum
=> 4094,
886 description
=> 'VLAN tag to apply to packets on this interface.',
891 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
892 description
=> 'VLAN trunks to pass through this interface.',
893 format_description
=> 'vlanid[;vlanid...]',
898 description
=> 'Whether this interface should be protected by the firewall.',
903 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
908 minimum
=> 1, maximum
=> 65520,
909 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
916 type
=> 'string', format
=> $net_fmt,
917 description
=> "Specify network devices.",
920 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
925 format
=> 'pve-ipv4-config',
926 format_description
=> 'IPv4Format/CIDR',
927 description
=> 'IPv4 address in CIDR format.',
934 format_description
=> 'GatewayIPv4',
935 description
=> 'Default gateway for IPv4 traffic.',
941 format
=> 'pve-ipv6-config',
942 format_description
=> 'IPv6Format/CIDR',
943 description
=> 'IPv6 address in CIDR format.',
950 format_description
=> 'GatewayIPv6',
951 description
=> 'Default gateway for IPv6 traffic.',
956 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
959 type
=> 'string', format
=> 'pve-qm-ipconfig',
960 description
=> <<'EODESCR',
961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
966 gateway should be provided.
967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
973 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
975 for (my $i = 0; $i < $MAX_NETS; $i++) {
976 $confdesc->{"net$i"} = $netdesc;
977 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
980 foreach my $key (keys %$confdesc_cloudinit) {
981 $confdesc->{$key} = $confdesc_cloudinit->{$key};
984 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
985 sub verify_volume_id_or_qm_path
{
986 my ($volid, $noerr) = @_;
988 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
992 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
993 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1004 type
=> 'string', format
=> 'pve-qm-usb-device',
1005 format_description
=> 'HOSTUSBDEVICE|spice',
1006 description
=> <<EODESCR,
1007 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1009 'bus-port(.port)*' (decimal numbers) or
1010 'vendor_id:product_id' (hexadeciaml numbers) or
1013 You can use the 'lsusb -t' command to list existing usb devices.
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1016 machines - use with special care.
1018 The value 'spice' can be used to add a usb redirection devices for spice.
1024 description
=> "Specifies whether if given host option is a USB3 device or port.",
1031 type
=> 'string', format
=> $usb_fmt,
1032 description
=> "Configure an USB device (n is 0 to 4).",
1034 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1039 pattern
=> '(/dev/.+|socket)',
1040 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1041 verbose_description
=> <<EODESCR,
1042 Create a serial device inside the VM (n is 0 to 3), and pass through a
1043 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1044 host side (use 'qm terminal' to open a terminal connection).
1046 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1047 use with special care.
1049 CAUTION: Experimental! User reported problems with this option.
1056 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1057 description
=> "Map host parallel devices (n is 0 to 2).",
1058 verbose_description
=> <<EODESCR,
1059 Map host parallel devices (n is 0 to 2).
1061 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1062 machines - use with special care.
1064 CAUTION: Experimental! User reported problems with this option.
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1069 $confdesc->{"parallel$i"} = $paralleldesc;
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1073 $confdesc->{"serial$i"} = $serialdesc;
1076 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1077 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1080 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1081 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1084 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1085 $confdesc->{"usb$i"} = $usbdesc;
1093 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1094 . " Deprecated, use 'order=' instead.",
1095 pattern
=> '[acdn]{1,4}',
1096 format_description
=> "[acdn]{1,4}",
1098 # note: this is also the fallback if boot: is not given at all
1104 format
=> 'pve-qm-bootdev-list',
1105 format_description
=> "device[;device...]",
1106 description
=> <<EODESC,
1107 The guest will attempt to boot from devices in the order they appear here.
1109 Disks, optical drives and passed-through storage USB devices will be directly
1110 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1111 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1113 Note that only devices in this list will be marked as bootable and thus loaded
1114 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1115 (e.g. software-raid), you need to specify all of them here.
1117 Overrides the deprecated 'legacy=[acdn]*' value when given.
1121 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1123 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1124 sub verify_bootdev
{
1125 my ($dev, $noerr) = @_;
1127 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && $dev !~ m/^efidisk/;
1131 return 0 if $dev !~ m/^$base\d+$/;
1132 return 0 if !$confdesc->{$dev};
1136 return $dev if $check->("net");
1137 return $dev if $check->("usb");
1138 return $dev if $check->("hostpci");
1141 die "invalid boot device '$dev'\n";
1144 sub print_bootorder
{
1146 return "" if !@$devs;
1147 my $data = { order
=> join(';', @$devs) };
1148 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1151 my $kvm_api_version = 0;
1154 return $kvm_api_version if $kvm_api_version;
1156 open my $fh, '<', '/dev/kvm' or return;
1158 # 0xae00 => KVM_GET_API_VERSION
1159 $kvm_api_version = ioctl($fh, 0xae00, 0);
1162 return $kvm_api_version;
1165 my $kvm_user_version = {};
1168 sub kvm_user_version
{
1171 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1172 my $st = stat($binary);
1174 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1175 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1176 $cachedmtime == $st->mtime;
1178 $kvm_user_version->{$binary} = 'unknown';
1179 $kvm_mtime->{$binary} = $st->mtime;
1183 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1184 $kvm_user_version->{$binary} = $2;
1188 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1191 return $kvm_user_version->{$binary};
1194 my sub extract_version
{
1195 my ($machine_type, $version) = @_;
1196 $version = kvm_user_version
() if !defined($version);
1197 PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1200 sub kernel_has_vhost_net
{
1201 return -c
'/dev/vhost-net';
1206 return defined($confdesc->{$key});
1210 sub get_cdrom_path
{
1212 return $cdrom_path if $cdrom_path;
1214 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1215 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1216 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1220 my ($storecfg, $vmid, $cdrom) = @_;
1222 if ($cdrom eq 'cdrom') {
1223 return get_cdrom_path
();
1224 } elsif ($cdrom eq 'none') {
1226 } elsif ($cdrom =~ m
|^/|) {
1229 return PVE
::Storage
::path
($storecfg, $cdrom);
1233 # try to convert old style file names to volume IDs
1234 sub filename_to_volume_id
{
1235 my ($vmid, $file, $media) = @_;
1237 if (!($file eq 'none' || $file eq 'cdrom' ||
1238 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1240 return if $file =~ m
|/|;
1242 if ($media && $media eq 'cdrom') {
1243 $file = "local:iso/$file";
1245 $file = "local:$vmid/$file";
1252 sub verify_media_type
{
1253 my ($opt, $vtype, $media) = @_;
1258 if ($media eq 'disk') {
1260 } elsif ($media eq 'cdrom') {
1263 die "internal error";
1266 return if ($vtype eq $etype);
1268 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1271 sub cleanup_drive_path
{
1272 my ($opt, $storecfg, $drive) = @_;
1274 # try to convert filesystem paths to volume IDs
1276 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1277 ($drive->{file
} !~ m
|^/dev/.+|) &&
1278 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1279 ($drive->{file
} !~ m/^\d+$/)) {
1280 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1281 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1283 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1284 verify_media_type
($opt, $vtype, $drive->{media
});
1285 $drive->{file
} = $volid;
1288 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1291 sub parse_hotplug_features
{
1296 return $res if $data eq '0';
1298 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1300 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1301 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1304 die "invalid hotplug feature '$feature'\n";
1310 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1311 sub pve_verify_hotplug_features
{
1312 my ($value, $noerr) = @_;
1314 return $value if parse_hotplug_features
($value);
1318 die "unable to parse hotplug option\n";
1322 my($fh, $noerr) = @_;
1325 my $SG_GET_VERSION_NUM = 0x2282;
1327 my $versionbuf = "\x00" x
8;
1328 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1330 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1333 my $version = unpack("I", $versionbuf);
1334 if ($version < 30000) {
1335 die "scsi generic interface too old\n" if !$noerr;
1339 my $buf = "\x00" x
36;
1340 my $sensebuf = "\x00" x
8;
1341 my $cmd = pack("C x3 C x1", 0x12, 36);
1343 # see /usr/include/scsi/sg.h
1344 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1346 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1347 length($sensebuf), 0, length($buf), $buf,
1348 $cmd, $sensebuf, 6000);
1350 $ret = ioctl($fh, $SG_IO, $packet);
1352 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1356 my @res = unpack($sg_io_hdr_t, $packet);
1357 if ($res[17] || $res[18]) {
1358 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363 (my $byte0, my $byte1, $res->{vendor
},
1364 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1366 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1367 $res->{type
} = $byte0 & 31;
1375 my $fh = IO
::File-
>new("+<$path") || return;
1376 my $res = scsi_inquiry
($fh, 1);
1382 sub print_tabletdevice_full
{
1383 my ($conf, $arch) = @_;
1385 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1387 # we use uhci for old VMs because tablet driver was buggy in older qemu
1389 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1395 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1398 sub print_keyboarddevice_full
{
1399 my ($conf, $arch, $machine) = @_;
1401 return if $arch ne 'aarch64';
1403 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1406 sub print_drivedevice_full
{
1407 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1412 my $drive_id = "$drive->{interface}$drive->{index}";
1413 if ($drive->{interface
} eq 'virtio') {
1414 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1415 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1416 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1417 } elsif ($drive->{interface
} eq 'scsi') {
1419 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1420 my $unit = $drive->{index} % $maxdev;
1421 my $devicetype = 'hd';
1423 if (drive_is_cdrom
($drive)) {
1426 if ($drive->{file
} =~ m
|^/|) {
1427 $path = $drive->{file
};
1428 if (my $info = path_is_scsi
($path)) {
1429 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1430 $devicetype = 'block';
1431 } elsif ($info->{type
} == 1) { # tape
1432 $devicetype = 'generic';
1436 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1439 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1440 my $version = extract_version
($machine_type, kvm_user_version
());
1441 if ($path =~ m/^iscsi\:\/\
// &&
1442 !min_version
($version, 4, 1)) {
1443 $devicetype = 'generic';
1447 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1448 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1450 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1451 .",lun=$drive->{index}";
1453 $device .= ",drive=drive-$drive_id,id=$drive_id";
1455 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1456 $device .= ",rotation_rate=1";
1458 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1460 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1461 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1462 my $controller = int($drive->{index} / $maxdev);
1463 my $unit = $drive->{index} % $maxdev;
1464 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1466 $device = "ide-$devicetype";
1467 if ($drive->{interface
} eq 'ide') {
1468 $device .= ",bus=ide.$controller,unit=$unit";
1470 $device .= ",bus=ahci$controller.$unit";
1472 $device .= ",drive=drive-$drive_id,id=$drive_id";
1474 if ($devicetype eq 'hd') {
1475 if (my $model = $drive->{model
}) {
1476 $model = URI
::Escape
::uri_unescape
($model);
1477 $device .= ",model=$model";
1479 if ($drive->{ssd
}) {
1480 $device .= ",rotation_rate=1";
1483 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1484 } elsif ($drive->{interface
} eq 'usb') {
1486 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1488 die "unsupported interface type";
1491 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1493 if (my $serial = $drive->{serial
}) {
1494 $serial = URI
::Escape
::uri_unescape
($serial);
1495 $device .= ",serial=$serial";
1502 sub get_initiator_name
{
1505 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1506 while (defined(my $line = <$fh>)) {
1507 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1516 sub print_drive_commandline_full
{
1517 my ($storecfg, $vmid, $drive) = @_;
1520 my $volid = $drive->{file
};
1523 if (drive_is_cdrom
($drive)) {
1524 $path = get_iso_path
($storecfg, $vmid, $volid);
1526 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1528 $path = PVE
::Storage
::path
($storecfg, $volid);
1529 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1530 $format = qemu_img_format
($scfg, $volname);
1538 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1539 foreach my $o (@qemu_drive_options) {
1540 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1543 # snapshot only accepts on|off
1544 if (defined($drive->{snapshot
})) {
1545 my $v = $drive->{snapshot
} ?
'on' : 'off';
1546 $opts .= ",snapshot=$v";
1549 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1550 my ($dir, $qmpname) = @$type;
1551 if (my $v = $drive->{"mbps$dir"}) {
1552 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1554 if (my $v = $drive->{"mbps${dir}_max"}) {
1555 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1557 if (my $v = $drive->{"bps${dir}_max_length"}) {
1558 $opts .= ",throttling.bps$qmpname-max-length=$v";
1560 if (my $v = $drive->{"iops${dir}"}) {
1561 $opts .= ",throttling.iops$qmpname=$v";
1563 if (my $v = $drive->{"iops${dir}_max"}) {
1564 $opts .= ",throttling.iops$qmpname-max=$v";
1566 if (my $v = $drive->{"iops${dir}_max_length"}) {
1567 $opts .= ",throttling.iops$qmpname-max-length=$v";
1571 $opts .= ",format=$format" if $format && !$drive->{format
};
1573 my $cache_direct = 0;
1575 if (my $cache = $drive->{cache
}) {
1576 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1577 } elsif (!drive_is_cdrom
($drive)) {
1578 $opts .= ",cache=none";
1582 # aio native works only with O_DIRECT
1583 if (!$drive->{aio
}) {
1585 $opts .= ",aio=native";
1587 $opts .= ",aio=threads";
1591 if (!drive_is_cdrom
($drive)) {
1593 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1594 $detectzeroes = 'off';
1595 } elsif ($drive->{discard
}) {
1596 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1598 # This used to be our default with discard not being specified:
1599 $detectzeroes = 'on';
1601 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1604 my $pathinfo = $path ?
"file=$path," : '';
1606 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1609 sub print_netdevice_full
{
1610 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1612 my $device = $net->{model
};
1613 if ($net->{model
} eq 'virtio') {
1614 $device = 'virtio-net-pci';
1617 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1618 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1619 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1620 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1621 # and out of each queue plus one config interrupt and control vector queue
1622 my $vectors = $net->{queues
} * 2 + 2;
1623 $tmpstr .= ",vectors=$vectors,mq=on";
1625 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1627 if (my $mtu = $net->{mtu
}) {
1628 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1629 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1632 } elsif ($mtu < 576) {
1633 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1634 } elsif ($mtu > $bridge_mtu) {
1635 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1637 $tmpstr .= ",host_mtu=$mtu";
1639 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1643 if ($use_old_bios_files) {
1645 if ($device eq 'virtio-net-pci') {
1646 $romfile = 'pxe-virtio.rom';
1647 } elsif ($device eq 'e1000') {
1648 $romfile = 'pxe-e1000.rom';
1649 } elsif ($device eq 'ne2k') {
1650 $romfile = 'pxe-ne2k_pci.rom';
1651 } elsif ($device eq 'pcnet') {
1652 $romfile = 'pxe-pcnet.rom';
1653 } elsif ($device eq 'rtl8139') {
1654 $romfile = 'pxe-rtl8139.rom';
1656 $tmpstr .= ",romfile=$romfile" if $romfile;
1662 sub print_netdev_full
{
1663 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1666 if ($netid =~ m/^net(\d+)$/) {
1670 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1672 my $ifname = "tap${vmid}i$i";
1674 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1675 die "interface name '$ifname' is too long (max 15 character)\n"
1676 if length($ifname) >= 16;
1678 my $vhostparam = '';
1679 if (is_native
($arch)) {
1680 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1683 my $vmname = $conf->{name
} || "vm$vmid";
1686 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1688 if ($net->{bridge
}) {
1689 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1690 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1692 $netdev = "type=user,id=$netid,hostname=$vmname";
1695 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1701 'cirrus' => 'cirrus-vga',
1703 'vmware' => 'vmware-svga',
1704 'virtio' => 'virtio-vga',
1707 sub print_vga_device
{
1708 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1710 my $type = $vga_map->{$vga->{type
}};
1711 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1712 $type = 'virtio-gpu';
1714 my $vgamem_mb = $vga->{memory
};
1716 my $max_outputs = '';
1718 $type = $id ?
'qxl' : 'qxl-vga';
1720 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1721 # set max outputs so linux can have up to 4 qxl displays with one device
1722 if (min_version
($machine_version, 4, 1)) {
1723 $max_outputs = ",max_outputs=4";
1728 die "no devicetype for $vga->{type}\n" if !$type;
1732 if ($vga->{type
} eq 'virtio') {
1733 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1734 $memory = ",max_hostmem=$bytes";
1736 # from https://www.spice-space.org/multiple-monitors.html
1737 $memory = ",vgamem_mb=$vga->{memory}";
1738 my $ram = $vgamem_mb * 4;
1739 my $vram = $vgamem_mb * 2;
1740 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1742 $memory = ",vgamem_mb=$vga->{memory}";
1744 } elsif ($qxlnum && $id) {
1745 $memory = ",ram_size=67108864,vram_size=33554432";
1749 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1750 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1753 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1754 my $vgaid = "vga" . ($id // '');
1757 if ($q35 && $vgaid eq 'vga') {
1758 # the first display uses pcie.0 bus on q35 machines
1759 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1761 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1764 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1767 sub parse_number_sets
{
1770 foreach my $part (split(/;/, $set)) {
1771 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1772 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1773 push @$res, [ $1, $2 ];
1775 die "invalid range: $part\n";
1784 my $res = parse_property_string
($numa_fmt, $data);
1785 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1786 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1790 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1794 my $res = eval { parse_property_string
($net_fmt, $data) };
1799 if (!defined($res->{macaddr
})) {
1800 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1801 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1806 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1807 sub parse_ipconfig
{
1810 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1816 if ($res->{gw
} && !$res->{ip
}) {
1817 warn 'gateway specified without specifying an IP address';
1820 if ($res->{gw6
} && !$res->{ip6
}) {
1821 warn 'IPv6 gateway specified without specifying an IPv6 address';
1824 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1825 warn 'gateway specified together with DHCP';
1828 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1830 warn "IPv6 gateway specified together with $res->{ip6} address";
1834 if (!$res->{ip
} && !$res->{ip6
}) {
1835 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1844 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1847 sub add_random_macs
{
1848 my ($settings) = @_;
1850 foreach my $opt (keys %$settings) {
1851 next if $opt !~ m/^net(\d+)$/;
1852 my $net = parse_net
($settings->{$opt});
1854 $settings->{$opt} = print_net
($net);
1858 sub vm_is_volid_owner
{
1859 my ($storecfg, $vmid, $volid) = @_;
1861 if ($volid !~ m
|^/|) {
1863 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1864 if ($owner && ($owner == $vmid)) {
1872 sub vmconfig_register_unused_drive
{
1873 my ($storecfg, $vmid, $conf, $drive) = @_;
1875 if (drive_is_cloudinit
($drive)) {
1876 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1878 } elsif (!drive_is_cdrom
($drive)) {
1879 my $volid = $drive->{file
};
1880 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1881 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1886 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1890 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1891 format_description
=> 'UUID',
1892 description
=> "Set SMBIOS1 UUID.",
1897 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1898 format_description
=> 'Base64 encoded string',
1899 description
=> "Set SMBIOS1 version.",
1904 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1905 format_description
=> 'Base64 encoded string',
1906 description
=> "Set SMBIOS1 serial number.",
1911 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1912 format_description
=> 'Base64 encoded string',
1913 description
=> "Set SMBIOS1 manufacturer.",
1918 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1919 format_description
=> 'Base64 encoded string',
1920 description
=> "Set SMBIOS1 product ID.",
1925 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1926 format_description
=> 'Base64 encoded string',
1927 description
=> "Set SMBIOS1 SKU string.",
1932 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1933 format_description
=> 'Base64 encoded string',
1934 description
=> "Set SMBIOS1 family string.",
1939 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
1947 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
1954 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
1957 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
1959 sub parse_watchdog
{
1964 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
1969 sub parse_guest_agent
{
1972 return {} if !defined($conf->{agent
});
1974 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
1977 # if the agent is disabled ignore the other potentially set properties
1978 return {} if !$res->{enabled
};
1983 my ($conf, $key) = @_;
1984 return undef if !defined($conf->{agent
});
1986 my $agent = parse_guest_agent
($conf);
1987 return $agent->{$key};
1993 return {} if !$value;
1994 my $res = eval { parse_property_string
($vga_fmt, $value) };
2004 my $res = eval { parse_property_string
($rng_fmt, $value) };
2009 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2010 sub verify_usb_device
{
2011 my ($value, $noerr) = @_;
2013 return $value if parse_usb_device
($value);
2017 die "unable to parse usb device\n";
2020 # add JSON properties for create and set function
2021 sub json_config_properties
{
2024 foreach my $opt (keys %$confdesc) {
2025 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2026 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2027 $prop->{$opt} = $confdesc->{$opt};
2033 # return copy of $confdesc_cloudinit to generate documentation
2034 sub cloudinit_config_properties
{
2036 return dclone
($confdesc_cloudinit);
2040 my ($key, $value) = @_;
2042 die "unknown setting '$key'\n" if !$confdesc->{$key};
2044 my $type = $confdesc->{$key}->{type
};
2046 if (!defined($value)) {
2047 die "got undefined value\n";
2050 if ($value =~ m/[\n\r]/) {
2051 die "property contains a line feed\n";
2054 if ($type eq 'boolean') {
2055 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2056 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2057 die "type check ('boolean') failed - got '$value'\n";
2058 } elsif ($type eq 'integer') {
2059 return int($1) if $value =~ m/^(\d+)$/;
2060 die "type check ('integer') failed - got '$value'\n";
2061 } elsif ($type eq 'number') {
2062 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2063 die "type check ('number') failed - got '$value'\n";
2064 } elsif ($type eq 'string') {
2065 if (my $fmt = $confdesc->{$key}->{format
}) {
2066 PVE
::JSONSchema
::check_format
($fmt, $value);
2069 $value =~ s/^\"(.*)\"$/$1/;
2072 die "internal error"
2077 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2079 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2081 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2083 if ($conf->{template
}) {
2084 # check if any base image is still used by a linked clone
2085 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2086 my ($ds, $drive) = @_;
2087 return if drive_is_cdrom
($drive);
2089 my $volid = $drive->{file
};
2090 return if !$volid || $volid =~ m
|^/|;
2092 die "base volume '$volid' is still in use by linked cloned\n"
2093 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2098 # only remove disks owned by this VM (referenced in the config)
2099 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2100 my ($ds, $drive) = @_;
2101 return if drive_is_cdrom
($drive, 1);
2103 my $volid = $drive->{file
};
2104 return if !$volid || $volid =~ m
|^/|;
2106 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2107 return if !$path || !$owner || ($owner != $vmid);
2109 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2110 warn "Could not remove disk '$volid', check manually: $@" if $@;
2113 if ($purge_unreferenced) { # also remove unreferenced disk
2114 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2115 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2116 my ($volid, $sid, $volname, $d) = @_;
2117 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2122 if (defined $replacement_conf) {
2123 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2125 PVE
::QemuConfig-
>destroy_config($vmid);
2129 sub parse_vm_config
{
2130 my ($filename, $raw) = @_;
2132 return if !defined($raw);
2135 digest
=> Digest
::SHA
::sha1_hex
($raw),
2140 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2141 || die "got strange filename '$filename'";
2149 my @lines = split(/\n/, $raw);
2150 foreach my $line (@lines) {
2151 next if $line =~ m/^\s*$/;
2153 if ($line =~ m/^\[PENDING\]\s*$/i) {
2154 $section = 'pending';
2155 if (defined($descr)) {
2157 $conf->{description
} = $descr;
2160 $conf = $res->{$section} = {};
2163 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2165 if (defined($descr)) {
2167 $conf->{description
} = $descr;
2170 $conf = $res->{snapshots
}->{$section} = {};
2174 if ($line =~ m/^\#(.*)\s*$/) {
2175 $descr = '' if !defined($descr);
2176 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2180 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2181 $descr = '' if !defined($descr);
2182 $descr .= PVE
::Tools
::decode_text
($2);
2183 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2184 $conf->{snapstate
} = $1;
2185 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2188 $conf->{$key} = $value;
2189 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2191 if ($section eq 'pending') {
2192 $conf->{delete} = $value; # we parse this later
2194 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2196 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2199 eval { $value = check_type
($key, $value); };
2201 warn "vm $vmid - unable to parse value of '$key' - $@";
2203 $key = 'ide2' if $key eq 'cdrom';
2204 my $fmt = $confdesc->{$key}->{format
};
2205 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2206 my $v = parse_drive
($key, $value);
2207 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2208 $v->{file
} = $volid;
2209 $value = print_drive
($v);
2211 warn "vm $vmid - unable to parse value of '$key'\n";
2216 $conf->{$key} = $value;
2221 if (defined($descr)) {
2223 $conf->{description
} = $descr;
2225 delete $res->{snapstate
}; # just to be sure
2230 sub write_vm_config
{
2231 my ($filename, $conf) = @_;
2233 delete $conf->{snapstate
}; # just to be sure
2235 if ($conf->{cdrom
}) {
2236 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2237 $conf->{ide2
} = $conf->{cdrom
};
2238 delete $conf->{cdrom
};
2241 # we do not use 'smp' any longer
2242 if ($conf->{sockets
}) {
2243 delete $conf->{smp
};
2244 } elsif ($conf->{smp
}) {
2245 $conf->{sockets
} = $conf->{smp
};
2246 delete $conf->{cores
};
2247 delete $conf->{smp
};
2250 my $used_volids = {};
2252 my $cleanup_config = sub {
2253 my ($cref, $pending, $snapname) = @_;
2255 foreach my $key (keys %$cref) {
2256 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2257 $key eq 'snapstate' || $key eq 'pending';
2258 my $value = $cref->{$key};
2259 if ($key eq 'delete') {
2260 die "propertry 'delete' is only allowed in [PENDING]\n"
2262 # fixme: check syntax?
2265 eval { $value = check_type
($key, $value); };
2266 die "unable to parse value of '$key' - $@" if $@;
2268 $cref->{$key} = $value;
2270 if (!$snapname && is_valid_drivename
($key)) {
2271 my $drive = parse_drive
($key, $value);
2272 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2277 &$cleanup_config($conf);
2279 &$cleanup_config($conf->{pending
}, 1);
2281 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2282 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2283 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2286 # remove 'unusedX' settings if we re-add a volume
2287 foreach my $key (keys %$conf) {
2288 my $value = $conf->{$key};
2289 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2290 delete $conf->{$key};
2294 my $generate_raw_config = sub {
2295 my ($conf, $pending) = @_;
2299 # add description as comment to top of file
2300 if (defined(my $descr = $conf->{description
})) {
2302 foreach my $cl (split(/\n/, $descr)) {
2303 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2306 $raw .= "#\n" if $pending;
2310 foreach my $key (sort keys %$conf) {
2311 next if $key =~ /^(digest|description|pending|snapshots)$/;
2312 $raw .= "$key: $conf->{$key}\n";
2317 my $raw = &$generate_raw_config($conf);
2319 if (scalar(keys %{$conf->{pending
}})){
2320 $raw .= "\n[PENDING]\n";
2321 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2324 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2325 $raw .= "\n[$snapname]\n";
2326 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2336 # we use static defaults from our JSON schema configuration
2337 foreach my $key (keys %$confdesc) {
2338 if (defined(my $default = $confdesc->{$key}->{default})) {
2339 $res->{$key} = $default;
2347 my $vmlist = PVE
::Cluster
::get_vmlist
();
2349 return $res if !$vmlist || !$vmlist->{ids
};
2350 my $ids = $vmlist->{ids
};
2351 my $nodename = nodename
();
2353 foreach my $vmid (keys %$ids) {
2354 my $d = $ids->{$vmid};
2355 next if !$d->{node
} || $d->{node
} ne $nodename;
2356 next if !$d->{type
} || $d->{type
} ne 'qemu';
2357 $res->{$vmid}->{exists} = 1;
2362 # test if VM uses local resources (to prevent migration)
2363 sub check_local_resources
{
2364 my ($conf, $noerr) = @_;
2368 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2369 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2371 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2373 foreach my $k (keys %$conf) {
2374 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2375 # sockets are safe: they will recreated be on the target side post-migrate
2376 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2377 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2380 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2385 # check if used storages are available on all nodes (use by migrate)
2386 sub check_storage_availability
{
2387 my ($storecfg, $conf, $node) = @_;
2389 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2390 my ($ds, $drive) = @_;
2392 my $volid = $drive->{file
};
2395 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2398 # check if storage is available on both nodes
2399 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2400 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2404 # list nodes where all VM images are available (used by has_feature API)
2406 my ($conf, $storecfg) = @_;
2408 my $nodelist = PVE
::Cluster
::get_nodelist
();
2409 my $nodehash = { map { $_ => 1 } @$nodelist };
2410 my $nodename = nodename
();
2412 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2413 my ($ds, $drive) = @_;
2415 my $volid = $drive->{file
};
2418 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2420 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2421 if ($scfg->{disable
}) {
2423 } elsif (my $avail = $scfg->{nodes
}) {
2424 foreach my $node (keys %$nodehash) {
2425 delete $nodehash->{$node} if !$avail->{$node};
2427 } elsif (!$scfg->{shared
}) {
2428 foreach my $node (keys %$nodehash) {
2429 delete $nodehash->{$node} if $node ne $nodename
2438 sub check_local_storage_availability
{
2439 my ($conf, $storecfg) = @_;
2441 my $nodelist = PVE
::Cluster
::get_nodelist
();
2442 my $nodehash = { map { $_ => {} } @$nodelist };
2444 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2445 my ($ds, $drive) = @_;
2447 my $volid = $drive->{file
};
2450 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2452 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2454 if ($scfg->{disable
}) {
2455 foreach my $node (keys %$nodehash) {
2456 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2458 } elsif (my $avail = $scfg->{nodes
}) {
2459 foreach my $node (keys %$nodehash) {
2460 if (!$avail->{$node}) {
2461 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2468 foreach my $node (values %$nodehash) {
2469 if (my $unavail = $node->{unavailable_storages
}) {
2470 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2477 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2479 my ($vmid, $nocheck, $node) = @_;
2481 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2482 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2487 my $vzlist = config_list
();
2489 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2491 while (defined(my $de = $fd->read)) {
2492 next if $de !~ m/^(\d+)\.pid$/;
2494 next if !defined($vzlist->{$vmid});
2495 if (my $pid = check_running
($vmid)) {
2496 $vzlist->{$vmid}->{pid
} = $pid;
2503 our $vmstatus_return_properties = {
2504 vmid
=> get_standard_option
('pve-vmid'),
2506 description
=> "Qemu process status.",
2508 enum
=> ['stopped', 'running'],
2511 description
=> "Maximum memory in bytes.",
2514 renderer
=> 'bytes',
2517 description
=> "Root disk size in bytes.",
2520 renderer
=> 'bytes',
2523 description
=> "VM name.",
2528 description
=> "Qemu QMP agent status.",
2533 description
=> "PID of running qemu process.",
2538 description
=> "Uptime.",
2541 renderer
=> 'duration',
2544 description
=> "Maximum usable CPUs.",
2549 description
=> "The current config lock, if any.",
2554 description
=> "The current configured tags, if any",
2560 my $last_proc_pid_stat;
2562 # get VM status information
2563 # This must be fast and should not block ($full == false)
2564 # We only query KVM using QMP if $full == true (this can be slow)
2566 my ($opt_vmid, $full) = @_;
2570 my $storecfg = PVE
::Storage
::config
();
2572 my $list = vzlist
();
2573 my $defaults = load_defaults
();
2575 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2577 my $cpucount = $cpuinfo->{cpus
} || 1;
2579 foreach my $vmid (keys %$list) {
2580 next if $opt_vmid && ($vmid ne $opt_vmid);
2582 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2584 my $d = { vmid
=> $vmid };
2585 $d->{pid
} = $list->{$vmid}->{pid
};
2587 # fixme: better status?
2588 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2590 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2591 if (defined($size)) {
2592 $d->{disk
} = 0; # no info available
2593 $d->{maxdisk
} = $size;
2599 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2600 * ($conf->{cores
} || $defaults->{cores
});
2601 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2602 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2604 $d->{name
} = $conf->{name
} || "VM $vmid";
2605 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2606 : $defaults->{memory
}*(1024*1024);
2608 if ($conf->{balloon
}) {
2609 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2610 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2611 : $defaults->{shares
};
2622 $d->{diskwrite
} = 0;
2624 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
2626 $d->{serial
} = 1 if conf_has_serial
($conf);
2627 $d->{lock} = $conf->{lock} if $conf->{lock};
2628 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2633 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2634 foreach my $dev (keys %$netdev) {
2635 next if $dev !~ m/^tap([1-9]\d*)i/;
2637 my $d = $res->{$vmid};
2640 $d->{netout
} += $netdev->{$dev}->{receive
};
2641 $d->{netin
} += $netdev->{$dev}->{transmit
};
2644 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
2645 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
2650 my $ctime = gettimeofday
;
2652 foreach my $vmid (keys %$list) {
2654 my $d = $res->{$vmid};
2655 my $pid = $d->{pid
};
2658 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2659 next if !$pstat; # not running
2661 my $used = $pstat->{utime} + $pstat->{stime
};
2663 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2665 if ($pstat->{vsize
}) {
2666 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2669 my $old = $last_proc_pid_stat->{$pid};
2671 $last_proc_pid_stat->{$pid} = {
2679 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2681 if ($dtime > 1000) {
2682 my $dutime = $used - $old->{used
};
2684 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2685 $last_proc_pid_stat->{$pid} = {
2691 $d->{cpu
} = $old->{cpu
};
2695 return $res if !$full;
2697 my $qmpclient = PVE
::QMPClient-
>new();
2699 my $ballooncb = sub {
2700 my ($vmid, $resp) = @_;
2702 my $info = $resp->{'return'};
2703 return if !$info->{max_mem
};
2705 my $d = $res->{$vmid};
2707 # use memory assigned to VM
2708 $d->{maxmem
} = $info->{max_mem
};
2709 $d->{balloon
} = $info->{actual
};
2711 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2712 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2713 $d->{freemem
} = $info->{free_mem
};
2716 $d->{ballooninfo
} = $info;
2719 my $blockstatscb = sub {
2720 my ($vmid, $resp) = @_;
2721 my $data = $resp->{'return'} || [];
2722 my $totalrdbytes = 0;
2723 my $totalwrbytes = 0;
2725 for my $blockstat (@$data) {
2726 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2727 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2729 $blockstat->{device
} =~ s/drive-//;
2730 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2732 $res->{$vmid}->{diskread
} = $totalrdbytes;
2733 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2736 my $statuscb = sub {
2737 my ($vmid, $resp) = @_;
2739 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2740 # this fails if ballon driver is not loaded, so this must be
2741 # the last commnand (following command are aborted if this fails).
2742 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2744 my $status = 'unknown';
2745 if (!defined($status = $resp->{'return'}->{status
})) {
2746 warn "unable to get VM status\n";
2750 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2753 foreach my $vmid (keys %$list) {
2754 next if $opt_vmid && ($vmid ne $opt_vmid);
2755 next if !$res->{$vmid}->{pid
}; # not running
2756 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2759 $qmpclient->queue_execute(undef, 2);
2761 foreach my $vmid (keys %$list) {
2762 next if $opt_vmid && ($vmid ne $opt_vmid);
2763 next if !$res->{$vmid}->{pid
}; #not running
2765 # we can't use the $qmpclient since it might have already aborted on
2766 # 'query-balloon', but this might also fail for older versions...
2767 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
2768 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2771 foreach my $vmid (keys %$list) {
2772 next if $opt_vmid && ($vmid ne $opt_vmid);
2773 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2779 sub conf_has_serial
{
2782 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2783 if ($conf->{"serial$i"}) {
2791 sub conf_has_audio
{
2792 my ($conf, $id) = @_;
2795 my $audio = $conf->{"audio$id"};
2796 return if !defined($audio);
2798 my $audioproperties = parse_property_string
($audio_fmt, $audio);
2799 my $audiodriver = $audioproperties->{driver
} // 'spice';
2802 dev
=> $audioproperties->{device
},
2803 dev_id
=> "audiodev$id",
2804 backend
=> $audiodriver,
2805 backend_id
=> "$audiodriver-backend${id}",
2810 my ($audio, $audiopciaddr, $machine_version) = @_;
2814 my $id = $audio->{dev_id
};
2816 if (min_version
($machine_version, 4, 2)) {
2817 $audiodev = ",audiodev=$audio->{backend_id}";
2820 if ($audio->{dev
} eq 'AC97') {
2821 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2822 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
2823 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2824 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2825 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2827 die "unkown audio device '$audio->{dev}', implement me!";
2830 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2835 sub vga_conf_has_spice
{
2838 my $vgaconf = parse_vga
($vga);
2839 my $vgatype = $vgaconf->{type
};
2840 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2847 return get_host_arch
() eq $arch;
2852 return $conf->{arch
} // get_host_arch
();
2855 my $default_machines = {
2860 sub get_vm_machine
{
2861 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2863 my $machine = $forcemachine || $conf->{machine
};
2865 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2867 $machine ||= $default_machines->{$arch};
2868 if ($add_pve_version) {
2869 $kvmversion //= kvm_user_version
();
2870 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
2871 $machine .= "+pve$pvever";
2875 if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2876 # for version-pinned machines that do not include a pve-version (e.g.
2877 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2878 $machine .= '+pve0';
2884 sub get_ovmf_files
($) {
2887 my $ovmf = $OVMF->{$arch}
2888 or die "no OVMF images known for architecture '$arch'\n";
2894 aarch64
=> '/usr/bin/qemu-system-aarch64',
2895 x86_64
=> '/usr/bin/qemu-system-x86_64',
2897 sub get_command_for_arch
($) {
2899 return '/usr/bin/kvm' if is_native
($arch);
2901 my $cmd = $Arch2Qemu->{$arch}
2902 or die "don't know how to emulate architecture '$arch'\n";
2906 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2907 # to use in a QEMU command line (-cpu element), first array_intersect the result
2908 # of query_supported_ with query_understood_. This is necessary because:
2910 # a) query_understood_ returns flags the host cannot use and
2911 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2912 # flags, but CPU settings - with most of them being flags. Those settings
2913 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
2915 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2916 # expensive. If you need the value returned from this, you can get it much
2917 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2918 # $accel being 'kvm' or 'tcg'.
2920 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2921 # changes, automatically populating pmxcfs.
2923 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2924 # since kvm and tcg machines support different flags
2926 sub query_supported_cpu_flags
{
2929 $arch //= get_host_arch
();
2930 my $default_machine = $default_machines->{$arch};
2934 # FIXME: Once this is merged, the code below should work for ARM as well:
2935 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2936 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2939 my $kvm_supported = defined(kvm_version
());
2940 my $qemu_cmd = get_command_for_arch
($arch);
2942 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
2944 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2945 my $query_supported_run_qemu = sub {
2951 '-machine', $default_machine,
2953 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2954 '-mon', 'chardev=qmp,mode=control',
2955 '-pidfile', $pidfile,
2960 push @$cmd, '-accel', 'tcg';
2963 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
2964 die "QEMU flag querying VM exited with code " . $rc if $rc;
2967 my $cmd_result = mon_cmd
(
2969 'query-cpu-model-expansion',
2971 model
=> { name
=> 'host' }
2974 my $props = $cmd_result->{model
}->{props
};
2975 foreach my $prop (keys %$props) {
2976 next if $props->{$prop} ne '1';
2977 # QEMU returns some flags multiple times, with '_', '.' or '-'
2978 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2979 # We only keep those with underscores, to match /proc/cpuinfo
2980 $prop =~ s/\.|-/_/g;
2981 $flags->{$prop} = 1;
2986 # force stop with 10 sec timeout and 'nocheck'
2987 # always stop, even if QMP failed
2988 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
2992 return [ sort keys %$flags ];
2995 # We need to query QEMU twice, since KVM and TCG have different supported flags
2996 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
2997 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
2998 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3000 if ($kvm_supported) {
3001 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3002 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3009 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3010 my $understood_cpu_flag_dir = "/usr/share/kvm";
3011 sub query_understood_cpu_flags
{
3012 my $arch = get_host_arch
();
3013 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3015 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3018 my $raw = file_get_contents
($filepath);
3019 $raw =~ s/^\s+|\s+$//g;
3020 my @flags = split(/\s+/, $raw);
3025 sub config_to_command
{
3026 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3029 my $globalFlags = [];
3030 my $machineFlags = [];
3035 my $ostype = $conf->{ostype
};
3036 my $winversion = windows_version
($ostype);
3037 my $kvm = $conf->{kvm
};
3038 my $nodename = nodename
();
3040 my $arch = get_vm_arch
($conf);
3041 my $kvm_binary = get_command_for_arch
($arch);
3042 my $kvmver = kvm_user_version
($kvm_binary);
3044 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3045 $kvmver //= "undefined";
3046 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3049 my $add_pve_version = min_version
($kvmver, 4, 1);
3051 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3052 my $machine_version = extract_version
($machine_type, $kvmver);
3053 $kvm //= 1 if is_native
($arch);
3055 $machine_version =~ m/(\d+)\.(\d+)/;
3056 my ($machine_major, $machine_minor) = ($1, $2);
3058 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3059 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3060 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3061 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3062 ." please upgrade node '$nodename'\n"
3063 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3064 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3065 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3066 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3067 ." node '$nodename'\n";
3070 # if a specific +pve version is required for a feature, use $version_guard
3071 # instead of min_version to allow machines to be run with the minimum
3073 my $required_pve_version = 0;
3074 my $version_guard = sub {
3075 my ($major, $minor, $pve) = @_;
3076 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3077 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3078 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3079 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3083 if ($kvm && !defined kvm_version
()) {
3084 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3085 ." or enable in BIOS.\n";
3088 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3089 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3090 my $use_old_bios_files = undef;
3091 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3093 my $cpuunits = defined($conf->{cpuunits
}) ?
3094 $conf->{cpuunits
} : $defaults->{cpuunits
};
3096 push @$cmd, $kvm_binary;
3098 push @$cmd, '-id', $vmid;
3100 my $vmname = $conf->{name
} || "vm$vmid";
3102 push @$cmd, '-name', $vmname;
3104 push @$cmd, '-no-shutdown';
3108 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3109 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3110 push @$cmd, '-mon', "chardev=qmp,mode=control";
3112 if (min_version
($machine_version, 2, 12)) {
3113 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3114 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3117 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3119 push @$cmd, '-daemonize';
3121 if ($conf->{smbios1
}) {
3122 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3123 if ($smbios_conf->{base64
}) {
3124 # Do not pass base64 flag to qemu
3125 delete $smbios_conf->{base64
};
3126 my $smbios_string = "";
3127 foreach my $key (keys %$smbios_conf) {
3129 if ($key eq "uuid") {
3130 $value = $smbios_conf->{uuid
}
3132 $value = decode_base64
($smbios_conf->{$key});
3134 # qemu accepts any binary data, only commas need escaping by double comma
3136 $smbios_string .= "," . $key . "=" . $value if $value;
3138 push @$cmd, '-smbios', "type=1" . $smbios_string;
3140 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3144 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3145 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3146 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3148 my ($path, $format);
3149 if (my $efidisk = $conf->{efidisk0
}) {
3150 my $d = parse_drive
('efidisk0', $efidisk);
3151 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3152 $format = $d->{format
};
3154 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3155 if (!defined($format)) {
3156 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3157 $format = qemu_img_format
($scfg, $volname);
3161 die "efidisk format must be specified\n"
3162 if !defined($format);
3165 warn "no efidisk configured! Using temporary efivars disk.\n";
3166 $path = "/tmp/$vmid-ovmf.fd";
3167 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3173 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3174 $size_str = ",size=" . (-s
$ovmf_vars);
3177 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3178 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3183 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3184 if (min_version
($machine_version, 4, 0)) {
3185 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3187 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3191 if ($conf->{vmgenid
}) {
3192 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3195 # add usb controllers
3196 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3197 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3198 push @$devices, @usbcontrollers if @usbcontrollers;
3199 my $vga = parse_vga
($conf->{vga
});
3201 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3202 $vga->{type
} = 'qxl' if $qxlnum;
3204 if (!$vga->{type
}) {
3205 if ($arch eq 'aarch64') {
3206 $vga->{type
} = 'virtio';
3207 } elsif (min_version
($machine_version, 2, 9)) {
3208 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3210 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3214 # enable absolute mouse coordinates (needed by vnc)
3216 if (defined($conf->{tablet
})) {
3217 $tablet = $conf->{tablet
};
3219 $tablet = $defaults->{tablet
};
3220 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3221 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3225 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3226 my $kbd = print_keyboarddevice_full
($conf, $arch);
3227 push @$devices, '-device', $kbd if defined($kbd);
3230 my $bootorder = device_bootorder
($conf);
3232 # host pci device passthrough
3233 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3234 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3237 my $usb_dev_features = {};
3238 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3240 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3241 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3242 push @$devices, @usbdevices if @usbdevices;
3245 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3246 if (my $path = $conf->{"serial$i"}) {
3247 if ($path eq 'socket') {
3248 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3249 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3250 # On aarch64, serial0 is the UART device. Qemu only allows
3251 # connecting UART devices via the '-serial' command line, as
3252 # the device has a fixed slot on the hardware...
3253 if ($arch eq 'aarch64' && $i == 0) {
3254 push @$devices, '-serial', "chardev:serial$i";
3256 push @$devices, '-device', "isa-serial,chardev=serial$i";
3259 die "no such serial device\n" if ! -c
$path;
3260 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3261 push @$devices, '-device', "isa-serial,chardev=serial$i";
3267 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3268 if (my $path = $conf->{"parallel$i"}) {
3269 die "no such parallel device\n" if ! -c
$path;
3270 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3271 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3272 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3276 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3277 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3278 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3279 push @$devices, @$audio_devs;
3283 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3284 $sockets = $conf->{sockets
} if $conf->{sockets
};
3286 my $cores = $conf->{cores
} || 1;
3288 my $maxcpus = $sockets * $cores;
3290 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3292 my $allowed_vcpus = $cpuinfo->{cpus
};
3294 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3295 if ($allowed_vcpus < $maxcpus);
3297 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3299 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3300 for (my $i = 2; $i <= $vcpus; $i++) {
3301 my $cpustr = print_cpu_device
($conf,$i);
3302 push @$cmd, '-device', $cpustr;
3307 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3309 push @$cmd, '-nodefaults';
3311 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3313 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3315 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3317 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3318 push @$devices, '-device', print_vga_device
(
3319 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3320 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3321 push @$cmd, '-vnc', "unix:$socket,password";
3323 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3324 push @$cmd, '-nographic';
3328 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3329 my $useLocaltime = $conf->{localtime};
3331 if ($winversion >= 5) { # windows
3332 $useLocaltime = 1 if !defined($conf->{localtime});
3334 # use time drift fix when acpi is enabled
3335 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3336 $tdf = 1 if !defined($conf->{tdf
});
3340 if ($winversion >= 6) {
3341 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3342 push @$cmd, '-no-hpet';
3345 push @$rtcFlags, 'driftfix=slew' if $tdf;
3347 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3348 push @$rtcFlags, "base=$conf->{startdate}";
3349 } elsif ($useLocaltime) {
3350 push @$rtcFlags, 'base=localtime';
3354 push @$cmd, '-cpu', $forcecpu;
3356 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3359 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3361 push @$cmd, '-S' if $conf->{freeze
};
3363 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3365 my $guest_agent = parse_guest_agent
($conf);
3367 if ($guest_agent->{enabled
}) {
3368 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3369 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3371 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3372 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3373 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3374 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3375 } elsif ($guest_agent->{type
} eq 'isa') {
3376 push @$devices, '-device', "isa-serial,chardev=qga0";
3380 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3381 if ($rng && $version_guard->(4, 1, 2)) {
3382 check_rng_source
($rng->{source
});
3384 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3385 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3386 my $limiter_str = "";
3388 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3391 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3392 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3393 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3401 for (my $i = 1; $i < $qxlnum; $i++){
3402 push @$devices, '-device', print_vga_device
(
3403 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3406 # assume other OS works like Linux
3407 my ($ram, $vram) = ("134217728", "67108864");
3408 if ($vga->{memory
}) {
3409 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3410 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3412 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3413 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3417 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3419 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3420 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3421 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3423 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3424 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3425 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3427 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3428 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3430 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3431 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3432 if ($spice_enhancement->{foldersharing
}) {
3433 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3434 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3437 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3438 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3439 if $spice_enhancement->{videostreaming
};
3441 push @$devices, '-spice', "$spice_opts";
3444 # enable balloon by default, unless explicitly disabled
3445 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3446 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3447 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3450 if ($conf->{watchdog
}) {
3451 my $wdopts = parse_watchdog
($conf->{watchdog
});
3452 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3453 my $watchdog = $wdopts->{model
} || 'i6300esb';
3454 push @$devices, '-device', "$watchdog$pciaddr";
3455 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3459 my $scsicontroller = {};
3460 my $ahcicontroller = {};
3461 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3463 # Add iscsi initiator name if available
3464 if (my $initiator = get_initiator_name
()) {
3465 push @$devices, '-iscsi', "initiator-name=$initiator";
3468 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3469 my ($ds, $drive) = @_;
3471 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3472 push @$vollist, $drive->{file
};
3475 # ignore efidisk here, already added in bios/fw handling code above
3476 return if $drive->{interface
} eq 'efidisk';
3478 $use_virtio = 1 if $ds =~ m/^virtio/;
3480 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3482 if ($drive->{interface
} eq 'virtio'){
3483 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3486 if ($drive->{interface
} eq 'scsi') {
3488 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3490 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3491 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3493 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3494 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3497 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3498 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3499 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3500 } elsif ($drive->{iothread
}) {
3501 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3505 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3506 $queues = ",num_queues=$drive->{queues}";
3509 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3510 if !$scsicontroller->{$controller};
3511 $scsicontroller->{$controller}=1;
3514 if ($drive->{interface
} eq 'sata') {
3515 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3516 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3517 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3518 if !$ahcicontroller->{$controller};
3519 $ahcicontroller->{$controller}=1;
3522 my $drive_cmd = print_drive_commandline_full
($storecfg, $vmid, $drive);
3523 $drive_cmd .= ',readonly' if PVE
::QemuConfig-
>is_template($conf);
3525 push @$devices, '-drive',$drive_cmd;
3526 push @$devices, '-device', print_drivedevice_full
(
3527 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3530 for (my $i = 0; $i < $MAX_NETS; $i++) {
3531 my $netname = "net$i";
3533 next if !$conf->{$netname};
3534 my $d = parse_net
($conf->{$netname});
3537 $use_virtio = 1 if $d->{model
} eq 'virtio';
3539 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
3541 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
3542 push @$devices, '-netdev', $netdevfull;
3544 my $netdevicefull = print_netdevice_full
(
3545 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3547 push @$devices, '-device', $netdevicefull;
3550 if ($conf->{ivshmem
}) {
3551 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3555 $bus = print_pcie_addr
("ivshmem");
3557 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3560 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3561 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3563 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3564 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3565 .",size=$ivshmem->{size}M";
3568 # pci.4 is nested in pci.1
3569 $bridges->{1} = 1 if $bridges->{4};
3573 if (min_version
($machine_version, 2, 3)) {
3578 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3582 for my $k (sort {$b cmp $a} keys %$bridges) {
3583 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3586 if ($k == 2 && $legacy_igd) {
3589 $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
3591 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3593 # add after -readconfig pve-q35.cfg
3594 splice @$devices, 2, 0, '-device', $devstr;
3596 unshift @$devices, '-device', $devstr if $k > 0;
3601 push @$machineFlags, 'accel=tcg';
3604 my $machine_type_min = $machine_type;
3605 if ($add_pve_version) {
3606 $machine_type_min =~ s/\+pve\d+$//;
3607 $machine_type_min .= "+pve$required_pve_version";
3609 push @$machineFlags, "type=${machine_type_min}";
3611 push @$cmd, @$devices;
3612 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3613 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3614 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3616 if (my $vmstate = $conf->{vmstate
}) {
3617 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3618 push @$vollist, $vmstate;
3619 push @$cmd, '-loadstate', $statepath;
3620 print "activating and using '$vmstate' as vmstate\n";
3624 if ($conf->{args
}) {
3625 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3629 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3632 sub check_rng_source
{
3635 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3636 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3639 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3640 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
3641 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3642 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3643 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3644 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3652 my $res = mon_cmd
($vmid, 'query-spice');
3654 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3657 sub vm_devices_list
{
3660 my $res = mon_cmd
($vmid, 'query-pci');
3661 my $devices_to_check = [];
3663 foreach my $pcibus (@$res) {
3664 push @$devices_to_check, @{$pcibus->{devices
}},
3667 while (@$devices_to_check) {
3669 for my $d (@$devices_to_check) {
3670 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3671 next if !$d->{'pci_bridge'};
3673 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3674 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3676 $devices_to_check = $to_check;
3679 my $resblock = mon_cmd
($vmid, 'query-block');
3680 foreach my $block (@$resblock) {
3681 if($block->{device
} =~ m/^drive-(\S+)/){
3686 my $resmice = mon_cmd
($vmid, 'query-mice');
3687 foreach my $mice (@$resmice) {
3688 if ($mice->{name
} eq 'QEMU HID Tablet') {
3689 $devices->{tablet
} = 1;
3694 # for usb devices there is no query-usb
3695 # but we can iterate over the entries in
3696 # qom-list path=/machine/peripheral
3697 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3698 foreach my $per (@$resperipheral) {
3699 if ($per->{name
} =~ m/^usb\d+$/) {
3700 $devices->{$per->{name
}} = 1;
3708 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3710 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3712 my $devices_list = vm_devices_list
($vmid);
3713 return 1 if defined($devices_list->{$deviceid});
3715 # add PCI bridge if we need it for the device
3716 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3718 if ($deviceid eq 'tablet') {
3720 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
3722 } elsif ($deviceid eq 'keyboard') {
3724 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
3726 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3728 die "usb hotplug currently not reliable\n";
3729 # since we can't reliably hot unplug all added usb devices and usb
3730 # passthrough breaks live migration we disable usb hotplugging for now
3731 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3733 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3735 qemu_iothread_add
($vmid, $deviceid, $device);
3737 qemu_driveadd
($storecfg, $vmid, $device);
3738 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3740 qemu_deviceadd
($vmid, $devicefull);
3741 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3743 eval { qemu_drivedel
($vmid, $deviceid); };
3748 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3751 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3752 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3753 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3755 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3757 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3758 qemu_iothread_add
($vmid, $deviceid, $device);
3759 $devicefull .= ",iothread=iothread-$deviceid";
3762 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3763 $devicefull .= ",num_queues=$device->{queues}";
3766 qemu_deviceadd
($vmid, $devicefull);
3767 qemu_deviceaddverify
($vmid, $deviceid);
3769 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3771 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3772 qemu_driveadd
($storecfg, $vmid, $device);
3774 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3775 eval { qemu_deviceadd
($vmid, $devicefull); };
3777 eval { qemu_drivedel
($vmid, $deviceid); };
3782 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3784 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
3786 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
3787 my $use_old_bios_files = undef;
3788 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3790 my $netdevicefull = print_netdevice_full
(
3791 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3792 qemu_deviceadd
($vmid, $netdevicefull);
3794 qemu_deviceaddverify
($vmid, $deviceid);
3795 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
3798 eval { qemu_netdevdel
($vmid, $deviceid); };
3803 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3806 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3807 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3809 qemu_deviceadd
($vmid, $devicefull);
3810 qemu_deviceaddverify
($vmid, $deviceid);
3813 die "can't hotplug device '$deviceid'\n";
3819 # fixme: this should raise exceptions on error!
3820 sub vm_deviceunplug
{
3821 my ($vmid, $conf, $deviceid) = @_;
3823 my $devices_list = vm_devices_list
($vmid);
3824 return 1 if !defined($devices_list->{$deviceid});
3826 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
3827 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3829 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3831 qemu_devicedel
($vmid, $deviceid);
3833 } elsif ($deviceid =~ m/^usb\d+$/) {
3835 die "usb hotplug currently not reliable\n";
3836 # when unplugging usb devices this way, there may be remaining usb
3837 # controllers/hubs so we disable it for now
3838 #qemu_devicedel($vmid, $deviceid);
3839 #qemu_devicedelverify($vmid, $deviceid);
3841 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3843 qemu_devicedel
($vmid, $deviceid);
3844 qemu_devicedelverify
($vmid, $deviceid);
3845 qemu_drivedel
($vmid, $deviceid);
3846 qemu_iothread_del
($conf, $vmid, $deviceid);
3848 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3850 qemu_devicedel
($vmid, $deviceid);
3851 qemu_devicedelverify
($vmid, $deviceid);
3852 qemu_iothread_del
($conf, $vmid, $deviceid);
3854 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3856 qemu_devicedel
($vmid, $deviceid);
3857 qemu_drivedel
($vmid, $deviceid);
3858 qemu_deletescsihw
($conf, $vmid, $deviceid);
3860 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3862 qemu_devicedel
($vmid, $deviceid);
3863 qemu_devicedelverify
($vmid, $deviceid);
3864 qemu_netdevdel
($vmid, $deviceid);
3867 die "can't unplug device '$deviceid'\n";
3873 sub qemu_deviceadd
{
3874 my ($vmid, $devicefull) = @_;
3876 $devicefull = "driver=".$devicefull;
3877 my %options = split(/[=,]/, $devicefull);
3879 mon_cmd
($vmid, "device_add" , %options);
3882 sub qemu_devicedel
{
3883 my ($vmid, $deviceid) = @_;
3885 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
3888 sub qemu_iothread_add
{
3889 my($vmid, $deviceid, $device) = @_;
3891 if ($device->{iothread
}) {
3892 my $iothreads = vm_iothreads_list
($vmid);
3893 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3897 sub qemu_iothread_del
{
3898 my($conf, $vmid, $deviceid) = @_;
3900 my $confid = $deviceid;
3901 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3902 $confid = 'scsi' . $1;
3904 my $device = parse_drive
($confid, $conf->{$confid});
3905 if ($device->{iothread
}) {
3906 my $iothreads = vm_iothreads_list
($vmid);
3907 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3911 sub qemu_objectadd
{
3912 my($vmid, $objectid, $qomtype) = @_;
3914 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
3919 sub qemu_objectdel
{
3920 my($vmid, $objectid) = @_;
3922 mon_cmd
($vmid, "object-del", id
=> $objectid);
3928 my ($storecfg, $vmid, $device) = @_;
3930 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device);
3931 $drive =~ s/\\/\\\\/g;
3932 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
3934 # If the command succeeds qemu prints: "OK
"
3935 return 1 if $ret =~ m/OK/s;
3937 die "adding drive failed
: $ret\n";
3941 my($vmid, $deviceid) = @_;
3943 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
3946 return 1 if $ret eq "";
3948 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3949 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3951 die "deleting drive
$deviceid failed
: $ret\n";
3954 sub qemu_deviceaddverify {
3955 my ($vmid, $deviceid) = @_;
3957 for (my $i = 0; $i <= 5; $i++) {
3958 my $devices_list = vm_devices_list($vmid);
3959 return 1 if defined($devices_list->{$deviceid});
3963 die "error on hotplug device
'$deviceid'\n";
3967 sub qemu_devicedelverify {
3968 my ($vmid, $deviceid) = @_;
3970 # need to verify that the device is correctly removed as device_del
3971 # is async and empty return is not reliable
3973 for (my $i = 0; $i <= 5; $i++) {
3974 my $devices_list = vm_devices_list($vmid);
3975 return 1 if !defined($devices_list->{$deviceid});
3979 die "error on hot-unplugging device
'$deviceid'\n";
3982 sub qemu_findorcreatescsihw {
3983 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3985 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3987 my $scsihwid="$controller_prefix$controller";
3988 my $devices_list = vm_devices_list($vmid);
3990 if(!defined($devices_list->{$scsihwid})) {
3991 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3997 sub qemu_deletescsihw {
3998 my ($conf, $vmid, $opt) = @_;
4000 my $device = parse_drive($opt, $conf->{$opt});
4002 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4003 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4007 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4009 my $devices_list = vm_devices_list($vmid);
4010 foreach my $opt (keys %{$devices_list}) {
4011 if (is_valid_drivename($opt)) {
4012 my $drive = parse_drive($opt, $conf->{$opt});
4013 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4019 my $scsihwid="scsihw
$controller";
4021 vm_deviceunplug($vmid, $conf, $scsihwid);
4026 sub qemu_add_pci_bridge {
4027 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4033 print_pci_addr($device, $bridges, $arch, $machine_type);
4035 while (my ($k, $v) = each %$bridges) {
4038 return 1 if !defined($bridgeid) || $bridgeid < 1;
4040 my $bridge = "pci
.$bridgeid";
4041 my $devices_list = vm_devices_list($vmid);
4043 if (!defined($devices_list->{$bridge})) {
4044 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4050 sub qemu_set_link_status {
4051 my ($vmid, $device, $up) = @_;
4053 mon_cmd($vmid, "set_link
", name => $device,
4054 up => $up ? JSON::true : JSON::false);
4057 sub qemu_netdevadd {
4058 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4060 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4061 my %options = split(/[=,]/, $netdev);
4063 if (defined(my $vhost = $options{vhost})) {
4064 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4067 if (defined(my $queues = $options{queues})) {
4068 $options{queues} = $queues + 0;
4071 mon_cmd($vmid, "netdev_add
", %options);
4075 sub qemu_netdevdel {
4076 my ($vmid, $deviceid) = @_;
4078 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4081 sub qemu_usb_hotplug {
4082 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4086 # remove the old one first
4087 vm_deviceunplug($vmid, $conf, $deviceid);
4089 # check if xhci controller is necessary and available
4090 if ($device->{usb3}) {
4092 my $devicelist = vm_devices_list($vmid);
4094 if (!$devicelist->{xhci}) {
4095 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4096 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4099 my $d = parse_usb_device($device->{host});
4100 $d->{usb3} = $device->{usb3};
4103 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4106 sub qemu_cpu_hotplug {
4107 my ($vmid, $conf, $vcpus) = @_;
4109 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4112 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4113 $sockets = $conf->{sockets} if $conf->{sockets};
4114 my $cores = $conf->{cores} || 1;
4115 my $maxcpus = $sockets * $cores;
4117 $vcpus = $maxcpus if !$vcpus;
4119 die "you can
't add more vcpus than maxcpus\n"
4120 if $vcpus > $maxcpus;
4122 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4124 if ($vcpus < $currentvcpus) {
4126 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4128 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4129 qemu_devicedel($vmid, "cpu$i");
4131 my $currentrunningvcpus = undef;
4133 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4134 last if scalar(@{$currentrunningvcpus}) == $i-1;
4135 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4139 #update conf after each succesfull cpu unplug
4140 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4141 PVE::QemuConfig->write_config($vmid, $conf);
4144 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4150 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4151 die "vcpus in running vm does not match its configuration\n"
4152 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4154 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4156 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4157 my $cpustr = print_cpu_device($conf, $i);
4158 qemu_deviceadd($vmid, $cpustr);
4161 my $currentrunningvcpus = undef;
4163 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4164 last if scalar(@{$currentrunningvcpus}) == $i;
4165 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4169 #update conf after each succesfull cpu hotplug
4170 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4171 PVE::QemuConfig->write_config($vmid, $conf);
4175 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4176 mon_cmd($vmid, "cpu-add", id => int($i));
4181 sub qemu_block_set_io_throttle {
4182 my ($vmid, $deviceid,
4183 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4184 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4185 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4186 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4188 return if !check_running($vmid) ;
4190 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4192 bps_rd => int($bps_rd),
4193 bps_wr => int($bps_wr),
4195 iops_rd => int($iops_rd),
4196 iops_wr => int($iops_wr),
4197 bps_max => int($bps_max),
4198 bps_rd_max => int($bps_rd_max),
4199 bps_wr_max => int($bps_wr_max),
4200 iops_max => int($iops_max),
4201 iops_rd_max => int($iops_rd_max),
4202 iops_wr_max => int($iops_wr_max),
4203 bps_max_length => int($bps_max_length),
4204 bps_rd_max_length => int($bps_rd_max_length),
4205 bps_wr_max_length => int($bps_wr_max_length),
4206 iops_max_length => int($iops_max_length),
4207 iops_rd_max_length => int($iops_rd_max_length),
4208 iops_wr_max_length => int($iops_wr_max_length),
4213 sub qemu_block_resize {
4214 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4216 my $running = check_running($vmid);
4218 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4220 return if !$running;
4222 my $padding = (1024 - $size % 1024) % 1024;
4223 $size = $size + $padding;
4225 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4229 sub qemu_volume_snapshot {
4230 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4232 my $running = check_running($vmid);
4234 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4235 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4237 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4241 sub qemu_volume_snapshot_delete {
4242 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4244 my $running = check_running($vmid);
4249 my $conf = PVE::QemuConfig->load_config($vmid);
4250 PVE::QemuConfig->foreach_volume($conf, sub {
4251 my ($ds, $drive) = @_;
4252 $running = 1 if $drive->{file} eq $volid;
4256 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4257 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4259 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4263 sub set_migration_caps {
4266 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4271 "auto-converge" => 1,
4273 "x-rdma-pin-all" => 0,
4276 "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration
'} ? 1 : 0,
4279 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4281 for my $supported_capability (@$supported_capabilities) {
4283 capability => $supported_capability->{capability},
4284 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4288 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4292 my ($conf, $func, @param) = @_;
4296 my $test_volid = sub {
4297 my ($key, $drive, $snapname) = @_;
4299 my $volid = $drive->{file};
4302 $volhash->{$volid}->{cdrom} //= 1;
4303 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4305 my $replicate = $drive->{replicate} // 1;
4306 $volhash->{$volid}->{replicate} //= 0;
4307 $volhash->{$volid}->{replicate} = 1 if $replicate;
4309 $volhash->{$volid}->{shared} //= 0;
4310 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4312 $volhash->{$volid}->{referenced_in_config} //= 0;
4313 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4315 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4316 if defined($snapname);
4318 my $size = $drive->{size};
4319 $volhash->{$volid}->{size} //= $size if $size;
4321 $volhash->{$volid}->{is_vmstate} //= 0;
4322 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4324 $volhash->{$volid}->{is_unused} //= 0;
4325 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4328 my $include_opts = {
4329 extra_keys => ['vmstate
'],
4330 include_unused => 1,
4333 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4334 foreach my $snapname (keys %{$conf->{snapshots}}) {
4335 my $snap = $conf->{snapshots}->{$snapname};
4336 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4339 foreach my $volid (keys %$volhash) {
4340 &$func($volid, $volhash->{$volid}, @param);
4344 my $fast_plug_option = {
4352 'vmstatestorage
' => 1,
4357 # hotplug changes in [PENDING]
4358 # $selection hash can be used to only apply specified options, for
4359 # example: { cores => 1 } (only apply changed 'cores
')
4360 # $errors ref is used to return error messages
4361 sub vmconfig_hotplug_pending {
4362 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4364 my $defaults = load_defaults();
4365 my $arch = get_vm_arch($conf);
4366 my $machine_type = get_vm_machine($conf, undef, $arch);
4368 # commit values which do not have any impact on running VM first
4369 # Note: those option cannot raise errors, we we do not care about
4370 # $selection and always apply them.
4372 my $add_error = sub {
4373 my ($opt, $msg) = @_;
4374 $errors->{$opt} = "hotplug problem - $msg";
4378 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4379 if ($fast_plug_option->{$opt}) {
4380 $conf->{$opt} = $conf->{pending}->{$opt};
4381 delete $conf->{pending}->{$opt};
4387 PVE::QemuConfig->write_config($vmid, $conf);
4390 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4392 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4393 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4394 foreach my $opt (sort keys %$pending_delete_hash) {
4395 next if $selection && !$selection->{$opt};
4396 my $force = $pending_delete_hash->{$opt}->{force};
4398 if ($opt eq 'hotplug
') {
4399 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4400 } elsif ($opt eq 'tablet
') {
4401 die "skip\n" if !$hotplug_features->{usb};
4402 if ($defaults->{tablet}) {
4403 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4404 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4405 if $arch eq 'aarch64
';
4407 vm_deviceunplug($vmid, $conf, 'tablet
');
4408 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4410 } elsif ($opt =~ m/^usb\d+/) {
4412 # since we cannot reliably hot unplug usb devices we are disabling it
4413 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4414 #vm_deviceunplug($vmid, $conf, $opt);
4415 } elsif ($opt eq 'vcpus
') {
4416 die "skip\n" if !$hotplug_features->{cpu};
4417 qemu_cpu_hotplug($vmid, $conf, undef);
4418 } elsif ($opt eq 'balloon
') {
4419 # enable balloon device is not hotpluggable
4420 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4421 # here we reset the ballooning value to memory
4422 my $balloon = $conf->{memory} || $defaults->{memory};
4423 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4424 } elsif ($fast_plug_option->{$opt}) {
4426 } elsif ($opt =~ m/^net(\d+)$/) {
4427 die "skip\n" if !$hotplug_features->{network};
4428 vm_deviceunplug($vmid, $conf, $opt);
4429 } elsif (is_valid_drivename($opt)) {
4430 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4431 vm_deviceunplug($vmid, $conf, $opt);
4432 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4433 } elsif ($opt =~ m/^memory$/) {
4434 die "skip\n" if !$hotplug_features->{memory};
4435 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4436 } elsif ($opt eq 'cpuunits
') {
4437 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4438 } elsif ($opt eq 'cpulimit
') {
4439 $cgroup->change_cpu_quota(-1, 100000);
4445 &$add_error($opt, $err) if $err ne "skip\n";
4447 delete $conf->{$opt};
4448 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4452 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4453 $apply_pending_cloudinit = sub {
4454 return if $apply_pending_cloudinit_done; # once is enough
4455 $apply_pending_cloudinit_done = 1; # once is enough
4457 my ($key, $value) = @_;
4459 my @cloudinit_opts = keys %$confdesc_cloudinit;
4460 foreach my $opt (keys %{$conf->{pending}}) {
4461 next if !grep { $_ eq $opt } @cloudinit_opts;
4462 $conf->{$opt} = delete $conf->{pending}->{$opt};
4465 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4466 foreach my $opt (sort keys %$pending_delete_hash) {
4467 next if !grep { $_ eq $opt } @cloudinit_opts;
4468 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4469 delete $conf->{$opt};
4472 my $new_conf = { %$conf };
4473 $new_conf->{$key} = $value;
4474 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4477 foreach my $opt (keys %{$conf->{pending}}) {
4478 next if $selection && !$selection->{$opt};
4479 my $value = $conf->{pending}->{$opt};
4481 if ($opt eq 'hotplug
') {
4482 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4483 } elsif ($opt eq 'tablet
') {
4484 die "skip\n" if !$hotplug_features->{usb};
4486 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4487 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4488 if $arch eq 'aarch64
';
4489 } elsif ($value == 0) {
4490 vm_deviceunplug($vmid, $conf, 'tablet
');
4491 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4493 } elsif ($opt =~ m/^usb\d+$/) {
4495 # since we cannot reliably hot unplug usb devices we disable it for now
4496 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4497 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4498 #die "skip\n" if !$d;
4499 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4500 } elsif ($opt eq 'vcpus
') {
4501 die "skip\n" if !$hotplug_features->{cpu};
4502 qemu_cpu_hotplug($vmid, $conf, $value);
4503 } elsif ($opt eq 'balloon
') {
4504 # enable/disable balloning device is not hotpluggable
4505 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4506 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4507 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4509 # allow manual ballooning if shares is set to zero
4510 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4511 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4512 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4514 } elsif ($opt =~ m/^net(\d+)$/) {
4515 # some changes can be done without hotplug
4516 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4517 $vmid, $opt, $value, $arch, $machine_type);
4518 } elsif (is_valid_drivename($opt)) {
4519 die "skip\n" if $opt eq 'efidisk0
';
4520 # some changes can be done without hotplug
4521 my $drive = parse_drive($opt, $value);
4522 if (drive_is_cloudinit($drive)) {
4523 &$apply_pending_cloudinit($opt, $value);
4525 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4526 $vmid, $opt, $value, $arch, $machine_type);
4527 } elsif ($opt =~ m/^memory$/) { #dimms
4528 die "skip\n" if !$hotplug_features->{memory};
4529 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4530 } elsif ($opt eq 'cpuunits
') {
4531 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4532 } elsif ($opt eq 'cpulimit
') {
4533 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4534 $cgroup->change_cpu_quota($cpulimit, 100000);
4536 die "skip\n"; # skip non-hot-pluggable options
4540 &$add_error($opt, $err) if $err ne "skip\n";
4542 $conf->{$opt} = $value;
4543 delete $conf->{pending}->{$opt};
4547 PVE::QemuConfig->write_config($vmid, $conf);
4550 sub try_deallocate_drive {
4551 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4553 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4554 my $volid = $drive->{file};
4555 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4556 my $sid = PVE::Storage::parse_volume_id($volid);
4557 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4559 # check if the disk is really unused
4560 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4561 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4562 PVE::Storage::vdisk_free($storecfg, $volid);
4565 # If vm is not owner of this disk remove from config
4573 sub vmconfig_delete_or_detach_drive {
4574 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4576 my $drive = parse_drive($opt, $conf->{$opt});
4578 my $rpcenv = PVE::RPCEnvironment::get();
4579 my $authuser = $rpcenv->get_user();
4582 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4583 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4585 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4591 sub vmconfig_apply_pending {
4592 my ($vmid, $conf, $storecfg, $errors) = @_;
4594 my $add_apply_error = sub {
4595 my ($opt, $msg) = @_;
4596 my $err_msg = "unable to apply pending change $opt : $msg";
4597 $errors->{$opt} = $err_msg;
4603 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4604 foreach my $opt (sort keys %$pending_delete_hash) {
4605 my $force = $pending_delete_hash->{$opt}->{force};
4607 if ($opt =~ m/^unused/) {
4608 die "internal error";
4609 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4610 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4614 $add_apply_error->($opt, $err);
4616 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4617 delete $conf->{$opt};
4621 PVE::QemuConfig->cleanup_pending($conf);
4623 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4624 next if $opt eq 'delete'; # just to be sure
4626 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4627 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4631 $add_apply_error->($opt, $err);
4633 $conf->{$opt} = delete $conf->{pending}->{$opt};
4637 # write all changes at once to avoid unnecessary i/o
4638 PVE::QemuConfig->write_config($vmid, $conf);
4641 sub vmconfig_update_net {
4642 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4644 my $newnet = parse_net($value);
4646 if ($conf->{$opt}) {
4647 my $oldnet = parse_net($conf->{$opt});
4649 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4650 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4651 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4652 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4654 # for non online change, we try to hot-unplug
4655 die "skip\n" if !$hotplug;
4656 vm_deviceunplug($vmid, $conf, $opt);
4659 die "internal error" if $opt !~ m/net(\d+)/;
4660 my $iface = "tap${vmid}i$1";
4662 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4663 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4664 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4665 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4666 PVE::Network::tap_unplug($iface);
4669 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4671 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4673 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4674 # Rate can be applied on its own but any change above needs to
4675 # include the rate in tap_plug since OVS resets everything.
4676 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4679 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4680 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4688 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4694 sub vmconfig_update_disk {
4695 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4697 my $drive = parse_drive($opt, $value);
4699 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4700 my $media = $drive->{media} || 'disk
';
4701 my $oldmedia = $old_drive->{media} || 'disk
';
4702 die "unable to change media type\n" if $media ne $oldmedia;
4704 if (!drive_is_cdrom($old_drive)) {
4706 if ($drive->{file} ne $old_drive->{file}) {
4708 die "skip\n" if !$hotplug;
4710 # unplug and register as unused
4711 vm_deviceunplug($vmid, $conf, $opt);
4712 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4715 # update existing disk
4717 # skip non hotpluggable value
4718 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4719 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4720 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4721 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4722 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4727 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4728 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4729 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4730 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4731 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4732 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4733 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4734 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4735 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4736 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4737 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4738 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4739 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4740 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4741 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4742 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4743 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4744 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4746 qemu_block_set_io_throttle(
4748 ($drive->{mbps} || 0)*1024*1024,
4749 ($drive->{mbps_rd} || 0)*1024*1024,
4750 ($drive->{mbps_wr} || 0)*1024*1024,
4751 $drive->{iops} || 0,
4752 $drive->{iops_rd} || 0,
4753 $drive->{iops_wr} || 0,
4754 ($drive->{mbps_max} || 0)*1024*1024,
4755 ($drive->{mbps_rd_max} || 0)*1024*1024,
4756 ($drive->{mbps_wr_max} || 0)*1024*1024,
4757 $drive->{iops_max} || 0,
4758 $drive->{iops_rd_max} || 0,
4759 $drive->{iops_wr_max} || 0,
4760 $drive->{bps_max_length} || 1,
4761 $drive->{bps_rd_max_length} || 1,
4762 $drive->{bps_wr_max_length} || 1,
4763 $drive->{iops_max_length} || 1,
4764 $drive->{iops_rd_max_length} || 1,
4765 $drive->{iops_wr_max_length} || 1,
4775 if ($drive->{file} eq 'none
') {
4776 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4777 if (drive_is_cloudinit($old_drive)) {
4778 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4781 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4783 # force eject if locked
4784 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4787 mon_cmd($vmid, "blockdev-change-medium",
4788 id => "$opt", filename => "$path");
4796 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4798 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4799 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4802 # called in locked context by incoming migration
4803 sub vm_migrate_get_nbd_disks {
4804 my ($storecfg, $conf, $replicated_volumes) = @_;
4806 my $local_volumes = {};
4807 PVE::QemuConfig->foreach_volume($conf, sub {
4808 my ($ds, $drive) = @_;
4810 return if drive_is_cdrom($drive);
4812 my $volid = $drive->{file};
4816 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4818 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4819 return if $scfg->{shared};
4821 # replicated disks re-use existing state via bitmap
4822 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4823 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4825 return $local_volumes;
4828 # called in locked context by incoming migration
4829 sub vm_migrate_alloc_nbd_disks {
4830 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4835 foreach my $opt (sort keys %$source_volumes) {
4836 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4838 if ($use_existing) {
4839 $nbd->{$opt}->{drivestr} = print_drive($drive);
4840 $nbd->{$opt}->{volid} = $volid;
4841 $nbd->{$opt}->{replicated} = 1;
4845 # If a remote storage is specified and the format of the original
4846 # volume is not available there, fall back to the default format.
4847 # Otherwise use the same format as the original.
4848 if (!$storagemap->{identity}) {
4849 $storeid = map_storage($storagemap, $storeid);
4850 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4851 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4852 my $fileFormat = qemu_img_format($scfg, $volname);
4853 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4855 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4856 $format = qemu_img_format($scfg, $volname);
4859 my $size = $drive->{size} / 1024;
4860 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4861 my $newdrive = $drive;
4862 $newdrive->{format} = $format;
4863 $newdrive->{file} = $newvolid;
4864 my $drivestr = print_drive($newdrive);
4865 $nbd->{$opt}->{drivestr} = $drivestr;
4866 $nbd->{$opt}->{volid} = $newvolid;
4872 # see vm_start_nolock for parameters, additionally:
4874 # storagemap = parsed storage map for allocating NBD disks
4876 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4878 return PVE::QemuConfig->lock_config($vmid, sub {
4879 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4881 die "you can't start a vm
if it
's a template\n"
4882 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4884 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended
');
4885 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup
');
4887 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4889 if ($has_backup_lock && $running) {
4890 # a backup is currently running, attempt to start the guest in the
4891 # existing QEMU instance
4892 return vm_resume($vmid);
4895 PVE::QemuConfig->check_lock($conf)
4896 if !($params->{skiplock} || $has_suspended_lock);
4898 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4900 die "VM $vmid already running\n" if $running;
4902 if (my $storagemap = $migrate_opts->{storagemap}) {
4903 my $replicated = $migrate_opts->{replicated_volumes};
4904 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4905 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4907 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4908 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4912 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4918 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
4919 # skiplock => 0/1, skip checking for config lock
4920 # skiptemplate => 0/1, skip checking whether VM is template
4921 # forcemachine => to force Qemu machine (rollback/migration)
4922 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
4923 # timeout => in seconds
4924 # paused => start VM in paused state (backup)
4925 # resume => resume from hibernation
4927 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4928 # migratedfrom => source node
4929 # spice_ticket => used for spice migration, passed via tunnel/stdin
4930 # network => CIDR of migration network
4931 # type => secure/insecure - tunnel over encrypted connection or plain-text
4932 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
4933 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4934 sub vm_start_nolock {
4935 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4937 my $statefile = $params->{statefile};
4938 my $resume = $params->{resume};
4940 my $migratedfrom = $migrate_opts->{migratedfrom};
4941 my $migration_type = $migrate_opts->{type};
4945 # clean up leftover reboot request files
4946 eval { clear_reboot_request($vmid); };
4949 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4950 vmconfig_apply_pending($vmid, $conf, $storecfg);
4951 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4954 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4956 my $defaults = load_defaults();
4958 # set environment variable useful inside network script
4959 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4961 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
4963 my $forcemachine = $params->{forcemachine};
4964 my $forcecpu = $params->{forcecpu};
4966 # enforce machine and CPU type on suspended vm to ensure HW compatibility
4967 $forcemachine = $conf->{runningmachine};
4968 $forcecpu = $conf->{runningcpu};
4969 print "Resuming suspended VM\n";
4972 my ($cmd, $vollist, $spice_port) =
4973 config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4976 my $get_migration_ip = sub {
4977 my ($nodename) = @_;
4979 return $migration_ip if defined($migration_ip);
4981 my $cidr = $migrate_opts->{network};
4983 if (!defined($cidr)) {
4984 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
4985 $cidr = $dc_conf->{migration}->{network};
4988 if (defined($cidr)) {
4989 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4991 die "could not get IP: no address configured on local " .
4992 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4994 die "could not get IP: multiple addresses configured on local " .
4995 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4997 $migration_ip = @$ips[0];
5000 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5001 if !defined($migration_ip);
5003 return $migration_ip;
5008 if ($statefile eq 'tcp
') {
5009 my $localip = "localhost";
5010 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5011 my $nodename = nodename();
5013 if (!defined($migration_type)) {
5014 if (defined($datacenterconf->{migration}->{type})) {
5015 $migration_type = $datacenterconf->{migration}->{type};
5017 $migration_type = 'secure
';
5021 if ($migration_type eq 'insecure
') {
5022 $localip = $get_migration_ip->($nodename);
5023 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5026 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5027 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5028 $migrate_uri = "tcp:${localip}:${migrate_port}";
5029 push @$cmd, '-incoming
', $migrate_uri;
5032 } elsif ($statefile eq 'unix
') {
5033 # should be default for secure migrations as a ssh TCP forward
5034 # tunnel is not deterministic reliable ready and fails regurarly
5035 # to set up in time, so use UNIX socket forwards
5036 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5037 unlink $socket_addr;
5039 $migrate_uri = "unix:$socket_addr";
5041 push @$cmd, '-incoming
', $migrate_uri;
5044 } elsif (-e $statefile) {
5045 push @$cmd, '-loadstate
', $statefile;
5047 my $statepath = PVE::Storage::path($storecfg, $statefile);
5048 push @$vollist, $statefile;
5049 push @$cmd, '-loadstate
', $statepath;
5051 } elsif ($params->{paused}) {
5056 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
5057 my $d = parse_hostpci($conf->{"hostpci$i"});
5059 my $pcidevices = $d->{pciid};
5060 foreach my $pcidevice (@$pcidevices) {
5061 my $pciid = $pcidevice->{id};
5063 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5064 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5065 die "no pci device info for device '$pciid'\n" if !$info;
5068 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5069 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5071 die "can't unbind
/bind PCI group to VFIO
'$pciid'\n"
5072 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5073 die "can
't reset PCI device '$pciid'\n"
5074 if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5079 PVE::Storage::activate_volumes($storecfg, $vollist);
5082 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5083 outfunc => sub {}, errfunc => sub {});
5085 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5086 # timeout should be more than enough here...
5087 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5089 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5090 : $defaults->{cpuunits};
5092 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5094 timeout => $statefile ? undef : $start_timeout,
5099 # when migrating, prefix QEMU output so other side can pick up any
5100 # errors that might occur and show the user
5101 if ($migratedfrom) {
5102 $run_params{quiet} = 1;
5103 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5107 Slice => 'qemu
.slice
',
5111 if (PVE::CGroup::cgroup_mode() == 2) {
5112 $properties{CPUWeight} = $cpuunits;
5114 $properties{CPUShares} = $cpuunits;
5117 if (my $cpulimit = $conf->{cpulimit}) {
5118 $properties{CPUQuota} = int($cpulimit * 100);
5120 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5122 my $run_qemu = sub {
5123 PVE::Tools::run_fork sub {
5124 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5126 my $exitcode = run_command($cmd, %run_params);
5127 die "QEMU exited with code $exitcode\n" if $exitcode;
5131 if ($conf->{hugepages}) {
5134 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5135 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5137 PVE::QemuServer::Memory::hugepages_mount();
5138 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5140 eval { $run_qemu->() };
5142 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5143 if !$conf->{keephugepages};
5147 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5148 if !$conf->{keephugepages};
5150 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5153 eval { $run_qemu->() };
5157 # deactivate volumes if start fails
5158 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5159 die "start failed: $err";
5162 print "migration listens on $migrate_uri\n" if $migrate_uri;
5163 $res->{migrate_uri} = $migrate_uri;
5165 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5166 eval { mon_cmd($vmid, "cont"); };
5170 #start nbd server for storage migration
5171 if (my $nbd = $migrate_opts->{nbd}) {
5172 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5174 my $migrate_storage_uri;
5175 # nbd_protocol_version > 0 for unix socket support
5176 if ($nbd_protocol_version > 0 && $migration_type eq 'secure
') {
5177 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5178 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix
', data => { path => $socket_path } } );
5179 $migrate_storage_uri = "nbd:unix:$socket_path";
5181 my $nodename = nodename();
5182 my $localip = $get_migration_ip->($nodename);
5183 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5184 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5186 mon_cmd($vmid, "nbd-server-start", addr => {
5189 host => "${localip}",
5190 port => "${storage_migrate_port}",
5193 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5194 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5197 $res->{migrate_storage_uri} = $migrate_storage_uri;
5199 foreach my $opt (sort keys %$nbd) {
5200 my $drivestr = $nbd->{$opt}->{drivestr};
5201 my $volid = $nbd->{$opt}->{volid};
5202 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5203 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5204 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5205 print "re-using replicated volume: $opt - $volid\n"
5206 if $nbd->{$opt}->{replicated};
5208 $res->{drives}->{$opt} = $nbd->{$opt};
5209 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5213 if ($migratedfrom) {
5215 set_migration_caps($vmid);
5220 print "spice listens on port $spice_port\n";
5221 $res->{spice_port} = $spice_port;
5222 if ($migrate_opts->{spice_ticket}) {
5223 mon_cmd($vmid, "set_password", protocol => 'spice
', password =>
5224 $migrate_opts->{spice_ticket});
5225 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5230 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5231 if !$statefile && $conf->{balloon};
5233 foreach my $opt (keys %$conf) {
5234 next if $opt !~ m/^net\d+$/;
5235 my $nicconf = parse_net($conf->{$opt});
5236 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5240 mon_cmd($vmid, 'qom-set
',
5241 path => "machine/peripheral/balloon0",
5242 property => "guest-stats-polling-interval",
5243 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5246 print "Resumed VM, removing state\n";
5247 if (my $vmstate = $conf->{vmstate}) {
5248 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5249 PVE::Storage::vdisk_free($storecfg, $vmstate);
5251 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5252 PVE
::QemuConfig-
>write_config($vmid, $conf);
5255 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5260 sub vm_commandline
{
5261 my ($storecfg, $vmid, $snapname) = @_;
5263 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5268 my $snapshot = $conf->{snapshots
}->{$snapname};
5269 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5271 # check for machine or CPU overrides in snapshot
5272 $forcemachine = $snapshot->{runningmachine
};
5273 $forcecpu = $snapshot->{runningcpu
};
5275 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5280 my $defaults = load_defaults
();
5282 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults,
5283 $forcemachine, $forcecpu);
5285 return PVE
::Tools
::cmd2string
($cmd);
5289 my ($vmid, $skiplock) = @_;
5291 PVE
::QemuConfig-
>lock_config($vmid, sub {
5293 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5295 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5297 mon_cmd
($vmid, "system_reset");
5301 sub get_vm_volumes
{
5305 foreach_volid
($conf, sub {
5306 my ($volid, $attr) = @_;
5308 return if $volid =~ m
|^/|;
5310 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5313 push @$vollist, $volid;
5319 sub vm_stop_cleanup
{
5320 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5325 my $vollist = get_vm_volumes
($conf);
5326 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5329 foreach my $ext (qw(mon qmp pid vnc qga)) {
5330 unlink "/var/run/qemu-server/${vmid}.$ext";
5333 if ($conf->{ivshmem
}) {
5334 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5335 # just delete it for now, VMs which have this already open do not
5336 # are affected, but new VMs will get a separated one. If this
5337 # becomes an issue we either add some sort of ref-counting or just
5338 # add a "don't delete on stop" flag to the ivshmem format.
5339 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5342 foreach my $key (keys %$conf) {
5343 next if $key !~ m/^hostpci(\d+)$/;
5344 my $hostpciindex = $1;
5345 my $d = parse_hostpci
($conf->{$key});
5346 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5348 foreach my $pci (@{$d->{pciid
}}) {
5349 my $pciid = $pci->{id
};
5350 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5354 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5356 warn $@ if $@; # avoid errors - just warn
5359 # call only in locked context
5361 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5363 my $pid = check_running
($vmid, $nocheck);
5368 $conf = PVE
::QemuConfig-
>load_config($vmid);
5369 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5370 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5371 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5372 $timeout = $opts->{down
} if $opts->{down
};
5374 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5379 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
5380 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5382 mon_cmd
($vmid, "system_powerdown");
5385 mon_cmd
($vmid, "quit");
5391 $timeout = 60 if !defined($timeout);
5394 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5399 if ($count >= $timeout) {
5401 warn "VM still running - terminating now with SIGTERM\n";
5404 die "VM quit/powerdown failed - got timeout\n";
5407 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5411 if (!check_running
($vmid, $nocheck)) {
5412 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5416 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5419 die "VM quit/powerdown failed\n";
5427 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5432 if ($count >= $timeout) {
5433 warn "VM still running - terminating now with SIGKILL\n";
5438 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5441 # Note: use $nocheck to skip tests if VM configuration file exists.
5442 # We need that when migration VMs to other nodes (files already moved)
5443 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5445 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5447 $force = 1 if !defined($force) && !$shutdown;
5450 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5451 kill 15, $pid if $pid;
5452 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5453 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5457 PVE
::QemuConfig-
>lock_config($vmid, sub {
5458 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5463 my ($vmid, $timeout) = @_;
5465 PVE
::QemuConfig-
>lock_config($vmid, sub {
5468 # only reboot if running, as qmeventd starts it again on a stop event
5469 return if !check_running
($vmid);
5471 create_reboot_request
($vmid);
5473 my $storecfg = PVE
::Storage
::config
();
5474 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5478 # avoid that the next normal shutdown will be confused for a reboot
5479 clear_reboot_request
($vmid);
5485 # note: if using the statestorage parameter, the caller has to check privileges
5487 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5494 PVE
::QemuConfig-
>lock_config($vmid, sub {
5496 $conf = PVE
::QemuConfig-
>load_config($vmid);
5498 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5499 PVE
::QemuConfig-
>check_lock($conf)
5500 if !($skiplock || $is_backing_up);
5502 die "cannot suspend to disk during backup\n"
5503 if $is_backing_up && $includestate;
5505 if ($includestate) {
5506 $conf->{lock} = 'suspending';
5507 my $date = strftime
("%Y-%m-%d", localtime(time()));
5508 $storecfg = PVE
::Storage
::config
();
5509 if (!$statestorage) {
5510 $statestorage = find_vmstate_storage
($conf, $storecfg);
5511 # check permissions for the storage
5512 my $rpcenv = PVE
::RPCEnvironment
::get
();
5513 if ($rpcenv->{type
} ne 'cli') {
5514 my $authuser = $rpcenv->get_user();
5515 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5520 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
5521 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5522 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5523 PVE
::QemuConfig-
>write_config($vmid, $conf);
5525 mon_cmd
($vmid, "stop");
5529 if ($includestate) {
5531 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5534 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5536 my $state = mon_cmd
($vmid, "query-savevm");
5537 if (!$state->{status
}) {
5538 die "savevm not active\n";
5539 } elsif ($state->{status
} eq 'active') {
5542 } elsif ($state->{status
} eq 'completed') {
5543 print "State saved, quitting\n";
5545 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5546 die "query-savevm failed with error '$state->{error}'\n"
5548 die "query-savevm returned status '$state->{status}'\n";
5554 PVE
::QemuConfig-
>lock_config($vmid, sub {
5555 $conf = PVE
::QemuConfig-
>load_config($vmid);
5557 # cleanup, but leave suspending lock, to indicate something went wrong
5559 mon_cmd
($vmid, "savevm-end");
5560 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5561 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5562 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5563 PVE
::QemuConfig-
>write_config($vmid, $conf);
5569 die "lock changed unexpectedly\n"
5570 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5572 mon_cmd
($vmid, "quit");
5573 $conf->{lock} = 'suspended';
5574 PVE
::QemuConfig-
>write_config($vmid, $conf);
5580 my ($vmid, $skiplock, $nocheck) = @_;
5582 PVE
::QemuConfig-
>lock_config($vmid, sub {
5583 my $res = mon_cmd
($vmid, 'query-status');
5584 my $resume_cmd = 'cont';
5587 if ($res->{status
}) {
5588 return if $res->{status
} eq 'running'; # job done, go home
5589 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
5590 $reset = 1 if $res->{status
} eq 'shutdown';
5595 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5597 PVE
::QemuConfig-
>check_lock($conf)
5598 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5602 # required if a VM shuts down during a backup and we get a resume
5603 # request before the backup finishes for example
5604 mon_cmd
($vmid, "system_reset");
5606 mon_cmd
($vmid, $resume_cmd);
5611 my ($vmid, $skiplock, $key) = @_;
5613 PVE
::QemuConfig-
>lock_config($vmid, sub {
5615 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5617 # there is no qmp command, so we use the human monitor command
5618 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5619 die $res if $res ne '';
5623 # vzdump restore implementaion
5625 sub tar_archive_read_firstfile
{
5626 my $archive = shift;
5628 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5630 # try to detect archive type first
5631 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5632 die "unable to open file '$archive'\n";
5633 my $firstfile = <$fh>;
5637 die "ERROR: archive contaions no data\n" if !$firstfile;
5643 sub tar_restore_cleanup
{
5644 my ($storecfg, $statfile) = @_;
5646 print STDERR
"starting cleanup\n";
5648 if (my $fd = IO
::File-
>new($statfile, "r")) {
5649 while (defined(my $line = <$fd>)) {
5650 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5653 if ($volid =~ m
|^/|) {
5654 unlink $volid || die 'unlink failed\n';
5656 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5658 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5660 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5662 print STDERR
"unable to parse line in statfile - $line";
5669 sub restore_file_archive
{
5670 my ($archive, $vmid, $user, $opts) = @_;
5672 return restore_vma_archive
($archive, $vmid, $user, $opts)
5675 my $info = PVE
::Storage
::archive_info
($archive);
5676 my $format = $opts->{format
} // $info->{format
};
5677 my $comp = $info->{compression
};
5679 # try to detect archive format
5680 if ($format eq 'tar') {
5681 return restore_tar_archive
($archive, $vmid, $user, $opts);
5683 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5687 # hepler to remove disks that will not be used after restore
5688 my $restore_cleanup_oldconf = sub {
5689 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5691 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
5692 my ($ds, $drive) = @_;
5694 return if drive_is_cdrom
($drive, 1);
5696 my $volid = $drive->{file
};
5697 return if !$volid || $volid =~ m
|^/|;
5699 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
5700 return if !$path || !$owner || ($owner != $vmid);
5702 # Note: only delete disk we want to restore
5703 # other volumes will become unused
5704 if ($virtdev_hash->{$ds}) {
5705 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
5712 # delete vmstate files, after the restore we have no snapshots anymore
5713 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5714 my $snap = $oldconf->{snapshots
}->{$snapname};
5715 if ($snap->{vmstate
}) {
5716 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
5724 # Helper to parse vzdump backup device hints
5726 # $rpcenv: Environment, used to ckeck storage permissions
5727 # $user: User ID, to check storage permissions
5728 # $storecfg: Storage configuration
5729 # $fh: the file handle for reading the configuration
5730 # $devinfo: should contain device sizes for all backu-up'ed devices
5731 # $options: backup options (pool, default storage)
5733 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5734 my $parse_backup_hints = sub {
5735 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5737 my $virtdev_hash = {};
5739 while (defined(my $line = <$fh>)) {
5740 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5741 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5742 die "archive does not contain data for drive '$virtdev'\n"
5743 if !$devinfo->{$devname};
5745 if (defined($options->{storage
})) {
5746 $storeid = $options->{storage
} || 'local';
5747 } elsif (!$storeid) {
5750 $format = 'raw' if !$format;
5751 $devinfo->{$devname}->{devname
} = $devname;
5752 $devinfo->{$devname}->{virtdev
} = $virtdev;
5753 $devinfo->{$devname}->{format
} = $format;
5754 $devinfo->{$devname}->{storeid
} = $storeid;
5756 # check permission on storage
5757 my $pool = $options->{pool
}; # todo: do we need that?
5758 if ($user ne 'root@pam') {
5759 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5762 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5763 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5765 my $drive = parse_drive
($virtdev, $2);
5766 if (drive_is_cloudinit
($drive)) {
5767 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
5768 $storeid = $options->{storage
} if defined ($options->{storage
});
5769 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5770 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
5772 $virtdev_hash->{$virtdev} = {
5774 storeid
=> $storeid,
5775 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
5782 return $virtdev_hash;
5785 # Helper to allocate and activate all volumes required for a restore
5787 # $storecfg: Storage configuration
5788 # $virtdev_hash: as returned by parse_backup_hints()
5790 # Returns: { $virtdev => $volid }
5791 my $restore_allocate_devices = sub {
5792 my ($storecfg, $virtdev_hash, $vmid) = @_;
5795 foreach my $virtdev (sort keys %$virtdev_hash) {
5796 my $d = $virtdev_hash->{$virtdev};
5797 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5798 my $storeid = $d->{storeid
};
5799 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5801 # test if requested format is supported
5802 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5803 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
5804 $d->{format
} = $defFormat if !$supported;
5807 if ($d->{is_cloudinit
}) {
5808 $name = "vm-$vmid-cloudinit";
5809 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5810 if ($scfg->{path
}) {
5811 $name .= ".$d->{format}";
5815 my $volid = PVE
::Storage
::vdisk_alloc
(
5816 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
5818 print STDERR
"new volume ID is '$volid'\n";
5819 $d->{volid
} = $volid;
5821 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
5823 $map->{$virtdev} = $volid;
5829 my $restore_update_config_line = sub {
5830 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5832 return if $line =~ m/^\#qmdump\#/;
5833 return if $line =~ m/^\#vzdump\#/;
5834 return if $line =~ m/^lock:/;
5835 return if $line =~ m/^unused\d+:/;
5836 return if $line =~ m/^parent:/;
5838 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5839 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5840 # try to convert old 1.X settings
5841 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5842 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5843 my ($model, $macaddr) = split(/\=/, $devconfig);
5844 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5847 bridge
=> "vmbr$ind",
5848 macaddr
=> $macaddr,
5850 my $netstr = print_net
($net);
5852 print $outfd "net$cookie->{netcount}: $netstr\n";
5853 $cookie->{netcount
}++;
5855 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5856 my ($id, $netstr) = ($1, $2);
5857 my $net = parse_net
($netstr);
5858 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5859 $netstr = print_net
($net);
5860 print $outfd "$id: $netstr\n";
5861 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5864 my $di = parse_drive
($virtdev, $value);
5865 if (defined($di->{backup
}) && !$di->{backup
}) {
5866 print $outfd "#$line";
5867 } elsif ($map->{$virtdev}) {
5868 delete $di->{format
}; # format can change on restore
5869 $di->{file
} = $map->{$virtdev};
5870 $value = print_drive
($di);
5871 print $outfd "$virtdev: $value\n";
5875 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5877 if ($vmgenid ne '0') {
5878 # always generate a new vmgenid if there was a valid one setup
5879 $vmgenid = generate_uuid
();
5881 print $outfd "vmgenid: $vmgenid\n";
5882 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5883 my ($uuid, $uuid_str);
5884 UUID
::generate
($uuid);
5885 UUID
::unparse
($uuid, $uuid_str);
5886 my $smbios1 = parse_smbios1
($2);
5887 $smbios1->{uuid
} = $uuid_str;
5888 print $outfd $1.print_smbios1
($smbios1)."\n";
5894 my $restore_deactivate_volumes = sub {
5895 my ($storecfg, $devinfo) = @_;
5898 foreach my $devname (keys %$devinfo) {
5899 my $volid = $devinfo->{$devname}->{volid
};
5900 push @$vollist, $volid if $volid;
5903 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5906 my $restore_destroy_volumes = sub {
5907 my ($storecfg, $devinfo) = @_;
5909 foreach my $devname (keys %$devinfo) {
5910 my $volid = $devinfo->{$devname}->{volid
};
5913 if ($volid =~ m
|^/|) {
5914 unlink $volid || die 'unlink failed\n';
5916 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5918 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5920 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5925 my ($cfg, $vmid) = @_;
5927 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5929 my $volid_hash = {};
5930 foreach my $storeid (keys %$info) {
5931 foreach my $item (@{$info->{$storeid}}) {
5932 next if !($item->{volid
} && $item->{size
});
5933 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5934 $volid_hash->{$item->{volid
}} = $item;
5941 sub update_disk_config
{
5942 my ($vmid, $conf, $volid_hash) = @_;
5945 my $prefix = "VM $vmid";
5947 # used and unused disks
5948 my $referenced = {};
5950 # Note: it is allowed to define multiple storages with same path (alias), so
5951 # we need to check both 'volid' and real 'path' (two different volid can point
5952 # to the same path).
5954 my $referencedpath = {};
5957 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5958 my ($opt, $drive) = @_;
5960 my $volid = $drive->{file
};
5962 my $volume = $volid_hash->{$volid};
5964 # mark volid as "in-use" for next step
5965 $referenced->{$volid} = 1;
5966 if ($volume && (my $path = $volume->{path
})) {
5967 $referencedpath->{$path} = 1;
5970 return if drive_is_cdrom
($drive);
5973 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
5974 if (defined($updated)) {
5976 $conf->{$opt} = print_drive
($updated);
5977 print "$prefix ($opt): $msg\n";
5981 # remove 'unusedX' entry if volume is used
5982 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
5983 my ($opt, $drive) = @_;
5985 my $volid = $drive->{file
};
5989 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
5990 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5991 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5993 delete $conf->{$opt};
5996 $referenced->{$volid} = 1;
5997 $referencedpath->{$path} = 1 if $path;
6000 foreach my $volid (sort keys %$volid_hash) {
6001 next if $volid =~ m/vm-$vmid-state-/;
6002 next if $referenced->{$volid};
6003 my $path = $volid_hash->{$volid}->{path
};
6004 next if !$path; # just to be sure
6005 next if $referencedpath->{$path};
6007 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6008 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6009 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6016 my ($vmid, $nolock, $dryrun) = @_;
6018 my $cfg = PVE
::Storage
::config
();
6020 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6021 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6022 foreach my $stor (keys %{$cfg->{ids
}}) {
6023 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
6026 print "rescan volumes...\n";
6027 my $volid_hash = scan_volids
($cfg, $vmid);
6029 my $updatefn = sub {
6032 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6034 PVE
::QemuConfig-
>check_lock($conf);
6037 foreach my $volid (keys %$volid_hash) {
6038 my $info = $volid_hash->{$volid};
6039 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6042 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6044 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6047 if (defined($vmid)) {
6051 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6054 my $vmlist = config_list
();
6055 foreach my $vmid (keys %$vmlist) {
6059 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6065 sub restore_proxmox_backup_archive
{
6066 my ($archive, $vmid, $user, $options) = @_;
6068 my $storecfg = PVE
::Storage
::config
();
6070 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6071 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6073 my $fingerprint = $scfg->{fingerprint
};
6074 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6076 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6078 # This is only used for `pbs-restore`!
6079 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6080 local $ENV{PBS_PASSWORD
} = $password;
6081 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6083 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6084 PVE
::Storage
::parse_volname
($storecfg, $archive);
6086 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6088 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6090 my $tmpdir = "/var/tmp/vzdumptmp$$";
6094 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6095 my $tmpfn = "$conffile.$$.tmp";
6096 # disable interrupts (always do cleanups)
6100 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6102 # Note: $oldconf is undef if VM does not exists
6103 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6104 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6106 my $rpcenv = PVE
::RPCEnvironment
::get
();
6115 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6117 my $cfgfn = "$tmpdir/qemu-server.conf";
6118 my $firewall_config_fn = "$tmpdir/fw.conf";
6119 my $index_fn = "$tmpdir/index.json";
6121 my $cmd = "restore";
6123 my $param = [$pbs_backup_name, "index.json", $index_fn];
6124 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6125 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6126 $index = decode_json
($index);
6128 # print Dumper($index);
6129 foreach my $info (@{$index->{files
}}) {
6130 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6132 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6133 $devinfo->{$devname}->{size
} = $1;
6135 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6140 my $is_qemu_server_backup = scalar(
6141 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6143 if (!$is_qemu_server_backup) {
6144 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6146 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6148 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6149 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6151 if ($has_firewall_config) {
6152 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6153 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6155 my $pve_firewall_dir = '/etc/pve/firewall';
6156 mkdir $pve_firewall_dir; # make sure the dir exists
6157 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6160 my $fh = IO
::File-
>new($cfgfn, "r") ||
6161 die "unable to read qemu-server.conf - $!\n";
6163 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6165 # fixme: rate limit?
6167 # create empty/temp config
6168 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6170 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6173 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6175 foreach my $virtdev (sort keys %$virtdev_hash) {
6176 my $d = $virtdev_hash->{$virtdev};
6177 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6179 my $volid = $d->{volid
};
6181 my $path = PVE
::Storage
::path
($storecfg, $volid);
6183 # This is the ONLY user of the PBS_ env vars set on top of this function!
6184 my $pbs_restore_cmd = [
6185 '/usr/bin/pbs-restore',
6186 '--repository', $repo,
6188 "$d->{devname}.img.fidx",
6193 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6194 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6196 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6197 push @$pbs_restore_cmd, '--skip-zero';
6200 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6201 print "restore proxmox backup image: $dbg_cmdstring\n";
6202 run_command
($pbs_restore_cmd);
6205 $fh->seek(0, 0) || die "seek failed - $!\n";
6207 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6209 my $cookie = { netcount
=> 0 };
6210 while (defined(my $line = <$fh>)) {
6211 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique
});
6219 $restore_deactivate_volumes->($storecfg, $devinfo);
6225 $restore_destroy_volumes->($storecfg, $devinfo);
6229 rename($tmpfn, $conffile) ||
6230 die "unable to commit configuration file '$conffile'\n";
6232 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6234 eval { rescan
($vmid, 1); };
6238 sub restore_vma_archive
{
6239 my ($archive, $vmid, $user, $opts, $comp) = @_;
6241 my $readfrom = $archive;
6243 my $cfg = PVE
::Storage
::config
();
6245 my $bwlimit = $opts->{bwlimit
};
6247 my $dbg_cmdstring = '';
6248 my $add_pipe = sub {
6250 push @$commands, $cmd;
6251 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6252 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6257 if ($archive eq '-') {
6260 # If we use a backup from a PVE defined storage we also consider that
6261 # storage's rate limit:
6262 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6263 if (defined($volid)) {
6264 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6265 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6267 print STDERR
"applying read rate limit: $readlimit\n";
6268 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6269 $add_pipe->($cstream);
6275 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6276 my $cmd = $info->{decompressor
};
6277 push @$cmd, $readfrom;
6281 my $tmpdir = "/var/tmp/vzdumptmp$$";
6284 # disable interrupts (always do cleanups)
6288 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6290 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6291 POSIX
::mkfifo
($mapfifo, 0600);
6293 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6295 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6302 my $rpcenv = PVE
::RPCEnvironment
::get
();
6304 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6305 my $tmpfn = "$conffile.$$.tmp";
6307 # Note: $oldconf is undef if VM does not exist
6308 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6309 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6313 my $print_devmap = sub {
6314 my $cfgfn = "$tmpdir/qemu-server.conf";
6316 # we can read the config - that is already extracted
6317 my $fh = IO
::File-
>new($cfgfn, "r") ||
6318 die "unable to read qemu-server.conf - $!\n";
6320 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6322 my $pve_firewall_dir = '/etc/pve/firewall';
6323 mkdir $pve_firewall_dir; # make sure the dir exists
6324 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6327 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6329 foreach my $key (keys %storage_limits) {
6330 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6332 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6333 $storage_limits{$key} = $limit * 1024;
6336 foreach my $devname (keys %$devinfo) {
6337 die "found no device mapping information for device '$devname'\n"
6338 if !$devinfo->{$devname}->{virtdev
};
6341 # create empty/temp config
6343 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6344 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6348 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6350 # print restore information to $fifofh
6351 foreach my $virtdev (sort keys %$virtdev_hash) {
6352 my $d = $virtdev_hash->{$virtdev};
6353 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6355 my $storeid = $d->{storeid
};
6356 my $volid = $d->{volid
};
6359 if (my $limit = $storage_limits{$storeid}) {
6360 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6363 my $write_zeros = 1;
6364 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6368 my $path = PVE
::Storage
::path
($cfg, $volid);
6370 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6372 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6375 $fh->seek(0, 0) || die "seek failed - $!\n";
6377 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6379 my $cookie = { netcount
=> 0 };
6380 while (defined(my $line = <$fh>)) {
6381 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6394 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6395 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6397 $oldtimeout = alarm($timeout);
6404 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6405 my ($dev_id, $size, $devname) = ($1, $2, $3);
6406 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6407 } elsif ($line =~ m/^CTIME: /) {
6408 # we correctly received the vma config, so we can disable
6409 # the timeout now for disk allocation (set to 10 minutes, so
6410 # that we always timeout if something goes wrong)
6413 print $fifofh "done\n";
6414 my $tmp = $oldtimeout || 0;
6415 $oldtimeout = undef;
6422 print "restore vma archive: $dbg_cmdstring\n";
6423 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6427 alarm($oldtimeout) if $oldtimeout;
6429 $restore_deactivate_volumes->($cfg, $devinfo);
6431 close($fifofh) if $fifofh;
6437 $restore_destroy_volumes->($cfg, $devinfo);
6441 rename($tmpfn, $conffile) ||
6442 die "unable to commit configuration file '$conffile'\n";
6444 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6446 eval { rescan
($vmid, 1); };
6450 sub restore_tar_archive
{
6451 my ($archive, $vmid, $user, $opts) = @_;
6453 if ($archive ne '-') {
6454 my $firstfile = tar_archive_read_firstfile
($archive);
6455 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6456 if $firstfile ne 'qemu-server.conf';
6459 my $storecfg = PVE
::Storage
::config
();
6461 # avoid zombie disks when restoring over an existing VM -> cleanup first
6462 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6463 # skiplock=1 because qmrestore has set the 'create' lock itself already
6464 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6465 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6467 my $tocmd = "/usr/lib/qemu-server/qmextract";
6469 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6470 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6471 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6472 $tocmd .= ' --info' if $opts->{info
};
6474 # tar option "xf" does not autodetect compression when read from STDIN,
6475 # so we pipe to zcat
6476 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6477 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6479 my $tmpdir = "/var/tmp/vzdumptmp$$";
6482 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6483 local $ENV{VZDUMP_VMID
} = $vmid;
6484 local $ENV{VZDUMP_USER
} = $user;
6486 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6487 my $tmpfn = "$conffile.$$.tmp";
6489 # disable interrupts (always do cleanups)
6493 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6501 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6503 if ($archive eq '-') {
6504 print "extracting archive from STDIN\n";
6505 run_command
($cmd, input
=> "<&STDIN");
6507 print "extracting archive '$archive'\n";
6511 return if $opts->{info
};
6515 my $statfile = "$tmpdir/qmrestore.stat";
6516 if (my $fd = IO
::File-
>new($statfile, "r")) {
6517 while (defined (my $line = <$fd>)) {
6518 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6519 $map->{$1} = $2 if $1;
6521 print STDERR
"unable to parse line in statfile - $line\n";
6527 my $confsrc = "$tmpdir/qemu-server.conf";
6529 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6531 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6533 my $cookie = { netcount
=> 0 };
6534 while (defined (my $line = <$srcfd>)) {
6535 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6543 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6549 rename $tmpfn, $conffile ||
6550 die "unable to commit configuration file '$conffile'\n";
6552 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6554 eval { rescan
($vmid, 1); };
6558 sub foreach_storage_used_by_vm
{
6559 my ($conf, $func) = @_;
6563 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6564 my ($ds, $drive) = @_;
6565 return if drive_is_cdrom
($drive);
6567 my $volid = $drive->{file
};
6569 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6570 $sidhash->{$sid} = $sid if $sid;
6573 foreach my $sid (sort keys %$sidhash) {
6578 my $qemu_snap_storage = {
6581 sub do_snapshots_with_qemu
{
6582 my ($storecfg, $volid) = @_;
6584 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6585 my $scfg = $storecfg->{ids
}->{$storage_name};
6586 die "could not find storage '$storage_name'\n" if !defined($scfg);
6588 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6592 if ($volid =~ m/\.(qcow2|qed)$/){
6599 sub qga_check_running
{
6600 my ($vmid, $nowarn) = @_;
6602 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6604 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6610 sub template_create
{
6611 my ($vmid, $conf, $disk) = @_;
6613 my $storecfg = PVE
::Storage
::config
();
6615 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6616 my ($ds, $drive) = @_;
6618 return if drive_is_cdrom
($drive);
6619 return if $disk && $ds ne $disk;
6621 my $volid = $drive->{file
};
6622 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6624 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6625 $drive->{file
} = $voliddst;
6626 $conf->{$ds} = print_drive
($drive);
6627 PVE
::QemuConfig-
>write_config($vmid, $conf);
6631 sub convert_iscsi_path
{
6634 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6639 my $initiator_name = get_initiator_name
();
6641 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6642 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6645 die "cannot convert iscsi path '$path', unkown format\n";
6648 sub qemu_img_convert
{
6649 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6651 my $storecfg = PVE
::Storage
::config
();
6652 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6653 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6655 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6659 my $src_is_iscsi = 0;
6663 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6664 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6665 $src_format = qemu_img_format
($src_scfg, $src_volname);
6666 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6667 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6668 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6669 } elsif (-f
$src_volid) {
6670 $src_path = $src_volid;
6671 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6676 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6678 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6679 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6680 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6681 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6684 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6685 push @$cmd, '-l', "snapshot.name=$snapname"
6686 if $snapname && $src_format && $src_format eq "qcow2";
6687 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6688 push @$cmd, '-T', $cachemode if defined($cachemode);
6690 if ($src_is_iscsi) {
6691 push @$cmd, '--image-opts';
6692 $src_path = convert_iscsi_path
($src_path);
6693 } elsif ($src_format) {
6694 push @$cmd, '-f', $src_format;
6697 if ($dst_is_iscsi) {
6698 push @$cmd, '--target-image-opts';
6699 $dst_path = convert_iscsi_path
($dst_path);
6701 push @$cmd, '-O', $dst_format;
6704 push @$cmd, $src_path;
6706 if (!$dst_is_iscsi && $is_zero_initialized) {
6707 push @$cmd, "zeroinit:$dst_path";
6709 push @$cmd, $dst_path;
6714 if($line =~ m/\((\S+)\/100\
%\)/){
6716 my $transferred = int($size * $percent / 100);
6717 my $remaining = $size - $transferred;
6719 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6724 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6726 die "copy failed: $err" if $err;
6729 sub qemu_img_format
{
6730 my ($scfg, $volname) = @_;
6732 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6739 sub qemu_drive_mirror
{
6740 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6742 $jobs = {} if !$jobs;
6746 $jobs->{"drive-$drive"} = {};
6748 if ($dst_volid =~ /^nbd:/) {
6749 $qemu_target = $dst_volid;
6752 my $storecfg = PVE
::Storage
::config
();
6753 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6755 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6757 $format = qemu_img_format
($dst_scfg, $dst_volname);
6759 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6761 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6764 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6765 $opts->{format
} = $format if $format;
6767 if (defined($src_bitmap)) {
6768 $opts->{sync
} = 'incremental';
6769 $opts->{bitmap
} = $src_bitmap;
6770 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6773 if (defined($bwlimit)) {
6774 $opts->{speed
} = $bwlimit * 1024;
6775 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6777 print "drive mirror is starting for drive-$drive\n";
6780 # if a job already runs for this device we get an error, catch it for cleanup
6781 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
6783 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6785 die "mirroring error: $err\n";
6788 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
6791 # $completion can be either
6792 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6793 # 'cancel': wait until all jobs are ready, block-job-cancel them
6794 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6795 sub qemu_drive_mirror_monitor
{
6796 my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6798 $completion //= 'complete';
6801 my $err_complete = 0;
6804 die "storage migration timed out\n" if $err_complete > 300;
6806 my $stats = mon_cmd
($vmid, "query-block-jobs");
6808 my $running_mirror_jobs = {};
6809 foreach my $stat (@$stats) {
6810 next if $stat->{type
} ne 'mirror';
6811 $running_mirror_jobs->{$stat->{device
}} = $stat;
6814 my $readycounter = 0;
6816 foreach my $job (keys %$jobs) {
6818 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6819 print "$job : finished\n";
6820 delete $jobs->{$job};
6824 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6826 my $busy = $running_mirror_jobs->{$job}->{busy
};
6827 my $ready = $running_mirror_jobs->{$job}->{ready
};
6828 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6829 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6830 my $remaining = $total - $transferred;
6831 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6833 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6836 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6839 last if scalar(keys %$jobs) == 0;
6841 if ($readycounter == scalar(keys %$jobs)) {
6842 print "all mirroring jobs are ready \n";
6843 last if $completion eq 'skip'; #do the complete later
6845 if ($vmiddst && $vmiddst != $vmid) {
6846 my $agent_running = $qga && qga_check_running
($vmid);
6847 if ($agent_running) {
6848 print "freeze filesystem\n";
6849 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6851 print "suspend vm\n";
6852 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6855 # if we clone a disk for a new target vm, we don't switch the disk
6856 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6858 if ($agent_running) {
6859 print "unfreeze filesystem\n";
6860 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6862 print "resume vm\n";
6863 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6869 foreach my $job (keys %$jobs) {
6870 # try to switch the disk if source and destination are on the same guest
6871 print "$job: Completing block job...\n";
6874 if ($completion eq 'complete') {
6875 $op = 'block-job-complete';
6876 } elsif ($completion eq 'cancel') {
6877 $op = 'block-job-cancel';
6879 die "invalid completion value: $completion\n";
6881 eval { mon_cmd
($vmid, $op, device
=> $job) };
6882 if ($@ =~ m/cannot be completed/) {
6883 print "$job: Block job cannot be completed, try again.\n";
6886 print "$job: Completed successfully.\n";
6887 $jobs->{$job}->{complete
} = 1;
6898 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6899 die "mirroring error: $err";
6904 sub qemu_blockjobs_cancel
{
6905 my ($vmid, $jobs) = @_;
6907 foreach my $job (keys %$jobs) {
6908 print "$job: Cancelling block job\n";
6909 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6910 $jobs->{$job}->{cancel
} = 1;
6914 my $stats = mon_cmd
($vmid, "query-block-jobs");
6916 my $running_jobs = {};
6917 foreach my $stat (@$stats) {
6918 $running_jobs->{$stat->{device
}} = $stat;
6921 foreach my $job (keys %$jobs) {
6923 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6924 print "$job: Done.\n";
6925 delete $jobs->{$job};
6929 last if scalar(keys %$jobs) == 0;
6936 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6937 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6942 print "create linked clone of drive $drivename ($drive->{file})\n";
6943 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6944 push @$newvollist, $newvolid;
6947 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6948 $storeid = $storage if $storage;
6950 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6952 print "create full clone of drive $drivename ($drive->{file})\n";
6955 if (drive_is_cloudinit
($drive)) {
6956 $name = "vm-$newvmid-cloudinit";
6957 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6958 if ($scfg->{path
}) {
6959 $name .= ".$dst_format";
6962 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
6963 } elsif ($drivename eq 'efidisk0') {
6964 $size = get_efivars_size
($conf);
6966 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
6968 $newvolid = PVE
::Storage
::vdisk_alloc
(
6969 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
6971 push @$newvollist, $newvolid;
6973 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
6975 if (drive_is_cloudinit
($drive)) {
6976 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
6977 # if this is the case, we have to complete any block-jobs still there from
6978 # previous drive-mirrors
6979 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
6980 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
6985 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6986 if (!$running || $snapname) {
6987 # TODO: handle bwlimits
6988 if ($drivename eq 'efidisk0') {
6989 # the relevant data on the efidisk may be smaller than the source
6990 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6991 # that is given by the OVMF_VARS.fd
6992 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
6993 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
6995 # better for Ceph if block size is not too small, see bug #3324
6998 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
6999 "if=$src_path", "of=$dst_path"]);
7001 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
7005 my $kvmver = get_running_qemu_version
($vmid);
7006 if (!min_version
($kvmver, 2, 7)) {
7007 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7008 if $drive->{iothread
};
7011 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7012 $completion, $qga, $bwlimit);
7017 my ($size) = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
7020 $disk->{format
} = undef;
7021 $disk->{file
} = $newvolid;
7022 $disk->{size
} = $size if defined($size);
7027 sub get_running_qemu_version
{
7029 my $res = mon_cmd
($vmid, "query-version");
7030 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7033 sub qemu_use_old_bios_files
{
7034 my ($machine_type) = @_;
7036 return if !$machine_type;
7038 my $use_old_bios_files = undef;
7040 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7042 $use_old_bios_files = 1;
7044 my $version = extract_version
($machine_type, kvm_user_version
());
7045 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7046 # load new efi bios files on migration. So this hack is required to allow
7047 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7048 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7049 $use_old_bios_files = !min_version
($version, 2, 4);
7052 return ($use_old_bios_files, $machine_type);
7055 sub get_efivars_size
{
7057 my $arch = get_vm_arch
($conf);
7058 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7059 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
7060 return -s
$ovmf_vars;
7063 sub update_efidisk_size
{
7066 return if !defined($conf->{efidisk0
});
7068 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
7069 $disk->{size
} = get_efivars_size
($conf);
7070 $conf->{efidisk0
} = print_drive
($disk);
7075 sub create_efidisk
($$$$$) {
7076 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7078 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7079 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7081 my $vars_size_b = -s
$ovmf_vars;
7082 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7083 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7084 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7086 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7087 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7089 return ($volid, $size/1024);
7092 sub vm_iothreads_list
{
7095 my $res = mon_cmd
($vmid, 'query-iothreads');
7098 foreach my $iothread (@$res) {
7099 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7106 my ($conf, $drive) = @_;
7110 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7112 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7118 my $controller = int($drive->{index} / $maxdev);
7119 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
7123 return ($maxdev, $controller, $controller_prefix);
7126 sub windows_version
{
7129 return 0 if !$ostype;
7133 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7135 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7137 } elsif ($ostype =~ m/^win(\d+)$/) {
7144 sub resolve_dst_disk_format
{
7145 my ($storecfg, $storeid, $src_volname, $format) = @_;
7146 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7149 # if no target format is specified, use the source disk format as hint
7151 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7152 $format = qemu_img_format
($scfg, $src_volname);
7158 # test if requested format is supported - else use default
7159 my $supported = grep { $_ eq $format } @$validFormats;
7160 $format = $defFormat if !$supported;
7164 # NOTE: if this logic changes, please update docs & possibly gui logic
7165 sub find_vmstate_storage
{
7166 my ($conf, $storecfg) = @_;
7168 # first, return storage from conf if set
7169 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7171 my ($target, $shared, $local);
7173 foreach_storage_used_by_vm
($conf, sub {
7175 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7176 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7177 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7180 # second, use shared storage where VM has at least one disk
7181 # third, use local storage where VM has at least one disk
7182 # fall back to local storage
7183 $target = $shared // $local // 'local';
7189 my ($uuid, $uuid_str);
7190 UUID
::generate
($uuid);
7191 UUID
::unparse
($uuid, $uuid_str);
7195 sub generate_smbios1_uuid
{
7196 return "uuid=".generate_uuid
();
7202 mon_cmd
($vmid, 'nbd-server-stop');
7205 sub create_reboot_request
{
7207 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7208 or die "failed to create reboot trigger file: $!\n";
7212 sub clear_reboot_request
{
7214 my $path = "/run/qemu-server/$vmid.reboot";
7217 $res = unlink($path);
7218 die "could not remove reboot request for $vmid: $!"
7219 if !$res && $! != POSIX
::ENOENT
;
7224 sub bootorder_from_legacy
{
7225 my ($conf, $bootcfg) = @_;
7227 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
7228 my $bootindex_hash = {};
7230 foreach my $o (split(//, $boot)) {
7231 $bootindex_hash->{$o} = $i*100;
7237 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7238 my ($ds, $drive) = @_;
7240 if (drive_is_cdrom
($drive, 1)) {
7241 if ($bootindex_hash->{d
}) {
7242 $bootorder->{$ds} = $bootindex_hash->{d
};
7243 $bootindex_hash->{d
} += 1;
7245 } elsif ($bootindex_hash->{c
}) {
7246 $bootorder->{$ds} = $bootindex_hash->{c
}
7247 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
7248 $bootindex_hash->{c
} += 1;
7252 if ($bootindex_hash->{n
}) {
7253 for (my $i = 0; $i < $MAX_NETS; $i++) {
7254 my $netname = "net$i";
7255 next if !$conf->{$netname};
7256 $bootorder->{$netname} = $bootindex_hash->{n
};
7257 $bootindex_hash->{n
} += 1;
7264 # Generate default device list for 'boot: order=' property. Matches legacy
7265 # default boot order, but with explicit device names. This is important, since
7266 # the fallback for when neither 'order' nor the old format is specified relies
7267 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7268 sub get_default_bootdevices
{
7274 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
7275 push @ret, $first if $first;
7278 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
7279 push @ret, $first if $first;
7282 for (my $i = 0; $i < $MAX_NETS; $i++) {
7283 my $netname = "net$i";
7284 next if !$conf->{$netname};
7285 push @ret, $netname;
7292 sub device_bootorder
{
7295 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
7297 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
7300 if (!defined($boot) || $boot->{legacy
}) {
7301 $bootorder = bootorder_from_legacy
($conf, $boot);
7302 } elsif ($boot->{order
}) {
7303 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7304 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
7305 $bootorder->{$dev} = $i++;
7312 # bash completion helper
7314 sub complete_backup_archives
{
7315 my ($cmdname, $pname, $cvalue) = @_;
7317 my $cfg = PVE
::Storage
::config
();
7321 if ($cvalue =~ m/^([^:]+):/) {
7325 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7328 foreach my $id (keys %$data) {
7329 foreach my $item (@{$data->{$id}}) {
7330 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7331 push @$res, $item->{volid
} if defined($item->{volid
});
7338 my $complete_vmid_full = sub {
7341 my $idlist = vmstatus
();
7345 foreach my $id (keys %$idlist) {
7346 my $d = $idlist->{$id};
7347 if (defined($running)) {
7348 next if $d->{template
};
7349 next if $running && $d->{status
} ne 'running';
7350 next if !$running && $d->{status
} eq 'running';
7359 return &$complete_vmid_full();
7362 sub complete_vmid_stopped
{
7363 return &$complete_vmid_full(0);
7366 sub complete_vmid_running
{
7367 return &$complete_vmid_full(1);
7370 sub complete_storage
{
7372 my $cfg = PVE
::Storage
::config
();
7373 my $ids = $cfg->{ids
};
7376 foreach my $sid (keys %$ids) {
7377 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7378 next if !$ids->{$sid}->{content
}->{images
};
7385 sub complete_migration_storage
{
7386 my ($cmd, $param, $current_value, $all_args) = @_;
7388 my $targetnode = @$all_args[1];
7390 my $cfg = PVE
::Storage
::config
();
7391 my $ids = $cfg->{ids
};
7394 foreach my $sid (keys %$ids) {
7395 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7396 next if !$ids->{$sid}->{content
}->{images
};
7405 my $qmpstatus = eval {
7406 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
7407 mon_cmd
($vmid, "query-status");
7410 return $qmpstatus && $qmpstatus->{status
} eq "paused";