1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
31 use PVE
::DataCenterConfig
;
32 use PVE
::Exception
qw(raise raise_param_exc);
33 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
35 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
38 use PVE
::RPCEnvironment
;
42 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
46 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
47 use PVE
::QemuServer
::Cloudinit
;
48 use PVE
::QemuServer
::CGroup
;
49 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
50 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
51 use PVE
::QemuServer
::Machine
;
52 use PVE
::QemuServer
::Memory
;
53 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
54 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
55 use PVE
::QemuServer
::USB
qw(parse_usb_device);
59 require PVE
::Network
::SDN
::Zones
;
63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
66 "$EDK2_FW_BASE/OVMF_CODE.fd",
67 "$EDK2_FW_BASE/OVMF_VARS.fd"
70 "$EDK2_FW_BASE/AAVMF_CODE.fd",
71 "$EDK2_FW_BASE/AAVMF_VARS.fd"
75 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
77 # Note about locking: we use flock on the config file protect
78 # against concurent actions.
79 # Aditionaly, we have a 'lock' setting in the config file. This
80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
81 # allowed when such lock is set. But you can ignore this kind of
82 # lock with the --skiplock flag.
84 cfs_register_file
('/qemu-server/',
88 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
89 description
=> "Some command save/restore state from this location.",
95 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
96 description
=> "Specifies the Qemu machine type.",
98 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
105 my ($map, $source) = @_;
107 return $source if !defined($map);
109 return $map->{entries
}->{$source}
110 if $map->{entries
} && defined($map->{entries
}->{$source});
112 return $map->{default} if $map->{default};
114 # identity (fallback)
118 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
119 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
121 format
=> 'storagepair-list',
125 #no warnings 'redefine';
129 $nodename_cache //= PVE
::INotify
::nodename
();
130 return $nodename_cache;
137 enum
=> [qw(i6300esb ib700)],
138 description
=> "Watchdog type to emulate.",
139 default => 'i6300esb',
144 enum
=> [qw(reset shutdown poweroff pause debug none)],
145 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
149 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
153 description
=> "Enable/disable Qemu GuestAgent.",
158 fstrim_cloned_disks
=> {
159 description
=> "Run fstrim after moving a disk or migrating the VM.",
165 description
=> "Select the agent type",
169 enum
=> [qw(virtio isa)],
175 description
=> "Select the VGA type.",
180 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
183 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
195 description
=> "The size of the file in MB.",
199 pattern
=> '[a-zA-Z0-9\-]+',
201 format_description
=> 'string',
202 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
209 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
210 description
=> "Configure an audio device."
214 enum
=> ['spice', 'none'],
217 description
=> "Driver backend for the audio device."
221 my $spice_enhancements_fmt = {
226 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
230 enum
=> ['off', 'all', 'filter'],
233 description
=> "Enable video streaming. Uses compression for detected video streams."
240 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
242 description
=> "The file on the host to gather entropy from. In most"
243 . " cases /dev/urandom should be preferred over /dev/random"
244 . " to avoid entropy-starvation issues on the host. Using"
245 . " urandom does *not* decrease security in any meaningful"
246 . " way, as it's still seeded from real entropy, and the"
247 . " bytes provided will most likely be mixed with real"
248 . " entropy on the guest as well. /dev/hwrng can be used"
249 . " to pass through a hardware RNG from the host.",
253 description
=> "Maximum bytes of entropy injected into the guest every"
254 . " 'period' milliseconds. Prefer a lower value when using"
255 . " /dev/random as source. Use 0 to disable limiting"
256 . " (potentially dangerous!).",
259 # default is 1 KiB/s, provides enough entropy to the guest to avoid
260 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
261 # of overwhelming the host, provided we're reading from /dev/urandom
266 description
=> "Every 'period' milliseconds the entropy-injection quota"
267 . " is reset, allowing the guest to retrieve another"
268 . " 'max_bytes' of entropy.",
278 description
=> "Specifies whether a VM will be started during system bootup.",
284 description
=> "Automatic restart after crash (currently ignored).",
289 type
=> 'string', format
=> 'pve-hotplug-features',
290 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
291 default => 'network,disk,usb',
296 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
302 description
=> "Lock/unlock the VM.",
303 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
308 description
=> "Limit of CPU usage.",
309 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
317 description
=> "CPU weight for a VM.",
318 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
326 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
333 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
339 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
347 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
348 "It should not be necessary to set it.",
349 enum
=> PVE
::Tools
::kvmkeymaplist
(),
354 type
=> 'string', format
=> 'dns-name',
355 description
=> "Set a name for the VM. Only used on the configuration web interface.",
360 description
=> "SCSI controller model",
361 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
367 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
372 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
373 description
=> "Specify guest operating system.",
374 verbose_description
=> <<EODESC,
375 Specify guest operating system. This is used to enable special
376 optimization/features for specific operating systems:
379 other;; unspecified OS
380 wxp;; Microsoft Windows XP
381 w2k;; Microsoft Windows 2000
382 w2k3;; Microsoft Windows 2003
383 w2k8;; Microsoft Windows 2008
384 wvista;; Microsoft Windows Vista
385 win7;; Microsoft Windows 7
386 win8;; Microsoft Windows 8/2012/2012r2
387 win10;; Microsoft Windows 10/2016
388 l24;; Linux 2.4 Kernel
389 l26;; Linux 2.6 - 5.X Kernel
390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
395 type
=> 'string', format
=> 'pve-qm-boot',
396 description
=> "Specify guest boot order. Use with 'order=', usage with"
397 . " no key or 'legacy=' is deprecated.",
401 type
=> 'string', format
=> 'pve-qm-bootdisk',
402 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
403 pattern
=> '(ide|sata|scsi|virtio)\d+',
408 description
=> "The number of CPUs. Please use option -sockets instead.",
415 description
=> "The number of CPU sockets.",
422 description
=> "The number of cores per socket.",
429 description
=> "Enable/disable NUMA.",
435 description
=> "Enable/disable hugepages memory.",
436 enum
=> [qw(any 2 1024)],
442 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
443 ." after VM shutdown and can be used for subsequent starts.",
448 description
=> "Number of hotplugged vcpus.",
455 description
=> "Enable/disable ACPI.",
460 description
=> "Enable/disable Qemu GuestAgent and its properties.",
462 format
=> $agent_fmt,
467 description
=> "Enable/disable KVM hardware virtualization.",
473 description
=> "Enable/disable time drift fix.",
479 description
=> "Set the real time clock to local time. This is enabled by default if ostype"
480 ." indicates a Microsoft OS.",
485 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
489 type
=> 'string', format
=> $vga_fmt,
490 description
=> "Configure the VGA hardware.",
491 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
492 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
493 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
494 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
495 ." display server. For win* OS you can select how many independent displays you want,"
496 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
497 ." using a serial device as terminal.",
501 type
=> 'string', format
=> 'pve-qm-watchdog',
502 description
=> "Create a virtual hardware watchdog device.",
503 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
504 ." action), the watchdog must be periodically polled by an agent inside the guest or"
505 ." else the watchdog will reset the guest (or execute the respective action specified)",
510 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
511 description
=> "Set the initial date of the real time clock. Valid format for date are:"
512 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
513 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
516 startup
=> get_standard_option
('pve-startup-order'),
520 description
=> "Enable/disable Template.",
526 description
=> "Arbitrary arguments passed to kvm.",
527 verbose_description
=> <<EODESCR,
528 Arbitrary arguments passed to kvm, for example:
530 args: -no-reboot -no-hpet
532 NOTE: this option is for experts only.
539 description
=> "Enable/disable the USB tablet device.",
540 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
541 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
542 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
543 ." may consider disabling this to save some context switches. This is turned off by"
544 ." default if you use spice (`qm set <vmid> --vga qxl`).",
549 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
553 migrate_downtime
=> {
556 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
562 type
=> 'string', format
=> 'pve-qm-ide',
563 typetext
=> '<volume>',
564 description
=> "This is an alias for option -ide2",
568 description
=> "Emulated CPU type.",
570 format
=> 'pve-vm-cpu-conf',
572 parent
=> get_standard_option
('pve-snapshot-name', {
574 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
578 description
=> "Timestamp for snapshots.",
584 type
=> 'string', format
=> 'pve-volume-id',
585 description
=> "Reference to a volume which stores the VM state. This is used internally"
588 vmstatestorage
=> get_standard_option
('pve-storage-id', {
589 description
=> "Default storage for VM state volumes/files.",
592 runningmachine
=> get_standard_option
('pve-qemu-machine', {
593 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
597 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
598 ." internally for snapshots.",
601 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
602 format_description
=> 'QEMU -cpu parameter'
604 machine
=> get_standard_option
('pve-qemu-machine'),
606 description
=> "Virtual processor architecture. Defaults to the host.",
609 enum
=> [qw(x86_64 aarch64)],
612 description
=> "Specify SMBIOS type 1 fields.",
613 type
=> 'string', format
=> 'pve-qm-smbios1',
620 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
621 ." remove disk operations.",
627 enum
=> [ qw(seabios ovmf) ],
628 description
=> "Select BIOS implementation.",
629 default => 'seabios',
633 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
634 format_description
=> 'UUID',
635 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
636 ." to disable explicitly.",
637 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
638 ." value identifier to the guest OS. This allows to notify the guest operating system"
639 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
640 ." execution or creation from a template). The guest operating system notices the"
641 ." change, and is then able to react as appropriate by marking its copies of"
642 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
643 ."Note that auto-creation only works when done through API/CLI create or update methods"
644 .", but not when manually editing the config file.",
645 default => "1 (autogenerated)",
650 format
=> 'pve-volume-id',
652 description
=> "Script that will be executed during various steps in the vms lifetime.",
656 format
=> $ivshmem_fmt,
657 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
663 format
=> $audio_fmt,
664 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
667 spice_enhancements
=> {
669 format
=> $spice_enhancements_fmt,
670 description
=> "Configure additional enhancements for SPICE.",
674 type
=> 'string', format
=> 'pve-tag-list',
675 description
=> 'Tags of the VM. This is only meta information.',
681 description
=> "Configure a VirtIO-based Random Number Generator.",
690 description
=> 'Specify a custom file containing all meta data passed to the VM via"
691 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
692 format
=> 'pve-volume-id',
693 format_description
=> 'volume',
698 description
=> 'Specify a custom file containing all network data passed to the VM via'
700 format
=> 'pve-volume-id',
701 format_description
=> 'volume',
706 description
=> 'Specify a custom file containing all user data passed to the VM via'
708 format
=> 'pve-volume-id',
709 format_description
=> 'volume',
712 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
714 my $confdesc_cloudinit = {
718 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
719 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
720 .' and `configdrive2` for windows.',
721 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
726 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
727 ." image's configured default user.",
732 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
733 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
734 .' support hashed passwords.',
739 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
741 format
=> 'pve-qm-cicustom',
746 description
=> "cloud-init: Sets DNS search domains for a container. Create will'
747 .' automatically use the setting from the host if neither searchdomain nor nameserver'
752 type
=> 'string', format
=> 'address-list',
753 description
=> "cloud-init: Sets DNS server IP address for a container. Create will'
754 .' automatically use the setting from the host if neither searchdomain nor nameserver'
760 format
=> 'urlencoded',
761 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
765 # what about other qemu settings ?
767 #machine => 'string',
780 ##soundhw => 'string',
782 while (my ($k, $v) = each %$confdesc) {
783 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
786 my $MAX_USB_DEVICES = 5;
788 my $MAX_SERIAL_PORTS = 4;
789 my $MAX_PARALLEL_PORTS = 3;
795 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
796 description
=> "CPUs accessing this NUMA node.",
797 format_description
=> "id[-id];...",
801 description
=> "Amount of memory this NUMA node provides.",
806 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
807 description
=> "Host NUMA nodes to use.",
808 format_description
=> "id[-id];...",
813 enum
=> [qw(preferred bind interleave)],
814 description
=> "NUMA allocation policy.",
818 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
821 type
=> 'string', format
=> $numa_fmt,
822 description
=> "NUMA topology.",
824 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
826 for (my $i = 0; $i < $MAX_NUMA; $i++) {
827 $confdesc->{"numa$i"} = $numadesc;
830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
831 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
832 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
835 my $net_fmt_bridge_descr = <<__EOD__;
836 Bridge to attach the network device to. The Proxmox VE standard bridge
839 If you do not specify a bridge, we create a kvm user (NATed) network
840 device, which provides DHCP and DNS services. The following addresses
847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
851 macaddr
=> get_standard_option
('mac-addr', {
852 description
=> "MAC address. That address must be unique withing your network. This is"
853 ." automatically generated if not specified.",
857 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
858 ." very low CPU overhead. If your guest does not support this driver, it is usually"
859 ." best to use 'e1000'.",
860 enum
=> $nic_model_list,
863 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
866 description
=> $net_fmt_bridge_descr,
867 format_description
=> 'bridge',
868 pattern
=> '[-_.\w\d]+',
873 minimum
=> 0, maximum
=> 16,
874 description
=> 'Number of packet queues to be used on the device.',
880 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
885 minimum
=> 1, maximum
=> 4094,
886 description
=> 'VLAN tag to apply to packets on this interface.',
891 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
892 description
=> 'VLAN trunks to pass through this interface.',
893 format_description
=> 'vlanid[;vlanid...]',
898 description
=> 'Whether this interface should be protected by the firewall.',
903 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
908 minimum
=> 1, maximum
=> 65520,
909 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
916 type
=> 'string', format
=> $net_fmt,
917 description
=> "Specify network devices.",
920 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
925 format
=> 'pve-ipv4-config',
926 format_description
=> 'IPv4Format/CIDR',
927 description
=> 'IPv4 address in CIDR format.',
934 format_description
=> 'GatewayIPv4',
935 description
=> 'Default gateway for IPv4 traffic.',
941 format
=> 'pve-ipv6-config',
942 format_description
=> 'IPv6Format/CIDR',
943 description
=> 'IPv6 address in CIDR format.',
950 format_description
=> 'GatewayIPv6',
951 description
=> 'Default gateway for IPv6 traffic.',
956 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
959 type
=> 'string', format
=> 'pve-qm-ipconfig',
960 description
=> <<'EODESCR',
961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
966 gateway should be provided.
967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
973 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
975 for (my $i = 0; $i < $MAX_NETS; $i++) {
976 $confdesc->{"net$i"} = $netdesc;
977 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
980 foreach my $key (keys %$confdesc_cloudinit) {
981 $confdesc->{$key} = $confdesc_cloudinit->{$key};
984 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
985 sub verify_volume_id_or_qm_path
{
986 my ($volid, $noerr) = @_;
988 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
992 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
993 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1004 type
=> 'string', format
=> 'pve-qm-usb-device',
1005 format_description
=> 'HOSTUSBDEVICE|spice',
1006 description
=> <<EODESCR,
1007 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1009 'bus-port(.port)*' (decimal numbers) or
1010 'vendor_id:product_id' (hexadeciaml numbers) or
1013 You can use the 'lsusb -t' command to list existing usb devices.
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1016 machines - use with special care.
1018 The value 'spice' can be used to add a usb redirection devices for spice.
1024 description
=> "Specifies whether if given host option is a USB3 device or port.",
1031 type
=> 'string', format
=> $usb_fmt,
1032 description
=> "Configure an USB device (n is 0 to 4).",
1034 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1039 pattern
=> '(/dev/.+|socket)',
1040 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1041 verbose_description
=> <<EODESCR,
1042 Create a serial device inside the VM (n is 0 to 3), and pass through a
1043 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1044 host side (use 'qm terminal' to open a terminal connection).
1046 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1047 use with special care.
1049 CAUTION: Experimental! User reported problems with this option.
1056 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1057 description
=> "Map host parallel devices (n is 0 to 2).",
1058 verbose_description
=> <<EODESCR,
1059 Map host parallel devices (n is 0 to 2).
1061 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1062 machines - use with special care.
1064 CAUTION: Experimental! User reported problems with this option.
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1069 $confdesc->{"parallel$i"} = $paralleldesc;
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1073 $confdesc->{"serial$i"} = $serialdesc;
1076 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1077 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1080 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1081 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1084 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1085 $confdesc->{"usb$i"} = $usbdesc;
1093 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1094 . " Deprecated, use 'order=' instead.",
1095 pattern
=> '[acdn]{1,4}',
1096 format_description
=> "[acdn]{1,4}",
1098 # note: this is also the fallback if boot: is not given at all
1104 format
=> 'pve-qm-bootdev-list',
1105 format_description
=> "device[;device...]",
1106 description
=> <<EODESC,
1107 The guest will attempt to boot from devices in the order they appear here.
1109 Disks, optical drives and passed-through storage USB devices will be directly
1110 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1111 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1113 Note that only devices in this list will be marked as bootable and thus loaded
1114 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1115 (e.g. software-raid), you need to specify all of them here.
1117 Overrides the deprecated 'legacy=[acdn]*' value when given.
1121 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1123 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1124 sub verify_bootdev
{
1125 my ($dev, $noerr) = @_;
1127 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && $dev !~ m/^efidisk/;
1131 return 0 if $dev !~ m/^$base\d+$/;
1132 return 0 if !$confdesc->{$dev};
1136 return $dev if $check->("net");
1137 return $dev if $check->("usb");
1138 return $dev if $check->("hostpci");
1141 die "invalid boot device '$dev'\n";
1144 sub print_bootorder
{
1146 return "" if !@$devs;
1147 my $data = { order
=> join(';', @$devs) };
1148 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1151 my $kvm_api_version = 0;
1154 return $kvm_api_version if $kvm_api_version;
1156 open my $fh, '<', '/dev/kvm' or return;
1158 # 0xae00 => KVM_GET_API_VERSION
1159 $kvm_api_version = ioctl($fh, 0xae00, 0);
1162 return $kvm_api_version;
1165 my $kvm_user_version = {};
1168 sub kvm_user_version
{
1171 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1172 my $st = stat($binary);
1174 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1175 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1176 $cachedmtime == $st->mtime;
1178 $kvm_user_version->{$binary} = 'unknown';
1179 $kvm_mtime->{$binary} = $st->mtime;
1183 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1184 $kvm_user_version->{$binary} = $2;
1188 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1191 return $kvm_user_version->{$binary};
1194 my sub extract_version
{
1195 my ($machine_type, $version) = @_;
1196 $version = kvm_user_version
() if !defined($version);
1197 PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1200 sub kernel_has_vhost_net
{
1201 return -c
'/dev/vhost-net';
1206 return defined($confdesc->{$key});
1210 sub get_cdrom_path
{
1212 return $cdrom_path if $cdrom_path;
1214 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1215 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1216 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1220 my ($storecfg, $vmid, $cdrom) = @_;
1222 if ($cdrom eq 'cdrom') {
1223 return get_cdrom_path
();
1224 } elsif ($cdrom eq 'none') {
1226 } elsif ($cdrom =~ m
|^/|) {
1229 return PVE
::Storage
::path
($storecfg, $cdrom);
1233 # try to convert old style file names to volume IDs
1234 sub filename_to_volume_id
{
1235 my ($vmid, $file, $media) = @_;
1237 if (!($file eq 'none' || $file eq 'cdrom' ||
1238 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1240 return if $file =~ m
|/|;
1242 if ($media && $media eq 'cdrom') {
1243 $file = "local:iso/$file";
1245 $file = "local:$vmid/$file";
1252 sub verify_media_type
{
1253 my ($opt, $vtype, $media) = @_;
1258 if ($media eq 'disk') {
1260 } elsif ($media eq 'cdrom') {
1263 die "internal error";
1266 return if ($vtype eq $etype);
1268 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1271 sub cleanup_drive_path
{
1272 my ($opt, $storecfg, $drive) = @_;
1274 # try to convert filesystem paths to volume IDs
1276 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1277 ($drive->{file
} !~ m
|^/dev/.+|) &&
1278 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1279 ($drive->{file
} !~ m/^\d+$/)) {
1280 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1281 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1283 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1284 verify_media_type
($opt, $vtype, $drive->{media
});
1285 $drive->{file
} = $volid;
1288 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1291 sub parse_hotplug_features
{
1296 return $res if $data eq '0';
1298 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1300 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1301 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1304 die "invalid hotplug feature '$feature'\n";
1310 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1311 sub pve_verify_hotplug_features
{
1312 my ($value, $noerr) = @_;
1314 return $value if parse_hotplug_features
($value);
1318 die "unable to parse hotplug option\n";
1322 my($fh, $noerr) = @_;
1325 my $SG_GET_VERSION_NUM = 0x2282;
1327 my $versionbuf = "\x00" x
8;
1328 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1330 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1333 my $version = unpack("I", $versionbuf);
1334 if ($version < 30000) {
1335 die "scsi generic interface too old\n" if !$noerr;
1339 my $buf = "\x00" x
36;
1340 my $sensebuf = "\x00" x
8;
1341 my $cmd = pack("C x3 C x1", 0x12, 36);
1343 # see /usr/include/scsi/sg.h
1344 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1346 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1347 length($sensebuf), 0, length($buf), $buf,
1348 $cmd, $sensebuf, 6000);
1350 $ret = ioctl($fh, $SG_IO, $packet);
1352 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1356 my @res = unpack($sg_io_hdr_t, $packet);
1357 if ($res[17] || $res[18]) {
1358 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363 (my $byte0, my $byte1, $res->{vendor
},
1364 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1366 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1367 $res->{type
} = $byte0 & 31;
1375 my $fh = IO
::File-
>new("+<$path") || return;
1376 my $res = scsi_inquiry
($fh, 1);
1382 sub print_tabletdevice_full
{
1383 my ($conf, $arch) = @_;
1385 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1387 # we use uhci for old VMs because tablet driver was buggy in older qemu
1389 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1395 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1398 sub print_keyboarddevice_full
{
1399 my ($conf, $arch, $machine) = @_;
1401 return if $arch ne 'aarch64';
1403 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1406 sub print_drivedevice_full
{
1407 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1412 my $drive_id = "$drive->{interface}$drive->{index}";
1413 if ($drive->{interface
} eq 'virtio') {
1414 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1415 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1416 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1417 } elsif ($drive->{interface
} eq 'scsi') {
1419 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1420 my $unit = $drive->{index} % $maxdev;
1421 my $devicetype = 'hd';
1423 if (drive_is_cdrom
($drive)) {
1426 if ($drive->{file
} =~ m
|^/|) {
1427 $path = $drive->{file
};
1428 if (my $info = path_is_scsi
($path)) {
1429 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1430 $devicetype = 'block';
1431 } elsif ($info->{type
} == 1) { # tape
1432 $devicetype = 'generic';
1436 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1439 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1440 my $version = extract_version
($machine_type, kvm_user_version
());
1441 if ($path =~ m/^iscsi\:\/\
// &&
1442 !min_version
($version, 4, 1)) {
1443 $devicetype = 'generic';
1447 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1448 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1450 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1451 .",lun=$drive->{index}";
1453 $device .= ",drive=drive-$drive_id,id=$drive_id";
1455 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1456 $device .= ",rotation_rate=1";
1458 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1460 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1461 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1462 my $controller = int($drive->{index} / $maxdev);
1463 my $unit = $drive->{index} % $maxdev;
1464 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1466 $device = "ide-$devicetype";
1467 if ($drive->{interface
} eq 'ide') {
1468 $device .= ",bus=ide.$controller,unit=$unit";
1470 $device .= ",bus=ahci$controller.$unit";
1472 $device .= ",drive=drive-$drive_id,id=$drive_id";
1474 if ($devicetype eq 'hd') {
1475 if (my $model = $drive->{model
}) {
1476 $model = URI
::Escape
::uri_unescape
($model);
1477 $device .= ",model=$model";
1479 if ($drive->{ssd
}) {
1480 $device .= ",rotation_rate=1";
1483 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1484 } elsif ($drive->{interface
} eq 'usb') {
1486 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1488 die "unsupported interface type";
1491 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1493 if (my $serial = $drive->{serial
}) {
1494 $serial = URI
::Escape
::uri_unescape
($serial);
1495 $device .= ",serial=$serial";
1502 sub get_initiator_name
{
1505 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1506 while (defined(my $line = <$fh>)) {
1507 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1516 sub print_drive_commandline_full
{
1517 my ($storecfg, $vmid, $drive) = @_;
1520 my $volid = $drive->{file
};
1523 if (drive_is_cdrom
($drive)) {
1524 $path = get_iso_path
($storecfg, $vmid, $volid);
1526 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1528 $path = PVE
::Storage
::path
($storecfg, $volid);
1529 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1530 $format = qemu_img_format
($scfg, $volname);
1538 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1539 foreach my $o (@qemu_drive_options) {
1540 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1543 # snapshot only accepts on|off
1544 if (defined($drive->{snapshot
})) {
1545 my $v = $drive->{snapshot
} ?
'on' : 'off';
1546 $opts .= ",snapshot=$v";
1549 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1550 my ($dir, $qmpname) = @$type;
1551 if (my $v = $drive->{"mbps$dir"}) {
1552 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1554 if (my $v = $drive->{"mbps${dir}_max"}) {
1555 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1557 if (my $v = $drive->{"bps${dir}_max_length"}) {
1558 $opts .= ",throttling.bps$qmpname-max-length=$v";
1560 if (my $v = $drive->{"iops${dir}"}) {
1561 $opts .= ",throttling.iops$qmpname=$v";
1563 if (my $v = $drive->{"iops${dir}_max"}) {
1564 $opts .= ",throttling.iops$qmpname-max=$v";
1566 if (my $v = $drive->{"iops${dir}_max_length"}) {
1567 $opts .= ",throttling.iops$qmpname-max-length=$v";
1571 $opts .= ",format=$format" if $format && !$drive->{format
};
1573 my $cache_direct = 0;
1575 if (my $cache = $drive->{cache
}) {
1576 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1577 } elsif (!drive_is_cdrom
($drive)) {
1578 $opts .= ",cache=none";
1582 # aio native works only with O_DIRECT
1583 if (!$drive->{aio
}) {
1585 $opts .= ",aio=native";
1587 $opts .= ",aio=threads";
1591 if (!drive_is_cdrom
($drive)) {
1593 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1594 $detectzeroes = 'off';
1595 } elsif ($drive->{discard
}) {
1596 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1598 # This used to be our default with discard not being specified:
1599 $detectzeroes = 'on';
1601 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1604 my $pathinfo = $path ?
"file=$path," : '';
1606 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1609 sub print_netdevice_full
{
1610 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1612 my $device = $net->{model
};
1613 if ($net->{model
} eq 'virtio') {
1614 $device = 'virtio-net-pci';
1617 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1618 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1619 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1620 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1621 # and out of each queue plus one config interrupt and control vector queue
1622 my $vectors = $net->{queues
} * 2 + 2;
1623 $tmpstr .= ",vectors=$vectors,mq=on";
1625 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1627 if (my $mtu = $net->{mtu
}) {
1628 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1629 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1632 } elsif ($mtu < 576) {
1633 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1634 } elsif ($mtu > $bridge_mtu) {
1635 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1637 $tmpstr .= ",host_mtu=$mtu";
1639 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1643 if ($use_old_bios_files) {
1645 if ($device eq 'virtio-net-pci') {
1646 $romfile = 'pxe-virtio.rom';
1647 } elsif ($device eq 'e1000') {
1648 $romfile = 'pxe-e1000.rom';
1649 } elsif ($device eq 'ne2k') {
1650 $romfile = 'pxe-ne2k_pci.rom';
1651 } elsif ($device eq 'pcnet') {
1652 $romfile = 'pxe-pcnet.rom';
1653 } elsif ($device eq 'rtl8139') {
1654 $romfile = 'pxe-rtl8139.rom';
1656 $tmpstr .= ",romfile=$romfile" if $romfile;
1662 sub print_netdev_full
{
1663 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1666 if ($netid =~ m/^net(\d+)$/) {
1670 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1672 my $ifname = "tap${vmid}i$i";
1674 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1675 die "interface name '$ifname' is too long (max 15 character)\n"
1676 if length($ifname) >= 16;
1678 my $vhostparam = '';
1679 if (is_native
($arch)) {
1680 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1683 my $vmname = $conf->{name
} || "vm$vmid";
1686 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1688 if ($net->{bridge
}) {
1689 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1690 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1692 $netdev = "type=user,id=$netid,hostname=$vmname";
1695 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1701 'cirrus' => 'cirrus-vga',
1703 'vmware' => 'vmware-svga',
1704 'virtio' => 'virtio-vga',
1707 sub print_vga_device
{
1708 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1710 my $type = $vga_map->{$vga->{type
}};
1711 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1712 $type = 'virtio-gpu';
1714 my $vgamem_mb = $vga->{memory
};
1716 my $max_outputs = '';
1718 $type = $id ?
'qxl' : 'qxl-vga';
1720 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1721 # set max outputs so linux can have up to 4 qxl displays with one device
1722 if (min_version
($machine_version, 4, 1)) {
1723 $max_outputs = ",max_outputs=4";
1728 die "no devicetype for $vga->{type}\n" if !$type;
1732 if ($vga->{type
} eq 'virtio') {
1733 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1734 $memory = ",max_hostmem=$bytes";
1736 # from https://www.spice-space.org/multiple-monitors.html
1737 $memory = ",vgamem_mb=$vga->{memory}";
1738 my $ram = $vgamem_mb * 4;
1739 my $vram = $vgamem_mb * 2;
1740 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1742 $memory = ",vgamem_mb=$vga->{memory}";
1744 } elsif ($qxlnum && $id) {
1745 $memory = ",ram_size=67108864,vram_size=33554432";
1749 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1750 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1753 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1754 my $vgaid = "vga" . ($id // '');
1757 if ($q35 && $vgaid eq 'vga') {
1758 # the first display uses pcie.0 bus on q35 machines
1759 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1761 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1764 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1767 sub parse_number_sets
{
1770 foreach my $part (split(/;/, $set)) {
1771 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1772 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1773 push @$res, [ $1, $2 ];
1775 die "invalid range: $part\n";
1784 my $res = parse_property_string
($numa_fmt, $data);
1785 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1786 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1790 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1794 my $res = eval { parse_property_string
($net_fmt, $data) };
1799 if (!defined($res->{macaddr
})) {
1800 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1801 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1806 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1807 sub parse_ipconfig
{
1810 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1816 if ($res->{gw
} && !$res->{ip
}) {
1817 warn 'gateway specified without specifying an IP address';
1820 if ($res->{gw6
} && !$res->{ip6
}) {
1821 warn 'IPv6 gateway specified without specifying an IPv6 address';
1824 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1825 warn 'gateway specified together with DHCP';
1828 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1830 warn "IPv6 gateway specified together with $res->{ip6} address";
1834 if (!$res->{ip
} && !$res->{ip6
}) {
1835 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1844 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1847 sub add_random_macs
{
1848 my ($settings) = @_;
1850 foreach my $opt (keys %$settings) {
1851 next if $opt !~ m/^net(\d+)$/;
1852 my $net = parse_net
($settings->{$opt});
1854 $settings->{$opt} = print_net
($net);
1858 sub vm_is_volid_owner
{
1859 my ($storecfg, $vmid, $volid) = @_;
1861 if ($volid !~ m
|^/|) {
1863 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1864 if ($owner && ($owner == $vmid)) {
1872 sub vmconfig_register_unused_drive
{
1873 my ($storecfg, $vmid, $conf, $drive) = @_;
1875 if (drive_is_cloudinit
($drive)) {
1876 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1878 } elsif (!drive_is_cdrom
($drive)) {
1879 my $volid = $drive->{file
};
1880 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1881 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1886 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1890 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1891 format_description
=> 'UUID',
1892 description
=> "Set SMBIOS1 UUID.",
1897 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1898 format_description
=> 'Base64 encoded string',
1899 description
=> "Set SMBIOS1 version.",
1904 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1905 format_description
=> 'Base64 encoded string',
1906 description
=> "Set SMBIOS1 serial number.",
1911 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1912 format_description
=> 'Base64 encoded string',
1913 description
=> "Set SMBIOS1 manufacturer.",
1918 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1919 format_description
=> 'Base64 encoded string',
1920 description
=> "Set SMBIOS1 product ID.",
1925 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1926 format_description
=> 'Base64 encoded string',
1927 description
=> "Set SMBIOS1 SKU string.",
1932 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1933 format_description
=> 'Base64 encoded string',
1934 description
=> "Set SMBIOS1 family string.",
1939 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
1947 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
1954 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
1957 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
1959 sub parse_watchdog
{
1964 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
1969 sub parse_guest_agent
{
1972 return {} if !defined($conf->{agent
});
1974 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
1977 # if the agent is disabled ignore the other potentially set properties
1978 return {} if !$res->{enabled
};
1983 my ($conf, $key) = @_;
1984 return undef if !defined($conf->{agent
});
1986 my $agent = parse_guest_agent
($conf);
1987 return $agent->{$key};
1993 return {} if !$value;
1994 my $res = eval { parse_property_string
($vga_fmt, $value) };
2004 my $res = eval { parse_property_string
($rng_fmt, $value) };
2009 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2010 sub verify_usb_device
{
2011 my ($value, $noerr) = @_;
2013 return $value if parse_usb_device
($value);
2017 die "unable to parse usb device\n";
2020 # add JSON properties for create and set function
2021 sub json_config_properties
{
2024 foreach my $opt (keys %$confdesc) {
2025 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2026 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2027 $prop->{$opt} = $confdesc->{$opt};
2033 # return copy of $confdesc_cloudinit to generate documentation
2034 sub cloudinit_config_properties
{
2036 return dclone
($confdesc_cloudinit);
2040 my ($key, $value) = @_;
2042 die "unknown setting '$key'\n" if !$confdesc->{$key};
2044 my $type = $confdesc->{$key}->{type
};
2046 if (!defined($value)) {
2047 die "got undefined value\n";
2050 if ($value =~ m/[\n\r]/) {
2051 die "property contains a line feed\n";
2054 if ($type eq 'boolean') {
2055 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2056 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2057 die "type check ('boolean') failed - got '$value'\n";
2058 } elsif ($type eq 'integer') {
2059 return int($1) if $value =~ m/^(\d+)$/;
2060 die "type check ('integer') failed - got '$value'\n";
2061 } elsif ($type eq 'number') {
2062 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2063 die "type check ('number') failed - got '$value'\n";
2064 } elsif ($type eq 'string') {
2065 if (my $fmt = $confdesc->{$key}->{format
}) {
2066 PVE
::JSONSchema
::check_format
($fmt, $value);
2069 $value =~ s/^\"(.*)\"$/$1/;
2072 die "internal error"
2077 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2079 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2081 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2083 if ($conf->{template
}) {
2084 # check if any base image is still used by a linked clone
2085 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2086 my ($ds, $drive) = @_;
2087 return if drive_is_cdrom
($drive);
2089 my $volid = $drive->{file
};
2090 return if !$volid || $volid =~ m
|^/|;
2092 die "base volume '$volid' is still in use by linked cloned\n"
2093 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2098 # only remove disks owned by this VM (referenced in the config)
2099 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2100 my ($ds, $drive) = @_;
2101 return if drive_is_cdrom
($drive, 1);
2103 my $volid = $drive->{file
};
2104 return if !$volid || $volid =~ m
|^/|;
2106 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2107 return if !$path || !$owner || ($owner != $vmid);
2109 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2110 warn "Could not remove disk '$volid', check manually: $@" if $@;
2113 if ($purge_unreferenced) { # also remove unreferenced disk
2114 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2115 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2116 my ($volid, $sid, $volname, $d) = @_;
2117 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2122 if (defined $replacement_conf) {
2123 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2125 PVE
::QemuConfig-
>destroy_config($vmid);
2129 sub parse_vm_config
{
2130 my ($filename, $raw) = @_;
2132 return if !defined($raw);
2135 digest
=> Digest
::SHA
::sha1_hex
($raw),
2140 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2141 || die "got strange filename '$filename'";
2149 my @lines = split(/\n/, $raw);
2150 foreach my $line (@lines) {
2151 next if $line =~ m/^\s*$/;
2153 if ($line =~ m/^\[PENDING\]\s*$/i) {
2154 $section = 'pending';
2155 if (defined($descr)) {
2157 $conf->{description
} = $descr;
2160 $conf = $res->{$section} = {};
2163 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2165 if (defined($descr)) {
2167 $conf->{description
} = $descr;
2170 $conf = $res->{snapshots
}->{$section} = {};
2174 if ($line =~ m/^\#(.*)\s*$/) {
2175 $descr = '' if !defined($descr);
2176 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2180 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2181 $descr = '' if !defined($descr);
2182 $descr .= PVE
::Tools
::decode_text
($2);
2183 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2184 $conf->{snapstate
} = $1;
2185 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2188 $conf->{$key} = $value;
2189 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2191 if ($section eq 'pending') {
2192 $conf->{delete} = $value; # we parse this later
2194 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2196 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2199 eval { $value = check_type
($key, $value); };
2201 warn "vm $vmid - unable to parse value of '$key' - $@";
2203 $key = 'ide2' if $key eq 'cdrom';
2204 my $fmt = $confdesc->{$key}->{format
};
2205 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2206 my $v = parse_drive
($key, $value);
2207 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2208 $v->{file
} = $volid;
2209 $value = print_drive
($v);
2211 warn "vm $vmid - unable to parse value of '$key'\n";
2216 $conf->{$key} = $value;
2219 warn "vm $vmid - unable to parse config: $line\n";
2223 if (defined($descr)) {
2225 $conf->{description
} = $descr;
2227 delete $res->{snapstate
}; # just to be sure
2232 sub write_vm_config
{
2233 my ($filename, $conf) = @_;
2235 delete $conf->{snapstate
}; # just to be sure
2237 if ($conf->{cdrom
}) {
2238 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2239 $conf->{ide2
} = $conf->{cdrom
};
2240 delete $conf->{cdrom
};
2243 # we do not use 'smp' any longer
2244 if ($conf->{sockets
}) {
2245 delete $conf->{smp
};
2246 } elsif ($conf->{smp
}) {
2247 $conf->{sockets
} = $conf->{smp
};
2248 delete $conf->{cores
};
2249 delete $conf->{smp
};
2252 my $used_volids = {};
2254 my $cleanup_config = sub {
2255 my ($cref, $pending, $snapname) = @_;
2257 foreach my $key (keys %$cref) {
2258 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2259 $key eq 'snapstate' || $key eq 'pending';
2260 my $value = $cref->{$key};
2261 if ($key eq 'delete') {
2262 die "propertry 'delete' is only allowed in [PENDING]\n"
2264 # fixme: check syntax?
2267 eval { $value = check_type
($key, $value); };
2268 die "unable to parse value of '$key' - $@" if $@;
2270 $cref->{$key} = $value;
2272 if (!$snapname && is_valid_drivename
($key)) {
2273 my $drive = parse_drive
($key, $value);
2274 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2279 &$cleanup_config($conf);
2281 &$cleanup_config($conf->{pending
}, 1);
2283 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2284 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2285 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2288 # remove 'unusedX' settings if we re-add a volume
2289 foreach my $key (keys %$conf) {
2290 my $value = $conf->{$key};
2291 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2292 delete $conf->{$key};
2296 my $generate_raw_config = sub {
2297 my ($conf, $pending) = @_;
2301 # add description as comment to top of file
2302 if (defined(my $descr = $conf->{description
})) {
2304 foreach my $cl (split(/\n/, $descr)) {
2305 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2308 $raw .= "#\n" if $pending;
2312 foreach my $key (sort keys %$conf) {
2313 next if $key =~ /^(digest|description|pending|snapshots)$/;
2314 $raw .= "$key: $conf->{$key}\n";
2319 my $raw = &$generate_raw_config($conf);
2321 if (scalar(keys %{$conf->{pending
}})){
2322 $raw .= "\n[PENDING]\n";
2323 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2326 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2327 $raw .= "\n[$snapname]\n";
2328 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2338 # we use static defaults from our JSON schema configuration
2339 foreach my $key (keys %$confdesc) {
2340 if (defined(my $default = $confdesc->{$key}->{default})) {
2341 $res->{$key} = $default;
2349 my $vmlist = PVE
::Cluster
::get_vmlist
();
2351 return $res if !$vmlist || !$vmlist->{ids
};
2352 my $ids = $vmlist->{ids
};
2353 my $nodename = nodename
();
2355 foreach my $vmid (keys %$ids) {
2356 my $d = $ids->{$vmid};
2357 next if !$d->{node
} || $d->{node
} ne $nodename;
2358 next if !$d->{type
} || $d->{type
} ne 'qemu';
2359 $res->{$vmid}->{exists} = 1;
2364 # test if VM uses local resources (to prevent migration)
2365 sub check_local_resources
{
2366 my ($conf, $noerr) = @_;
2370 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2371 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2373 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2375 foreach my $k (keys %$conf) {
2376 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2377 # sockets are safe: they will recreated be on the target side post-migrate
2378 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2379 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2382 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2387 # check if used storages are available on all nodes (use by migrate)
2388 sub check_storage_availability
{
2389 my ($storecfg, $conf, $node) = @_;
2391 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2392 my ($ds, $drive) = @_;
2394 my $volid = $drive->{file
};
2397 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2400 # check if storage is available on both nodes
2401 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2402 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2406 # list nodes where all VM images are available (used by has_feature API)
2408 my ($conf, $storecfg) = @_;
2410 my $nodelist = PVE
::Cluster
::get_nodelist
();
2411 my $nodehash = { map { $_ => 1 } @$nodelist };
2412 my $nodename = nodename
();
2414 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2415 my ($ds, $drive) = @_;
2417 my $volid = $drive->{file
};
2420 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2422 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2423 if ($scfg->{disable
}) {
2425 } elsif (my $avail = $scfg->{nodes
}) {
2426 foreach my $node (keys %$nodehash) {
2427 delete $nodehash->{$node} if !$avail->{$node};
2429 } elsif (!$scfg->{shared
}) {
2430 foreach my $node (keys %$nodehash) {
2431 delete $nodehash->{$node} if $node ne $nodename
2440 sub check_local_storage_availability
{
2441 my ($conf, $storecfg) = @_;
2443 my $nodelist = PVE
::Cluster
::get_nodelist
();
2444 my $nodehash = { map { $_ => {} } @$nodelist };
2446 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2447 my ($ds, $drive) = @_;
2449 my $volid = $drive->{file
};
2452 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2454 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2456 if ($scfg->{disable
}) {
2457 foreach my $node (keys %$nodehash) {
2458 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2460 } elsif (my $avail = $scfg->{nodes
}) {
2461 foreach my $node (keys %$nodehash) {
2462 if (!$avail->{$node}) {
2463 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2470 foreach my $node (values %$nodehash) {
2471 if (my $unavail = $node->{unavailable_storages
}) {
2472 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2479 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2481 my ($vmid, $nocheck, $node) = @_;
2483 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2484 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2489 my $vzlist = config_list
();
2491 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2493 while (defined(my $de = $fd->read)) {
2494 next if $de !~ m/^(\d+)\.pid$/;
2496 next if !defined($vzlist->{$vmid});
2497 if (my $pid = check_running
($vmid)) {
2498 $vzlist->{$vmid}->{pid
} = $pid;
2505 our $vmstatus_return_properties = {
2506 vmid
=> get_standard_option
('pve-vmid'),
2508 description
=> "Qemu process status.",
2510 enum
=> ['stopped', 'running'],
2513 description
=> "Maximum memory in bytes.",
2516 renderer
=> 'bytes',
2519 description
=> "Root disk size in bytes.",
2522 renderer
=> 'bytes',
2525 description
=> "VM name.",
2530 description
=> "Qemu QMP agent status.",
2535 description
=> "PID of running qemu process.",
2540 description
=> "Uptime.",
2543 renderer
=> 'duration',
2546 description
=> "Maximum usable CPUs.",
2551 description
=> "The current config lock, if any.",
2556 description
=> "The current configured tags, if any",
2562 my $last_proc_pid_stat;
2564 # get VM status information
2565 # This must be fast and should not block ($full == false)
2566 # We only query KVM using QMP if $full == true (this can be slow)
2568 my ($opt_vmid, $full) = @_;
2572 my $storecfg = PVE
::Storage
::config
();
2574 my $list = vzlist
();
2575 my $defaults = load_defaults
();
2577 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2579 my $cpucount = $cpuinfo->{cpus
} || 1;
2581 foreach my $vmid (keys %$list) {
2582 next if $opt_vmid && ($vmid ne $opt_vmid);
2584 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2586 my $d = { vmid
=> $vmid };
2587 $d->{pid
} = $list->{$vmid}->{pid
};
2589 # fixme: better status?
2590 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2592 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2593 if (defined($size)) {
2594 $d->{disk
} = 0; # no info available
2595 $d->{maxdisk
} = $size;
2601 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2602 * ($conf->{cores
} || $defaults->{cores
});
2603 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2604 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2606 $d->{name
} = $conf->{name
} || "VM $vmid";
2607 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2608 : $defaults->{memory
}*(1024*1024);
2610 if ($conf->{balloon
}) {
2611 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2612 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2613 : $defaults->{shares
};
2624 $d->{diskwrite
} = 0;
2626 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
2628 $d->{serial
} = 1 if conf_has_serial
($conf);
2629 $d->{lock} = $conf->{lock} if $conf->{lock};
2630 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2635 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2636 foreach my $dev (keys %$netdev) {
2637 next if $dev !~ m/^tap([1-9]\d*)i/;
2639 my $d = $res->{$vmid};
2642 $d->{netout
} += $netdev->{$dev}->{receive
};
2643 $d->{netin
} += $netdev->{$dev}->{transmit
};
2646 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
2647 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
2652 my $ctime = gettimeofday
;
2654 foreach my $vmid (keys %$list) {
2656 my $d = $res->{$vmid};
2657 my $pid = $d->{pid
};
2660 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2661 next if !$pstat; # not running
2663 my $used = $pstat->{utime} + $pstat->{stime
};
2665 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2667 if ($pstat->{vsize
}) {
2668 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2671 my $old = $last_proc_pid_stat->{$pid};
2673 $last_proc_pid_stat->{$pid} = {
2681 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2683 if ($dtime > 1000) {
2684 my $dutime = $used - $old->{used
};
2686 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2687 $last_proc_pid_stat->{$pid} = {
2693 $d->{cpu
} = $old->{cpu
};
2697 return $res if !$full;
2699 my $qmpclient = PVE
::QMPClient-
>new();
2701 my $ballooncb = sub {
2702 my ($vmid, $resp) = @_;
2704 my $info = $resp->{'return'};
2705 return if !$info->{max_mem
};
2707 my $d = $res->{$vmid};
2709 # use memory assigned to VM
2710 $d->{maxmem
} = $info->{max_mem
};
2711 $d->{balloon
} = $info->{actual
};
2713 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2714 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2715 $d->{freemem
} = $info->{free_mem
};
2718 $d->{ballooninfo
} = $info;
2721 my $blockstatscb = sub {
2722 my ($vmid, $resp) = @_;
2723 my $data = $resp->{'return'} || [];
2724 my $totalrdbytes = 0;
2725 my $totalwrbytes = 0;
2727 for my $blockstat (@$data) {
2728 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2729 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2731 $blockstat->{device
} =~ s/drive-//;
2732 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2734 $res->{$vmid}->{diskread
} = $totalrdbytes;
2735 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2738 my $statuscb = sub {
2739 my ($vmid, $resp) = @_;
2741 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2742 # this fails if ballon driver is not loaded, so this must be
2743 # the last commnand (following command are aborted if this fails).
2744 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2746 my $status = 'unknown';
2747 if (!defined($status = $resp->{'return'}->{status
})) {
2748 warn "unable to get VM status\n";
2752 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2755 foreach my $vmid (keys %$list) {
2756 next if $opt_vmid && ($vmid ne $opt_vmid);
2757 next if !$res->{$vmid}->{pid
}; # not running
2758 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2761 $qmpclient->queue_execute(undef, 2);
2763 foreach my $vmid (keys %$list) {
2764 next if $opt_vmid && ($vmid ne $opt_vmid);
2765 next if !$res->{$vmid}->{pid
}; #not running
2767 # we can't use the $qmpclient since it might have already aborted on
2768 # 'query-balloon', but this might also fail for older versions...
2769 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
2770 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2773 foreach my $vmid (keys %$list) {
2774 next if $opt_vmid && ($vmid ne $opt_vmid);
2775 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2781 sub conf_has_serial
{
2784 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2785 if ($conf->{"serial$i"}) {
2793 sub conf_has_audio
{
2794 my ($conf, $id) = @_;
2797 my $audio = $conf->{"audio$id"};
2798 return if !defined($audio);
2800 my $audioproperties = parse_property_string
($audio_fmt, $audio);
2801 my $audiodriver = $audioproperties->{driver
} // 'spice';
2804 dev
=> $audioproperties->{device
},
2805 dev_id
=> "audiodev$id",
2806 backend
=> $audiodriver,
2807 backend_id
=> "$audiodriver-backend${id}",
2812 my ($audio, $audiopciaddr, $machine_version) = @_;
2816 my $id = $audio->{dev_id
};
2818 if (min_version
($machine_version, 4, 2)) {
2819 $audiodev = ",audiodev=$audio->{backend_id}";
2822 if ($audio->{dev
} eq 'AC97') {
2823 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2824 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
2825 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2826 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2827 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2829 die "unkown audio device '$audio->{dev}', implement me!";
2832 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2837 sub vga_conf_has_spice
{
2840 my $vgaconf = parse_vga
($vga);
2841 my $vgatype = $vgaconf->{type
};
2842 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2849 return get_host_arch
() eq $arch;
2854 return $conf->{arch
} // get_host_arch
();
2857 my $default_machines = {
2862 sub get_vm_machine
{
2863 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2865 my $machine = $forcemachine || $conf->{machine
};
2867 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2869 $machine ||= $default_machines->{$arch};
2870 if ($add_pve_version) {
2871 $kvmversion //= kvm_user_version
();
2872 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
2873 $machine .= "+pve$pvever";
2877 if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2878 # for version-pinned machines that do not include a pve-version (e.g.
2879 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2880 $machine .= '+pve0';
2886 sub get_ovmf_files
($) {
2889 my $ovmf = $OVMF->{$arch}
2890 or die "no OVMF images known for architecture '$arch'\n";
2896 aarch64
=> '/usr/bin/qemu-system-aarch64',
2897 x86_64
=> '/usr/bin/qemu-system-x86_64',
2899 sub get_command_for_arch
($) {
2901 return '/usr/bin/kvm' if is_native
($arch);
2903 my $cmd = $Arch2Qemu->{$arch}
2904 or die "don't know how to emulate architecture '$arch'\n";
2908 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2909 # to use in a QEMU command line (-cpu element), first array_intersect the result
2910 # of query_supported_ with query_understood_. This is necessary because:
2912 # a) query_understood_ returns flags the host cannot use and
2913 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2914 # flags, but CPU settings - with most of them being flags. Those settings
2915 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
2917 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2918 # expensive. If you need the value returned from this, you can get it much
2919 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2920 # $accel being 'kvm' or 'tcg'.
2922 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2923 # changes, automatically populating pmxcfs.
2925 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2926 # since kvm and tcg machines support different flags
2928 sub query_supported_cpu_flags
{
2931 $arch //= get_host_arch
();
2932 my $default_machine = $default_machines->{$arch};
2936 # FIXME: Once this is merged, the code below should work for ARM as well:
2937 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2938 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2941 my $kvm_supported = defined(kvm_version
());
2942 my $qemu_cmd = get_command_for_arch
($arch);
2944 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
2946 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2947 my $query_supported_run_qemu = sub {
2953 '-machine', $default_machine,
2955 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2956 '-mon', 'chardev=qmp,mode=control',
2957 '-pidfile', $pidfile,
2962 push @$cmd, '-accel', 'tcg';
2965 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
2966 die "QEMU flag querying VM exited with code " . $rc if $rc;
2969 my $cmd_result = mon_cmd
(
2971 'query-cpu-model-expansion',
2973 model
=> { name
=> 'host' }
2976 my $props = $cmd_result->{model
}->{props
};
2977 foreach my $prop (keys %$props) {
2978 next if $props->{$prop} ne '1';
2979 # QEMU returns some flags multiple times, with '_', '.' or '-'
2980 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2981 # We only keep those with underscores, to match /proc/cpuinfo
2982 $prop =~ s/\.|-/_/g;
2983 $flags->{$prop} = 1;
2988 # force stop with 10 sec timeout and 'nocheck'
2989 # always stop, even if QMP failed
2990 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
2994 return [ sort keys %$flags ];
2997 # We need to query QEMU twice, since KVM and TCG have different supported flags
2998 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
2999 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3000 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3002 if ($kvm_supported) {
3003 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3004 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3011 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3012 my $understood_cpu_flag_dir = "/usr/share/kvm";
3013 sub query_understood_cpu_flags
{
3014 my $arch = get_host_arch
();
3015 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3017 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3020 my $raw = file_get_contents
($filepath);
3021 $raw =~ s/^\s+|\s+$//g;
3022 my @flags = split(/\s+/, $raw);
3027 sub config_to_command
{
3028 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3031 my $globalFlags = [];
3032 my $machineFlags = [];
3037 my $ostype = $conf->{ostype
};
3038 my $winversion = windows_version
($ostype);
3039 my $kvm = $conf->{kvm
};
3040 my $nodename = nodename
();
3042 my $arch = get_vm_arch
($conf);
3043 my $kvm_binary = get_command_for_arch
($arch);
3044 my $kvmver = kvm_user_version
($kvm_binary);
3046 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3047 $kvmver //= "undefined";
3048 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3051 my $add_pve_version = min_version
($kvmver, 4, 1);
3053 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3054 my $machine_version = extract_version
($machine_type, $kvmver);
3055 $kvm //= 1 if is_native
($arch);
3057 $machine_version =~ m/(\d+)\.(\d+)/;
3058 my ($machine_major, $machine_minor) = ($1, $2);
3060 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3061 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3062 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3063 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3064 ." please upgrade node '$nodename'\n"
3065 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3066 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3067 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3068 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3069 ." node '$nodename'\n";
3072 # if a specific +pve version is required for a feature, use $version_guard
3073 # instead of min_version to allow machines to be run with the minimum
3075 my $required_pve_version = 0;
3076 my $version_guard = sub {
3077 my ($major, $minor, $pve) = @_;
3078 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3079 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3080 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3081 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3085 if ($kvm && !defined kvm_version
()) {
3086 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3087 ." or enable in BIOS.\n";
3090 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3091 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3092 my $use_old_bios_files = undef;
3093 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3095 my $cpuunits = defined($conf->{cpuunits
}) ?
3096 $conf->{cpuunits
} : $defaults->{cpuunits
};
3098 push @$cmd, $kvm_binary;
3100 push @$cmd, '-id', $vmid;
3102 my $vmname = $conf->{name
} || "vm$vmid";
3104 push @$cmd, '-name', $vmname;
3106 push @$cmd, '-no-shutdown';
3110 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3111 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3112 push @$cmd, '-mon', "chardev=qmp,mode=control";
3114 if (min_version
($machine_version, 2, 12)) {
3115 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3116 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3119 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3121 push @$cmd, '-daemonize';
3123 if ($conf->{smbios1
}) {
3124 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3125 if ($smbios_conf->{base64
}) {
3126 # Do not pass base64 flag to qemu
3127 delete $smbios_conf->{base64
};
3128 my $smbios_string = "";
3129 foreach my $key (keys %$smbios_conf) {
3131 if ($key eq "uuid") {
3132 $value = $smbios_conf->{uuid
}
3134 $value = decode_base64
($smbios_conf->{$key});
3136 # qemu accepts any binary data, only commas need escaping by double comma
3138 $smbios_string .= "," . $key . "=" . $value if $value;
3140 push @$cmd, '-smbios', "type=1" . $smbios_string;
3142 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3146 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3147 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3148 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3150 my ($path, $format);
3151 if (my $efidisk = $conf->{efidisk0
}) {
3152 my $d = parse_drive
('efidisk0', $efidisk);
3153 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3154 $format = $d->{format
};
3156 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3157 if (!defined($format)) {
3158 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3159 $format = qemu_img_format
($scfg, $volname);
3163 die "efidisk format must be specified\n"
3164 if !defined($format);
3167 warn "no efidisk configured! Using temporary efivars disk.\n";
3168 $path = "/tmp/$vmid-ovmf.fd";
3169 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3175 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3176 $size_str = ",size=" . (-s
$ovmf_vars);
3179 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3180 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3185 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3186 if (min_version
($machine_version, 4, 0)) {
3187 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3189 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3193 if ($conf->{vmgenid
}) {
3194 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3197 # add usb controllers
3198 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3199 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3200 push @$devices, @usbcontrollers if @usbcontrollers;
3201 my $vga = parse_vga
($conf->{vga
});
3203 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3204 $vga->{type
} = 'qxl' if $qxlnum;
3206 if (!$vga->{type
}) {
3207 if ($arch eq 'aarch64') {
3208 $vga->{type
} = 'virtio';
3209 } elsif (min_version
($machine_version, 2, 9)) {
3210 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3212 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3216 # enable absolute mouse coordinates (needed by vnc)
3218 if (defined($conf->{tablet
})) {
3219 $tablet = $conf->{tablet
};
3221 $tablet = $defaults->{tablet
};
3222 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3223 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3227 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3228 my $kbd = print_keyboarddevice_full
($conf, $arch);
3229 push @$devices, '-device', $kbd if defined($kbd);
3232 my $bootorder = device_bootorder
($conf);
3234 # host pci device passthrough
3235 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3236 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3239 my $usb_dev_features = {};
3240 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3242 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3243 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3244 push @$devices, @usbdevices if @usbdevices;
3247 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3248 if (my $path = $conf->{"serial$i"}) {
3249 if ($path eq 'socket') {
3250 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3251 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3252 # On aarch64, serial0 is the UART device. Qemu only allows
3253 # connecting UART devices via the '-serial' command line, as
3254 # the device has a fixed slot on the hardware...
3255 if ($arch eq 'aarch64' && $i == 0) {
3256 push @$devices, '-serial', "chardev:serial$i";
3258 push @$devices, '-device', "isa-serial,chardev=serial$i";
3261 die "no such serial device\n" if ! -c
$path;
3262 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3263 push @$devices, '-device', "isa-serial,chardev=serial$i";
3269 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3270 if (my $path = $conf->{"parallel$i"}) {
3271 die "no such parallel device\n" if ! -c
$path;
3272 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3273 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3274 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3278 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3279 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3280 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3281 push @$devices, @$audio_devs;
3285 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3286 $sockets = $conf->{sockets
} if $conf->{sockets
};
3288 my $cores = $conf->{cores
} || 1;
3290 my $maxcpus = $sockets * $cores;
3292 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3294 my $allowed_vcpus = $cpuinfo->{cpus
};
3296 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3297 if ($allowed_vcpus < $maxcpus);
3299 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3301 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3302 for (my $i = 2; $i <= $vcpus; $i++) {
3303 my $cpustr = print_cpu_device
($conf,$i);
3304 push @$cmd, '-device', $cpustr;
3309 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3311 push @$cmd, '-nodefaults';
3313 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3315 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3317 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3319 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3320 push @$devices, '-device', print_vga_device
(
3321 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3322 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3323 push @$cmd, '-vnc', "unix:$socket,password";
3325 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3326 push @$cmd, '-nographic';
3330 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3331 my $useLocaltime = $conf->{localtime};
3333 if ($winversion >= 5) { # windows
3334 $useLocaltime = 1 if !defined($conf->{localtime});
3336 # use time drift fix when acpi is enabled
3337 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3338 $tdf = 1 if !defined($conf->{tdf
});
3342 if ($winversion >= 6) {
3343 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3344 push @$cmd, '-no-hpet';
3347 push @$rtcFlags, 'driftfix=slew' if $tdf;
3349 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3350 push @$rtcFlags, "base=$conf->{startdate}";
3351 } elsif ($useLocaltime) {
3352 push @$rtcFlags, 'base=localtime';
3356 push @$cmd, '-cpu', $forcecpu;
3358 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3361 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3363 push @$cmd, '-S' if $conf->{freeze
};
3365 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3367 my $guest_agent = parse_guest_agent
($conf);
3369 if ($guest_agent->{enabled
}) {
3370 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3371 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3373 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3374 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3375 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3376 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3377 } elsif ($guest_agent->{type
} eq 'isa') {
3378 push @$devices, '-device', "isa-serial,chardev=qga0";
3382 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3383 if ($rng && $version_guard->(4, 1, 2)) {
3384 check_rng_source
($rng->{source
});
3386 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3387 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3388 my $limiter_str = "";
3390 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3393 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3394 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3395 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3403 for (my $i = 1; $i < $qxlnum; $i++){
3404 push @$devices, '-device', print_vga_device
(
3405 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3408 # assume other OS works like Linux
3409 my ($ram, $vram) = ("134217728", "67108864");
3410 if ($vga->{memory
}) {
3411 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3412 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3414 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3415 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3419 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3421 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3422 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3423 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3425 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3426 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3427 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3429 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3430 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3432 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3433 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3434 if ($spice_enhancement->{foldersharing
}) {
3435 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3436 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3439 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3440 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3441 if $spice_enhancement->{videostreaming
};
3443 push @$devices, '-spice', "$spice_opts";
3446 # enable balloon by default, unless explicitly disabled
3447 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3448 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3449 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3452 if ($conf->{watchdog
}) {
3453 my $wdopts = parse_watchdog
($conf->{watchdog
});
3454 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3455 my $watchdog = $wdopts->{model
} || 'i6300esb';
3456 push @$devices, '-device', "$watchdog$pciaddr";
3457 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3461 my $scsicontroller = {};
3462 my $ahcicontroller = {};
3463 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3465 # Add iscsi initiator name if available
3466 if (my $initiator = get_initiator_name
()) {
3467 push @$devices, '-iscsi', "initiator-name=$initiator";
3470 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3471 my ($ds, $drive) = @_;
3473 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3474 push @$vollist, $drive->{file
};
3477 # ignore efidisk here, already added in bios/fw handling code above
3478 return if $drive->{interface
} eq 'efidisk';
3480 $use_virtio = 1 if $ds =~ m/^virtio/;
3482 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3484 if ($drive->{interface
} eq 'virtio'){
3485 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3488 if ($drive->{interface
} eq 'scsi') {
3490 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3492 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3493 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3495 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3496 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3499 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3500 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3501 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3502 } elsif ($drive->{iothread
}) {
3503 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3507 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3508 $queues = ",num_queues=$drive->{queues}";
3511 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3512 if !$scsicontroller->{$controller};
3513 $scsicontroller->{$controller}=1;
3516 if ($drive->{interface
} eq 'sata') {
3517 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3518 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3519 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3520 if !$ahcicontroller->{$controller};
3521 $ahcicontroller->{$controller}=1;
3524 my $drive_cmd = print_drive_commandline_full
($storecfg, $vmid, $drive);
3525 $drive_cmd .= ',readonly' if PVE
::QemuConfig-
>is_template($conf);
3527 push @$devices, '-drive',$drive_cmd;
3528 push @$devices, '-device', print_drivedevice_full
(
3529 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3532 for (my $i = 0; $i < $MAX_NETS; $i++) {
3533 my $netname = "net$i";
3535 next if !$conf->{$netname};
3536 my $d = parse_net
($conf->{$netname});
3539 $use_virtio = 1 if $d->{model
} eq 'virtio';
3541 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
3543 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
3544 push @$devices, '-netdev', $netdevfull;
3546 my $netdevicefull = print_netdevice_full
(
3547 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3549 push @$devices, '-device', $netdevicefull;
3552 if ($conf->{ivshmem
}) {
3553 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3557 $bus = print_pcie_addr
("ivshmem");
3559 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3562 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3563 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3565 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3566 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3567 .",size=$ivshmem->{size}M";
3570 # pci.4 is nested in pci.1
3571 $bridges->{1} = 1 if $bridges->{4};
3575 if (min_version
($machine_version, 2, 3)) {
3580 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3584 for my $k (sort {$b cmp $a} keys %$bridges) {
3585 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3588 if ($k == 2 && $legacy_igd) {
3591 $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
3593 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3595 # add after -readconfig pve-q35.cfg
3596 splice @$devices, 2, 0, '-device', $devstr;
3598 unshift @$devices, '-device', $devstr if $k > 0;
3603 push @$machineFlags, 'accel=tcg';
3606 my $machine_type_min = $machine_type;
3607 if ($add_pve_version) {
3608 $machine_type_min =~ s/\+pve\d+$//;
3609 $machine_type_min .= "+pve$required_pve_version";
3611 push @$machineFlags, "type=${machine_type_min}";
3613 push @$cmd, @$devices;
3614 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3615 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3616 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3618 if (my $vmstate = $conf->{vmstate
}) {
3619 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3620 push @$vollist, $vmstate;
3621 push @$cmd, '-loadstate', $statepath;
3622 print "activating and using '$vmstate' as vmstate\n";
3626 if ($conf->{args
}) {
3627 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3631 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3634 sub check_rng_source
{
3637 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3638 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3641 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3642 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
3643 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3644 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3645 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3646 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3654 my $res = mon_cmd
($vmid, 'query-spice');
3656 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3659 sub vm_devices_list
{
3662 my $res = mon_cmd
($vmid, 'query-pci');
3663 my $devices_to_check = [];
3665 foreach my $pcibus (@$res) {
3666 push @$devices_to_check, @{$pcibus->{devices
}},
3669 while (@$devices_to_check) {
3671 for my $d (@$devices_to_check) {
3672 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3673 next if !$d->{'pci_bridge'};
3675 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3676 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3678 $devices_to_check = $to_check;
3681 my $resblock = mon_cmd
($vmid, 'query-block');
3682 foreach my $block (@$resblock) {
3683 if($block->{device
} =~ m/^drive-(\S+)/){
3688 my $resmice = mon_cmd
($vmid, 'query-mice');
3689 foreach my $mice (@$resmice) {
3690 if ($mice->{name
} eq 'QEMU HID Tablet') {
3691 $devices->{tablet
} = 1;
3696 # for usb devices there is no query-usb
3697 # but we can iterate over the entries in
3698 # qom-list path=/machine/peripheral
3699 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3700 foreach my $per (@$resperipheral) {
3701 if ($per->{name
} =~ m/^usb\d+$/) {
3702 $devices->{$per->{name
}} = 1;
3710 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3712 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3714 my $devices_list = vm_devices_list
($vmid);
3715 return 1 if defined($devices_list->{$deviceid});
3717 # add PCI bridge if we need it for the device
3718 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3720 if ($deviceid eq 'tablet') {
3722 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
3724 } elsif ($deviceid eq 'keyboard') {
3726 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
3728 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3730 die "usb hotplug currently not reliable\n";
3731 # since we can't reliably hot unplug all added usb devices and usb
3732 # passthrough breaks live migration we disable usb hotplugging for now
3733 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3735 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3737 qemu_iothread_add
($vmid, $deviceid, $device);
3739 qemu_driveadd
($storecfg, $vmid, $device);
3740 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3742 qemu_deviceadd
($vmid, $devicefull);
3743 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3745 eval { qemu_drivedel
($vmid, $deviceid); };
3750 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3753 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3754 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3755 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3757 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3759 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3760 qemu_iothread_add
($vmid, $deviceid, $device);
3761 $devicefull .= ",iothread=iothread-$deviceid";
3764 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3765 $devicefull .= ",num_queues=$device->{queues}";
3768 qemu_deviceadd
($vmid, $devicefull);
3769 qemu_deviceaddverify
($vmid, $deviceid);
3771 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3773 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3774 qemu_driveadd
($storecfg, $vmid, $device);
3776 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3777 eval { qemu_deviceadd
($vmid, $devicefull); };
3779 eval { qemu_drivedel
($vmid, $deviceid); };
3784 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3786 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
3788 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
3789 my $use_old_bios_files = undef;
3790 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3792 my $netdevicefull = print_netdevice_full
(
3793 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3794 qemu_deviceadd
($vmid, $netdevicefull);
3796 qemu_deviceaddverify
($vmid, $deviceid);
3797 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
3800 eval { qemu_netdevdel
($vmid, $deviceid); };
3805 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3808 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3809 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3811 qemu_deviceadd
($vmid, $devicefull);
3812 qemu_deviceaddverify
($vmid, $deviceid);
3815 die "can't hotplug device '$deviceid'\n";
3821 # fixme: this should raise exceptions on error!
3822 sub vm_deviceunplug
{
3823 my ($vmid, $conf, $deviceid) = @_;
3825 my $devices_list = vm_devices_list
($vmid);
3826 return 1 if !defined($devices_list->{$deviceid});
3828 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
3829 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3831 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3833 qemu_devicedel
($vmid, $deviceid);
3835 } elsif ($deviceid =~ m/^usb\d+$/) {
3837 die "usb hotplug currently not reliable\n";
3838 # when unplugging usb devices this way, there may be remaining usb
3839 # controllers/hubs so we disable it for now
3840 #qemu_devicedel($vmid, $deviceid);
3841 #qemu_devicedelverify($vmid, $deviceid);
3843 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3845 qemu_devicedel
($vmid, $deviceid);
3846 qemu_devicedelverify
($vmid, $deviceid);
3847 qemu_drivedel
($vmid, $deviceid);
3848 qemu_iothread_del
($conf, $vmid, $deviceid);
3850 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3852 qemu_devicedel
($vmid, $deviceid);
3853 qemu_devicedelverify
($vmid, $deviceid);
3854 qemu_iothread_del
($conf, $vmid, $deviceid);
3856 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3858 qemu_devicedel
($vmid, $deviceid);
3859 qemu_drivedel
($vmid, $deviceid);
3860 qemu_deletescsihw
($conf, $vmid, $deviceid);
3862 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3864 qemu_devicedel
($vmid, $deviceid);
3865 qemu_devicedelverify
($vmid, $deviceid);
3866 qemu_netdevdel
($vmid, $deviceid);
3869 die "can't unplug device '$deviceid'\n";
3875 sub qemu_deviceadd
{
3876 my ($vmid, $devicefull) = @_;
3878 $devicefull = "driver=".$devicefull;
3879 my %options = split(/[=,]/, $devicefull);
3881 mon_cmd
($vmid, "device_add" , %options);
3884 sub qemu_devicedel
{
3885 my ($vmid, $deviceid) = @_;
3887 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
3890 sub qemu_iothread_add
{
3891 my($vmid, $deviceid, $device) = @_;
3893 if ($device->{iothread
}) {
3894 my $iothreads = vm_iothreads_list
($vmid);
3895 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3899 sub qemu_iothread_del
{
3900 my($conf, $vmid, $deviceid) = @_;
3902 my $confid = $deviceid;
3903 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3904 $confid = 'scsi' . $1;
3906 my $device = parse_drive
($confid, $conf->{$confid});
3907 if ($device->{iothread
}) {
3908 my $iothreads = vm_iothreads_list
($vmid);
3909 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3913 sub qemu_objectadd
{
3914 my($vmid, $objectid, $qomtype) = @_;
3916 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
3921 sub qemu_objectdel
{
3922 my($vmid, $objectid) = @_;
3924 mon_cmd
($vmid, "object-del", id
=> $objectid);
3930 my ($storecfg, $vmid, $device) = @_;
3932 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device);
3933 $drive =~ s/\\/\\\\/g;
3934 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
3936 # If the command succeeds qemu prints: "OK
"
3937 return 1 if $ret =~ m/OK/s;
3939 die "adding drive failed
: $ret\n";
3943 my($vmid, $deviceid) = @_;
3945 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
3948 return 1 if $ret eq "";
3950 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3951 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3953 die "deleting drive
$deviceid failed
: $ret\n";
3956 sub qemu_deviceaddverify {
3957 my ($vmid, $deviceid) = @_;
3959 for (my $i = 0; $i <= 5; $i++) {
3960 my $devices_list = vm_devices_list($vmid);
3961 return 1 if defined($devices_list->{$deviceid});
3965 die "error on hotplug device
'$deviceid'\n";
3969 sub qemu_devicedelverify {
3970 my ($vmid, $deviceid) = @_;
3972 # need to verify that the device is correctly removed as device_del
3973 # is async and empty return is not reliable
3975 for (my $i = 0; $i <= 5; $i++) {
3976 my $devices_list = vm_devices_list($vmid);
3977 return 1 if !defined($devices_list->{$deviceid});
3981 die "error on hot-unplugging device
'$deviceid'\n";
3984 sub qemu_findorcreatescsihw {
3985 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3987 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3989 my $scsihwid="$controller_prefix$controller";
3990 my $devices_list = vm_devices_list($vmid);
3992 if(!defined($devices_list->{$scsihwid})) {
3993 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3999 sub qemu_deletescsihw {
4000 my ($conf, $vmid, $opt) = @_;
4002 my $device = parse_drive($opt, $conf->{$opt});
4004 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4005 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4009 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4011 my $devices_list = vm_devices_list($vmid);
4012 foreach my $opt (keys %{$devices_list}) {
4013 if (is_valid_drivename($opt)) {
4014 my $drive = parse_drive($opt, $conf->{$opt});
4015 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4021 my $scsihwid="scsihw
$controller";
4023 vm_deviceunplug($vmid, $conf, $scsihwid);
4028 sub qemu_add_pci_bridge {
4029 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4035 print_pci_addr($device, $bridges, $arch, $machine_type);
4037 while (my ($k, $v) = each %$bridges) {
4040 return 1 if !defined($bridgeid) || $bridgeid < 1;
4042 my $bridge = "pci
.$bridgeid";
4043 my $devices_list = vm_devices_list($vmid);
4045 if (!defined($devices_list->{$bridge})) {
4046 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4052 sub qemu_set_link_status {
4053 my ($vmid, $device, $up) = @_;
4055 mon_cmd($vmid, "set_link
", name => $device,
4056 up => $up ? JSON::true : JSON::false);
4059 sub qemu_netdevadd {
4060 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4062 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4063 my %options = split(/[=,]/, $netdev);
4065 if (defined(my $vhost = $options{vhost})) {
4066 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4069 if (defined(my $queues = $options{queues})) {
4070 $options{queues} = $queues + 0;
4073 mon_cmd($vmid, "netdev_add
", %options);
4077 sub qemu_netdevdel {
4078 my ($vmid, $deviceid) = @_;
4080 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4083 sub qemu_usb_hotplug {
4084 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4088 # remove the old one first
4089 vm_deviceunplug($vmid, $conf, $deviceid);
4091 # check if xhci controller is necessary and available
4092 if ($device->{usb3}) {
4094 my $devicelist = vm_devices_list($vmid);
4096 if (!$devicelist->{xhci}) {
4097 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4098 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4101 my $d = parse_usb_device($device->{host});
4102 $d->{usb3} = $device->{usb3};
4105 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4108 sub qemu_cpu_hotplug {
4109 my ($vmid, $conf, $vcpus) = @_;
4111 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4114 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4115 $sockets = $conf->{sockets} if $conf->{sockets};
4116 my $cores = $conf->{cores} || 1;
4117 my $maxcpus = $sockets * $cores;
4119 $vcpus = $maxcpus if !$vcpus;
4121 die "you can
't add more vcpus than maxcpus\n"
4122 if $vcpus > $maxcpus;
4124 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4126 if ($vcpus < $currentvcpus) {
4128 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4130 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4131 qemu_devicedel($vmid, "cpu$i");
4133 my $currentrunningvcpus = undef;
4135 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4136 last if scalar(@{$currentrunningvcpus}) == $i-1;
4137 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4141 #update conf after each succesfull cpu unplug
4142 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4143 PVE::QemuConfig->write_config($vmid, $conf);
4146 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4152 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4153 die "vcpus in running vm does not match its configuration\n"
4154 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4156 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4158 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4159 my $cpustr = print_cpu_device($conf, $i);
4160 qemu_deviceadd($vmid, $cpustr);
4163 my $currentrunningvcpus = undef;
4165 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4166 last if scalar(@{$currentrunningvcpus}) == $i;
4167 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4171 #update conf after each succesfull cpu hotplug
4172 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4173 PVE::QemuConfig->write_config($vmid, $conf);
4177 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4178 mon_cmd($vmid, "cpu-add", id => int($i));
4183 sub qemu_block_set_io_throttle {
4184 my ($vmid, $deviceid,
4185 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4186 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4187 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4188 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4190 return if !check_running($vmid) ;
4192 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4194 bps_rd => int($bps_rd),
4195 bps_wr => int($bps_wr),
4197 iops_rd => int($iops_rd),
4198 iops_wr => int($iops_wr),
4199 bps_max => int($bps_max),
4200 bps_rd_max => int($bps_rd_max),
4201 bps_wr_max => int($bps_wr_max),
4202 iops_max => int($iops_max),
4203 iops_rd_max => int($iops_rd_max),
4204 iops_wr_max => int($iops_wr_max),
4205 bps_max_length => int($bps_max_length),
4206 bps_rd_max_length => int($bps_rd_max_length),
4207 bps_wr_max_length => int($bps_wr_max_length),
4208 iops_max_length => int($iops_max_length),
4209 iops_rd_max_length => int($iops_rd_max_length),
4210 iops_wr_max_length => int($iops_wr_max_length),
4215 sub qemu_block_resize {
4216 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4218 my $running = check_running($vmid);
4220 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4222 return if !$running;
4224 my $padding = (1024 - $size % 1024) % 1024;
4225 $size = $size + $padding;
4227 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4231 sub qemu_volume_snapshot {
4232 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4234 my $running = check_running($vmid);
4236 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4237 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4239 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4243 sub qemu_volume_snapshot_delete {
4244 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4246 my $running = check_running($vmid);
4251 my $conf = PVE::QemuConfig->load_config($vmid);
4252 PVE::QemuConfig->foreach_volume($conf, sub {
4253 my ($ds, $drive) = @_;
4254 $running = 1 if $drive->{file} eq $volid;
4258 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4259 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4261 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4265 sub set_migration_caps {
4268 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4273 "auto-converge" => 1,
4275 "x-rdma-pin-all" => 0,
4278 "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration
'} ? 1 : 0,
4281 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4283 for my $supported_capability (@$supported_capabilities) {
4285 capability => $supported_capability->{capability},
4286 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4290 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4294 my ($conf, $func, @param) = @_;
4298 my $test_volid = sub {
4299 my ($key, $drive, $snapname) = @_;
4301 my $volid = $drive->{file};
4304 $volhash->{$volid}->{cdrom} //= 1;
4305 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4307 my $replicate = $drive->{replicate} // 1;
4308 $volhash->{$volid}->{replicate} //= 0;
4309 $volhash->{$volid}->{replicate} = 1 if $replicate;
4311 $volhash->{$volid}->{shared} //= 0;
4312 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4314 $volhash->{$volid}->{referenced_in_config} //= 0;
4315 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4317 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4318 if defined($snapname);
4320 my $size = $drive->{size};
4321 $volhash->{$volid}->{size} //= $size if $size;
4323 $volhash->{$volid}->{is_vmstate} //= 0;
4324 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4326 $volhash->{$volid}->{is_unused} //= 0;
4327 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4330 my $include_opts = {
4331 extra_keys => ['vmstate
'],
4332 include_unused => 1,
4335 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4336 foreach my $snapname (keys %{$conf->{snapshots}}) {
4337 my $snap = $conf->{snapshots}->{$snapname};
4338 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4341 foreach my $volid (keys %$volhash) {
4342 &$func($volid, $volhash->{$volid}, @param);
4346 my $fast_plug_option = {
4354 'vmstatestorage
' => 1,
4359 # hotplug changes in [PENDING]
4360 # $selection hash can be used to only apply specified options, for
4361 # example: { cores => 1 } (only apply changed 'cores
')
4362 # $errors ref is used to return error messages
4363 sub vmconfig_hotplug_pending {
4364 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4366 my $defaults = load_defaults();
4367 my $arch = get_vm_arch($conf);
4368 my $machine_type = get_vm_machine($conf, undef, $arch);
4370 # commit values which do not have any impact on running VM first
4371 # Note: those option cannot raise errors, we we do not care about
4372 # $selection and always apply them.
4374 my $add_error = sub {
4375 my ($opt, $msg) = @_;
4376 $errors->{$opt} = "hotplug problem - $msg";
4380 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4381 if ($fast_plug_option->{$opt}) {
4382 $conf->{$opt} = $conf->{pending}->{$opt};
4383 delete $conf->{pending}->{$opt};
4389 PVE::QemuConfig->write_config($vmid, $conf);
4392 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4394 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4395 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4396 foreach my $opt (sort keys %$pending_delete_hash) {
4397 next if $selection && !$selection->{$opt};
4398 my $force = $pending_delete_hash->{$opt}->{force};
4400 if ($opt eq 'hotplug
') {
4401 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4402 } elsif ($opt eq 'tablet
') {
4403 die "skip\n" if !$hotplug_features->{usb};
4404 if ($defaults->{tablet}) {
4405 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4406 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4407 if $arch eq 'aarch64
';
4409 vm_deviceunplug($vmid, $conf, 'tablet
');
4410 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4412 } elsif ($opt =~ m/^usb\d+/) {
4414 # since we cannot reliably hot unplug usb devices we are disabling it
4415 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4416 #vm_deviceunplug($vmid, $conf, $opt);
4417 } elsif ($opt eq 'vcpus
') {
4418 die "skip\n" if !$hotplug_features->{cpu};
4419 qemu_cpu_hotplug($vmid, $conf, undef);
4420 } elsif ($opt eq 'balloon
') {
4421 # enable balloon device is not hotpluggable
4422 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4423 # here we reset the ballooning value to memory
4424 my $balloon = $conf->{memory} || $defaults->{memory};
4425 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4426 } elsif ($fast_plug_option->{$opt}) {
4428 } elsif ($opt =~ m/^net(\d+)$/) {
4429 die "skip\n" if !$hotplug_features->{network};
4430 vm_deviceunplug($vmid, $conf, $opt);
4431 } elsif (is_valid_drivename($opt)) {
4432 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4433 vm_deviceunplug($vmid, $conf, $opt);
4434 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4435 } elsif ($opt =~ m/^memory$/) {
4436 die "skip\n" if !$hotplug_features->{memory};
4437 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4438 } elsif ($opt eq 'cpuunits
') {
4439 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4440 } elsif ($opt eq 'cpulimit
') {
4441 $cgroup->change_cpu_quota(-1, 100000);
4447 &$add_error($opt, $err) if $err ne "skip\n";
4449 delete $conf->{$opt};
4450 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4454 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4455 $apply_pending_cloudinit = sub {
4456 return if $apply_pending_cloudinit_done; # once is enough
4457 $apply_pending_cloudinit_done = 1; # once is enough
4459 my ($key, $value) = @_;
4461 my @cloudinit_opts = keys %$confdesc_cloudinit;
4462 foreach my $opt (keys %{$conf->{pending}}) {
4463 next if !grep { $_ eq $opt } @cloudinit_opts;
4464 $conf->{$opt} = delete $conf->{pending}->{$opt};
4467 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4468 foreach my $opt (sort keys %$pending_delete_hash) {
4469 next if !grep { $_ eq $opt } @cloudinit_opts;
4470 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4471 delete $conf->{$opt};
4474 my $new_conf = { %$conf };
4475 $new_conf->{$key} = $value;
4476 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4479 foreach my $opt (keys %{$conf->{pending}}) {
4480 next if $selection && !$selection->{$opt};
4481 my $value = $conf->{pending}->{$opt};
4483 if ($opt eq 'hotplug
') {
4484 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4485 } elsif ($opt eq 'tablet
') {
4486 die "skip\n" if !$hotplug_features->{usb};
4488 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4489 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4490 if $arch eq 'aarch64
';
4491 } elsif ($value == 0) {
4492 vm_deviceunplug($vmid, $conf, 'tablet
');
4493 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4495 } elsif ($opt =~ m/^usb\d+$/) {
4497 # since we cannot reliably hot unplug usb devices we disable it for now
4498 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4499 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4500 #die "skip\n" if !$d;
4501 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4502 } elsif ($opt eq 'vcpus
') {
4503 die "skip\n" if !$hotplug_features->{cpu};
4504 qemu_cpu_hotplug($vmid, $conf, $value);
4505 } elsif ($opt eq 'balloon
') {
4506 # enable/disable balloning device is not hotpluggable
4507 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4508 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4509 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4511 # allow manual ballooning if shares is set to zero
4512 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4513 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4514 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4516 } elsif ($opt =~ m/^net(\d+)$/) {
4517 # some changes can be done without hotplug
4518 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4519 $vmid, $opt, $value, $arch, $machine_type);
4520 } elsif (is_valid_drivename($opt)) {
4521 die "skip\n" if $opt eq 'efidisk0
';
4522 # some changes can be done without hotplug
4523 my $drive = parse_drive($opt, $value);
4524 if (drive_is_cloudinit($drive)) {
4525 &$apply_pending_cloudinit($opt, $value);
4527 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4528 $vmid, $opt, $value, $arch, $machine_type);
4529 } elsif ($opt =~ m/^memory$/) { #dimms
4530 die "skip\n" if !$hotplug_features->{memory};
4531 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4532 } elsif ($opt eq 'cpuunits
') {
4533 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4534 } elsif ($opt eq 'cpulimit
') {
4535 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4536 $cgroup->change_cpu_quota($cpulimit, 100000);
4538 die "skip\n"; # skip non-hot-pluggable options
4542 &$add_error($opt, $err) if $err ne "skip\n";
4544 $conf->{$opt} = $value;
4545 delete $conf->{pending}->{$opt};
4549 PVE::QemuConfig->write_config($vmid, $conf);
4552 sub try_deallocate_drive {
4553 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4555 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4556 my $volid = $drive->{file};
4557 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4558 my $sid = PVE::Storage::parse_volume_id($volid);
4559 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4561 # check if the disk is really unused
4562 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4563 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4564 PVE::Storage::vdisk_free($storecfg, $volid);
4567 # If vm is not owner of this disk remove from config
4575 sub vmconfig_delete_or_detach_drive {
4576 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4578 my $drive = parse_drive($opt, $conf->{$opt});
4580 my $rpcenv = PVE::RPCEnvironment::get();
4581 my $authuser = $rpcenv->get_user();
4584 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4585 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4587 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4593 sub vmconfig_apply_pending {
4594 my ($vmid, $conf, $storecfg, $errors) = @_;
4596 my $add_apply_error = sub {
4597 my ($opt, $msg) = @_;
4598 my $err_msg = "unable to apply pending change $opt : $msg";
4599 $errors->{$opt} = $err_msg;
4605 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4606 foreach my $opt (sort keys %$pending_delete_hash) {
4607 my $force = $pending_delete_hash->{$opt}->{force};
4609 if ($opt =~ m/^unused/) {
4610 die "internal error";
4611 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4612 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4616 $add_apply_error->($opt, $err);
4618 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4619 delete $conf->{$opt};
4623 PVE::QemuConfig->cleanup_pending($conf);
4625 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4626 next if $opt eq 'delete'; # just to be sure
4628 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4629 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4633 $add_apply_error->($opt, $err);
4635 $conf->{$opt} = delete $conf->{pending}->{$opt};
4639 # write all changes at once to avoid unnecessary i/o
4640 PVE::QemuConfig->write_config($vmid, $conf);
4643 sub vmconfig_update_net {
4644 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4646 my $newnet = parse_net($value);
4648 if ($conf->{$opt}) {
4649 my $oldnet = parse_net($conf->{$opt});
4651 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4652 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4653 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4654 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4656 # for non online change, we try to hot-unplug
4657 die "skip\n" if !$hotplug;
4658 vm_deviceunplug($vmid, $conf, $opt);
4661 die "internal error" if $opt !~ m/net(\d+)/;
4662 my $iface = "tap${vmid}i$1";
4664 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4665 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4666 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4667 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4668 PVE::Network::tap_unplug($iface);
4671 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4673 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4675 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4676 # Rate can be applied on its own but any change above needs to
4677 # include the rate in tap_plug since OVS resets everything.
4678 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4681 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4682 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4690 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4696 sub vmconfig_update_disk {
4697 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4699 my $drive = parse_drive($opt, $value);
4701 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4702 my $media = $drive->{media} || 'disk
';
4703 my $oldmedia = $old_drive->{media} || 'disk
';
4704 die "unable to change media type\n" if $media ne $oldmedia;
4706 if (!drive_is_cdrom($old_drive)) {
4708 if ($drive->{file} ne $old_drive->{file}) {
4710 die "skip\n" if !$hotplug;
4712 # unplug and register as unused
4713 vm_deviceunplug($vmid, $conf, $opt);
4714 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4717 # update existing disk
4719 # skip non hotpluggable value
4720 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4721 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4722 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4723 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4724 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4729 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4730 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4731 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4732 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4733 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4734 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4735 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4736 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4737 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4738 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4739 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4740 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4741 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4742 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4743 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4744 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4745 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4746 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4748 qemu_block_set_io_throttle(
4750 ($drive->{mbps} || 0)*1024*1024,
4751 ($drive->{mbps_rd} || 0)*1024*1024,
4752 ($drive->{mbps_wr} || 0)*1024*1024,
4753 $drive->{iops} || 0,
4754 $drive->{iops_rd} || 0,
4755 $drive->{iops_wr} || 0,
4756 ($drive->{mbps_max} || 0)*1024*1024,
4757 ($drive->{mbps_rd_max} || 0)*1024*1024,
4758 ($drive->{mbps_wr_max} || 0)*1024*1024,
4759 $drive->{iops_max} || 0,
4760 $drive->{iops_rd_max} || 0,
4761 $drive->{iops_wr_max} || 0,
4762 $drive->{bps_max_length} || 1,
4763 $drive->{bps_rd_max_length} || 1,
4764 $drive->{bps_wr_max_length} || 1,
4765 $drive->{iops_max_length} || 1,
4766 $drive->{iops_rd_max_length} || 1,
4767 $drive->{iops_wr_max_length} || 1,
4777 if ($drive->{file} eq 'none
') {
4778 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4779 if (drive_is_cloudinit($old_drive)) {
4780 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4783 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4785 # force eject if locked
4786 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4789 mon_cmd($vmid, "blockdev-change-medium",
4790 id => "$opt", filename => "$path");
4798 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4800 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4801 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4804 # called in locked context by incoming migration
4805 sub vm_migrate_get_nbd_disks {
4806 my ($storecfg, $conf, $replicated_volumes) = @_;
4808 my $local_volumes = {};
4809 PVE::QemuConfig->foreach_volume($conf, sub {
4810 my ($ds, $drive) = @_;
4812 return if drive_is_cdrom($drive);
4814 my $volid = $drive->{file};
4818 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4820 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4821 return if $scfg->{shared};
4823 # replicated disks re-use existing state via bitmap
4824 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4825 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4827 return $local_volumes;
4830 # called in locked context by incoming migration
4831 sub vm_migrate_alloc_nbd_disks {
4832 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4837 foreach my $opt (sort keys %$source_volumes) {
4838 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4840 if ($use_existing) {
4841 $nbd->{$opt}->{drivestr} = print_drive($drive);
4842 $nbd->{$opt}->{volid} = $volid;
4843 $nbd->{$opt}->{replicated} = 1;
4847 # If a remote storage is specified and the format of the original
4848 # volume is not available there, fall back to the default format.
4849 # Otherwise use the same format as the original.
4850 if (!$storagemap->{identity}) {
4851 $storeid = map_storage($storagemap, $storeid);
4852 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4853 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4854 my $fileFormat = qemu_img_format($scfg, $volname);
4855 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4857 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4858 $format = qemu_img_format($scfg, $volname);
4861 my $size = $drive->{size} / 1024;
4862 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4863 my $newdrive = $drive;
4864 $newdrive->{format} = $format;
4865 $newdrive->{file} = $newvolid;
4866 my $drivestr = print_drive($newdrive);
4867 $nbd->{$opt}->{drivestr} = $drivestr;
4868 $nbd->{$opt}->{volid} = $newvolid;
4874 # see vm_start_nolock for parameters, additionally:
4876 # storagemap = parsed storage map for allocating NBD disks
4878 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4880 return PVE::QemuConfig->lock_config($vmid, sub {
4881 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4883 die "you can't start a vm
if it
's a template\n"
4884 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4886 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended
');
4887 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup
');
4889 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4891 if ($has_backup_lock && $running) {
4892 # a backup is currently running, attempt to start the guest in the
4893 # existing QEMU instance
4894 return vm_resume($vmid);
4897 PVE::QemuConfig->check_lock($conf)
4898 if !($params->{skiplock} || $has_suspended_lock);
4900 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4902 die "VM $vmid already running\n" if $running;
4904 if (my $storagemap = $migrate_opts->{storagemap}) {
4905 my $replicated = $migrate_opts->{replicated_volumes};
4906 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4907 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4909 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4910 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4914 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4920 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
4921 # skiplock => 0/1, skip checking for config lock
4922 # skiptemplate => 0/1, skip checking whether VM is template
4923 # forcemachine => to force Qemu machine (rollback/migration)
4924 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
4925 # timeout => in seconds
4926 # paused => start VM in paused state (backup)
4927 # resume => resume from hibernation
4929 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4930 # migratedfrom => source node
4931 # spice_ticket => used for spice migration, passed via tunnel/stdin
4932 # network => CIDR of migration network
4933 # type => secure/insecure - tunnel over encrypted connection or plain-text
4934 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
4935 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4936 sub vm_start_nolock {
4937 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4939 my $statefile = $params->{statefile};
4940 my $resume = $params->{resume};
4942 my $migratedfrom = $migrate_opts->{migratedfrom};
4943 my $migration_type = $migrate_opts->{type};
4947 # clean up leftover reboot request files
4948 eval { clear_reboot_request($vmid); };
4951 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4952 vmconfig_apply_pending($vmid, $conf, $storecfg);
4953 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4956 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4958 my $defaults = load_defaults();
4960 # set environment variable useful inside network script
4961 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4963 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
4965 my $forcemachine = $params->{forcemachine};
4966 my $forcecpu = $params->{forcecpu};
4968 # enforce machine and CPU type on suspended vm to ensure HW compatibility
4969 $forcemachine = $conf->{runningmachine};
4970 $forcecpu = $conf->{runningcpu};
4971 print "Resuming suspended VM\n";
4974 my ($cmd, $vollist, $spice_port) =
4975 config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4978 my $get_migration_ip = sub {
4979 my ($nodename) = @_;
4981 return $migration_ip if defined($migration_ip);
4983 my $cidr = $migrate_opts->{network};
4985 if (!defined($cidr)) {
4986 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
4987 $cidr = $dc_conf->{migration}->{network};
4990 if (defined($cidr)) {
4991 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4993 die "could not get IP: no address configured on local " .
4994 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4996 die "could not get IP: multiple addresses configured on local " .
4997 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4999 $migration_ip = @$ips[0];
5002 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5003 if !defined($migration_ip);
5005 return $migration_ip;
5010 if ($statefile eq 'tcp
') {
5011 my $localip = "localhost";
5012 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5013 my $nodename = nodename();
5015 if (!defined($migration_type)) {
5016 if (defined($datacenterconf->{migration}->{type})) {
5017 $migration_type = $datacenterconf->{migration}->{type};
5019 $migration_type = 'secure
';
5023 if ($migration_type eq 'insecure
') {
5024 $localip = $get_migration_ip->($nodename);
5025 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5028 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5029 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5030 $migrate_uri = "tcp:${localip}:${migrate_port}";
5031 push @$cmd, '-incoming
', $migrate_uri;
5034 } elsif ($statefile eq 'unix
') {
5035 # should be default for secure migrations as a ssh TCP forward
5036 # tunnel is not deterministic reliable ready and fails regurarly
5037 # to set up in time, so use UNIX socket forwards
5038 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5039 unlink $socket_addr;
5041 $migrate_uri = "unix:$socket_addr";
5043 push @$cmd, '-incoming
', $migrate_uri;
5046 } elsif (-e $statefile) {
5047 push @$cmd, '-loadstate
', $statefile;
5049 my $statepath = PVE::Storage::path($storecfg, $statefile);
5050 push @$vollist, $statefile;
5051 push @$cmd, '-loadstate
', $statepath;
5053 } elsif ($params->{paused}) {
5058 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
5059 my $d = parse_hostpci($conf->{"hostpci$i"});
5061 my $pcidevices = $d->{pciid};
5062 foreach my $pcidevice (@$pcidevices) {
5063 my $pciid = $pcidevice->{id};
5065 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5066 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5067 die "no pci device info for device '$pciid'\n" if !$info;
5070 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5071 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5073 die "can't unbind
/bind PCI group to VFIO
'$pciid'\n"
5074 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5075 die "can
't reset PCI device '$pciid'\n"
5076 if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5081 PVE::Storage::activate_volumes($storecfg, $vollist);
5084 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5085 outfunc => sub {}, errfunc => sub {});
5087 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5088 # timeout should be more than enough here...
5089 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5091 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5092 : $defaults->{cpuunits};
5094 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5096 timeout => $statefile ? undef : $start_timeout,
5101 # when migrating, prefix QEMU output so other side can pick up any
5102 # errors that might occur and show the user
5103 if ($migratedfrom) {
5104 $run_params{quiet} = 1;
5105 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5109 Slice => 'qemu
.slice
',
5113 if (PVE::CGroup::cgroup_mode() == 2) {
5114 $properties{CPUWeight} = $cpuunits;
5116 $properties{CPUShares} = $cpuunits;
5119 if (my $cpulimit = $conf->{cpulimit}) {
5120 $properties{CPUQuota} = int($cpulimit * 100);
5122 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5124 my $run_qemu = sub {
5125 PVE::Tools::run_fork sub {
5126 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5128 my $exitcode = run_command($cmd, %run_params);
5129 die "QEMU exited with code $exitcode\n" if $exitcode;
5133 if ($conf->{hugepages}) {
5136 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5137 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5139 PVE::QemuServer::Memory::hugepages_mount();
5140 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5142 eval { $run_qemu->() };
5144 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5145 if !$conf->{keephugepages};
5149 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5150 if !$conf->{keephugepages};
5152 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5155 eval { $run_qemu->() };
5159 # deactivate volumes if start fails
5160 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5161 die "start failed: $err";
5164 print "migration listens on $migrate_uri\n" if $migrate_uri;
5165 $res->{migrate_uri} = $migrate_uri;
5167 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5168 eval { mon_cmd($vmid, "cont"); };
5172 #start nbd server for storage migration
5173 if (my $nbd = $migrate_opts->{nbd}) {
5174 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5176 my $migrate_storage_uri;
5177 # nbd_protocol_version > 0 for unix socket support
5178 if ($nbd_protocol_version > 0 && $migration_type eq 'secure
') {
5179 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5180 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix
', data => { path => $socket_path } } );
5181 $migrate_storage_uri = "nbd:unix:$socket_path";
5183 my $nodename = nodename();
5184 my $localip = $get_migration_ip->($nodename);
5185 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5186 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5188 mon_cmd($vmid, "nbd-server-start", addr => {
5191 host => "${localip}",
5192 port => "${storage_migrate_port}",
5195 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5196 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5199 $res->{migrate_storage_uri} = $migrate_storage_uri;
5201 foreach my $opt (sort keys %$nbd) {
5202 my $drivestr = $nbd->{$opt}->{drivestr};
5203 my $volid = $nbd->{$opt}->{volid};
5204 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5205 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5206 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5207 print "re-using replicated volume: $opt - $volid\n"
5208 if $nbd->{$opt}->{replicated};
5210 $res->{drives}->{$opt} = $nbd->{$opt};
5211 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5215 if ($migratedfrom) {
5217 set_migration_caps($vmid);
5222 print "spice listens on port $spice_port\n";
5223 $res->{spice_port} = $spice_port;
5224 if ($migrate_opts->{spice_ticket}) {
5225 mon_cmd($vmid, "set_password", protocol => 'spice
', password =>
5226 $migrate_opts->{spice_ticket});
5227 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5232 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5233 if !$statefile && $conf->{balloon};
5235 foreach my $opt (keys %$conf) {
5236 next if $opt !~ m/^net\d+$/;
5237 my $nicconf = parse_net($conf->{$opt});
5238 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5242 mon_cmd($vmid, 'qom-set
',
5243 path => "machine/peripheral/balloon0",
5244 property => "guest-stats-polling-interval",
5245 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5248 print "Resumed VM, removing state\n";
5249 if (my $vmstate = $conf->{vmstate}) {
5250 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5251 PVE::Storage::vdisk_free($storecfg, $vmstate);
5253 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5254 PVE
::QemuConfig-
>write_config($vmid, $conf);
5257 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5262 sub vm_commandline
{
5263 my ($storecfg, $vmid, $snapname) = @_;
5265 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5270 my $snapshot = $conf->{snapshots
}->{$snapname};
5271 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5273 # check for machine or CPU overrides in snapshot
5274 $forcemachine = $snapshot->{runningmachine
};
5275 $forcecpu = $snapshot->{runningcpu
};
5277 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5282 my $defaults = load_defaults
();
5284 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults,
5285 $forcemachine, $forcecpu);
5287 return PVE
::Tools
::cmd2string
($cmd);
5291 my ($vmid, $skiplock) = @_;
5293 PVE
::QemuConfig-
>lock_config($vmid, sub {
5295 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5297 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5299 mon_cmd
($vmid, "system_reset");
5303 sub get_vm_volumes
{
5307 foreach_volid
($conf, sub {
5308 my ($volid, $attr) = @_;
5310 return if $volid =~ m
|^/|;
5312 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5315 push @$vollist, $volid;
5321 sub vm_stop_cleanup
{
5322 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5327 my $vollist = get_vm_volumes
($conf);
5328 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5331 foreach my $ext (qw(mon qmp pid vnc qga)) {
5332 unlink "/var/run/qemu-server/${vmid}.$ext";
5335 if ($conf->{ivshmem
}) {
5336 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5337 # just delete it for now, VMs which have this already open do not
5338 # are affected, but new VMs will get a separated one. If this
5339 # becomes an issue we either add some sort of ref-counting or just
5340 # add a "don't delete on stop" flag to the ivshmem format.
5341 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5344 foreach my $key (keys %$conf) {
5345 next if $key !~ m/^hostpci(\d+)$/;
5346 my $hostpciindex = $1;
5347 my $d = parse_hostpci
($conf->{$key});
5348 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5350 foreach my $pci (@{$d->{pciid
}}) {
5351 my $pciid = $pci->{id
};
5352 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5356 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5358 warn $@ if $@; # avoid errors - just warn
5361 # call only in locked context
5363 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5365 my $pid = check_running
($vmid, $nocheck);
5370 $conf = PVE
::QemuConfig-
>load_config($vmid);
5371 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5372 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5373 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5374 $timeout = $opts->{down
} if $opts->{down
};
5376 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5381 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
5382 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5384 mon_cmd
($vmid, "system_powerdown");
5387 mon_cmd
($vmid, "quit");
5393 $timeout = 60 if !defined($timeout);
5396 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5401 if ($count >= $timeout) {
5403 warn "VM still running - terminating now with SIGTERM\n";
5406 die "VM quit/powerdown failed - got timeout\n";
5409 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5413 if (!check_running
($vmid, $nocheck)) {
5414 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5418 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5421 die "VM quit/powerdown failed\n";
5429 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5434 if ($count >= $timeout) {
5435 warn "VM still running - terminating now with SIGKILL\n";
5440 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5443 # Note: use $nocheck to skip tests if VM configuration file exists.
5444 # We need that when migration VMs to other nodes (files already moved)
5445 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5447 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5449 $force = 1 if !defined($force) && !$shutdown;
5452 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5453 kill 15, $pid if $pid;
5454 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5455 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5459 PVE
::QemuConfig-
>lock_config($vmid, sub {
5460 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5465 my ($vmid, $timeout) = @_;
5467 PVE
::QemuConfig-
>lock_config($vmid, sub {
5470 # only reboot if running, as qmeventd starts it again on a stop event
5471 return if !check_running
($vmid);
5473 create_reboot_request
($vmid);
5475 my $storecfg = PVE
::Storage
::config
();
5476 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5480 # avoid that the next normal shutdown will be confused for a reboot
5481 clear_reboot_request
($vmid);
5487 # note: if using the statestorage parameter, the caller has to check privileges
5489 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5496 PVE
::QemuConfig-
>lock_config($vmid, sub {
5498 $conf = PVE
::QemuConfig-
>load_config($vmid);
5500 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5501 PVE
::QemuConfig-
>check_lock($conf)
5502 if !($skiplock || $is_backing_up);
5504 die "cannot suspend to disk during backup\n"
5505 if $is_backing_up && $includestate;
5507 if ($includestate) {
5508 $conf->{lock} = 'suspending';
5509 my $date = strftime
("%Y-%m-%d", localtime(time()));
5510 $storecfg = PVE
::Storage
::config
();
5511 if (!$statestorage) {
5512 $statestorage = find_vmstate_storage
($conf, $storecfg);
5513 # check permissions for the storage
5514 my $rpcenv = PVE
::RPCEnvironment
::get
();
5515 if ($rpcenv->{type
} ne 'cli') {
5516 my $authuser = $rpcenv->get_user();
5517 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5522 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
5523 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5524 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5525 PVE
::QemuConfig-
>write_config($vmid, $conf);
5527 mon_cmd
($vmid, "stop");
5531 if ($includestate) {
5533 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5536 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5538 my $state = mon_cmd
($vmid, "query-savevm");
5539 if (!$state->{status
}) {
5540 die "savevm not active\n";
5541 } elsif ($state->{status
} eq 'active') {
5544 } elsif ($state->{status
} eq 'completed') {
5545 print "State saved, quitting\n";
5547 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5548 die "query-savevm failed with error '$state->{error}'\n"
5550 die "query-savevm returned status '$state->{status}'\n";
5556 PVE
::QemuConfig-
>lock_config($vmid, sub {
5557 $conf = PVE
::QemuConfig-
>load_config($vmid);
5559 # cleanup, but leave suspending lock, to indicate something went wrong
5561 mon_cmd
($vmid, "savevm-end");
5562 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5563 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5564 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5565 PVE
::QemuConfig-
>write_config($vmid, $conf);
5571 die "lock changed unexpectedly\n"
5572 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5574 mon_cmd
($vmid, "quit");
5575 $conf->{lock} = 'suspended';
5576 PVE
::QemuConfig-
>write_config($vmid, $conf);
5582 my ($vmid, $skiplock, $nocheck) = @_;
5584 PVE
::QemuConfig-
>lock_config($vmid, sub {
5585 my $res = mon_cmd
($vmid, 'query-status');
5586 my $resume_cmd = 'cont';
5589 if ($res->{status
}) {
5590 return if $res->{status
} eq 'running'; # job done, go home
5591 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
5592 $reset = 1 if $res->{status
} eq 'shutdown';
5597 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5599 PVE
::QemuConfig-
>check_lock($conf)
5600 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5604 # required if a VM shuts down during a backup and we get a resume
5605 # request before the backup finishes for example
5606 mon_cmd
($vmid, "system_reset");
5608 mon_cmd
($vmid, $resume_cmd);
5613 my ($vmid, $skiplock, $key) = @_;
5615 PVE
::QemuConfig-
>lock_config($vmid, sub {
5617 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5619 # there is no qmp command, so we use the human monitor command
5620 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5621 die $res if $res ne '';
5625 # vzdump restore implementaion
5627 sub tar_archive_read_firstfile
{
5628 my $archive = shift;
5630 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5632 # try to detect archive type first
5633 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5634 die "unable to open file '$archive'\n";
5635 my $firstfile = <$fh>;
5639 die "ERROR: archive contaions no data\n" if !$firstfile;
5645 sub tar_restore_cleanup
{
5646 my ($storecfg, $statfile) = @_;
5648 print STDERR
"starting cleanup\n";
5650 if (my $fd = IO
::File-
>new($statfile, "r")) {
5651 while (defined(my $line = <$fd>)) {
5652 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5655 if ($volid =~ m
|^/|) {
5656 unlink $volid || die 'unlink failed\n';
5658 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5660 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5662 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5664 print STDERR
"unable to parse line in statfile - $line";
5671 sub restore_file_archive
{
5672 my ($archive, $vmid, $user, $opts) = @_;
5674 return restore_vma_archive
($archive, $vmid, $user, $opts)
5677 my $info = PVE
::Storage
::archive_info
($archive);
5678 my $format = $opts->{format
} // $info->{format
};
5679 my $comp = $info->{compression
};
5681 # try to detect archive format
5682 if ($format eq 'tar') {
5683 return restore_tar_archive
($archive, $vmid, $user, $opts);
5685 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5689 # hepler to remove disks that will not be used after restore
5690 my $restore_cleanup_oldconf = sub {
5691 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5693 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
5694 my ($ds, $drive) = @_;
5696 return if drive_is_cdrom
($drive, 1);
5698 my $volid = $drive->{file
};
5699 return if !$volid || $volid =~ m
|^/|;
5701 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
5702 return if !$path || !$owner || ($owner != $vmid);
5704 # Note: only delete disk we want to restore
5705 # other volumes will become unused
5706 if ($virtdev_hash->{$ds}) {
5707 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
5714 # delete vmstate files, after the restore we have no snapshots anymore
5715 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5716 my $snap = $oldconf->{snapshots
}->{$snapname};
5717 if ($snap->{vmstate
}) {
5718 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
5726 # Helper to parse vzdump backup device hints
5728 # $rpcenv: Environment, used to ckeck storage permissions
5729 # $user: User ID, to check storage permissions
5730 # $storecfg: Storage configuration
5731 # $fh: the file handle for reading the configuration
5732 # $devinfo: should contain device sizes for all backu-up'ed devices
5733 # $options: backup options (pool, default storage)
5735 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5736 my $parse_backup_hints = sub {
5737 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5739 my $virtdev_hash = {};
5741 while (defined(my $line = <$fh>)) {
5742 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5743 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5744 die "archive does not contain data for drive '$virtdev'\n"
5745 if !$devinfo->{$devname};
5747 if (defined($options->{storage
})) {
5748 $storeid = $options->{storage
} || 'local';
5749 } elsif (!$storeid) {
5752 $format = 'raw' if !$format;
5753 $devinfo->{$devname}->{devname
} = $devname;
5754 $devinfo->{$devname}->{virtdev
} = $virtdev;
5755 $devinfo->{$devname}->{format
} = $format;
5756 $devinfo->{$devname}->{storeid
} = $storeid;
5758 # check permission on storage
5759 my $pool = $options->{pool
}; # todo: do we need that?
5760 if ($user ne 'root@pam') {
5761 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5764 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5765 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5767 my $drive = parse_drive
($virtdev, $2);
5768 if (drive_is_cloudinit
($drive)) {
5769 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
5770 $storeid = $options->{storage
} if defined ($options->{storage
});
5771 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5772 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
5774 $virtdev_hash->{$virtdev} = {
5776 storeid
=> $storeid,
5777 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
5784 return $virtdev_hash;
5787 # Helper to allocate and activate all volumes required for a restore
5789 # $storecfg: Storage configuration
5790 # $virtdev_hash: as returned by parse_backup_hints()
5792 # Returns: { $virtdev => $volid }
5793 my $restore_allocate_devices = sub {
5794 my ($storecfg, $virtdev_hash, $vmid) = @_;
5797 foreach my $virtdev (sort keys %$virtdev_hash) {
5798 my $d = $virtdev_hash->{$virtdev};
5799 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5800 my $storeid = $d->{storeid
};
5801 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5803 # test if requested format is supported
5804 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5805 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
5806 $d->{format
} = $defFormat if !$supported;
5809 if ($d->{is_cloudinit
}) {
5810 $name = "vm-$vmid-cloudinit";
5811 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5812 if ($scfg->{path
}) {
5813 $name .= ".$d->{format}";
5817 my $volid = PVE
::Storage
::vdisk_alloc
(
5818 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
5820 print STDERR
"new volume ID is '$volid'\n";
5821 $d->{volid
} = $volid;
5823 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
5825 $map->{$virtdev} = $volid;
5831 my $restore_update_config_line = sub {
5832 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5834 return if $line =~ m/^\#qmdump\#/;
5835 return if $line =~ m/^\#vzdump\#/;
5836 return if $line =~ m/^lock:/;
5837 return if $line =~ m/^unused\d+:/;
5838 return if $line =~ m/^parent:/;
5840 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5841 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5842 # try to convert old 1.X settings
5843 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5844 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5845 my ($model, $macaddr) = split(/\=/, $devconfig);
5846 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5849 bridge
=> "vmbr$ind",
5850 macaddr
=> $macaddr,
5852 my $netstr = print_net
($net);
5854 print $outfd "net$cookie->{netcount}: $netstr\n";
5855 $cookie->{netcount
}++;
5857 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5858 my ($id, $netstr) = ($1, $2);
5859 my $net = parse_net
($netstr);
5860 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5861 $netstr = print_net
($net);
5862 print $outfd "$id: $netstr\n";
5863 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5866 my $di = parse_drive
($virtdev, $value);
5867 if (defined($di->{backup
}) && !$di->{backup
}) {
5868 print $outfd "#$line";
5869 } elsif ($map->{$virtdev}) {
5870 delete $di->{format
}; # format can change on restore
5871 $di->{file
} = $map->{$virtdev};
5872 $value = print_drive
($di);
5873 print $outfd "$virtdev: $value\n";
5877 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5879 if ($vmgenid ne '0') {
5880 # always generate a new vmgenid if there was a valid one setup
5881 $vmgenid = generate_uuid
();
5883 print $outfd "vmgenid: $vmgenid\n";
5884 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5885 my ($uuid, $uuid_str);
5886 UUID
::generate
($uuid);
5887 UUID
::unparse
($uuid, $uuid_str);
5888 my $smbios1 = parse_smbios1
($2);
5889 $smbios1->{uuid
} = $uuid_str;
5890 print $outfd $1.print_smbios1
($smbios1)."\n";
5896 my $restore_deactivate_volumes = sub {
5897 my ($storecfg, $devinfo) = @_;
5900 foreach my $devname (keys %$devinfo) {
5901 my $volid = $devinfo->{$devname}->{volid
};
5902 push @$vollist, $volid if $volid;
5905 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5908 my $restore_destroy_volumes = sub {
5909 my ($storecfg, $devinfo) = @_;
5911 foreach my $devname (keys %$devinfo) {
5912 my $volid = $devinfo->{$devname}->{volid
};
5915 if ($volid =~ m
|^/|) {
5916 unlink $volid || die 'unlink failed\n';
5918 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5920 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5922 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5927 my ($cfg, $vmid) = @_;
5929 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5931 my $volid_hash = {};
5932 foreach my $storeid (keys %$info) {
5933 foreach my $item (@{$info->{$storeid}}) {
5934 next if !($item->{volid
} && $item->{size
});
5935 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5936 $volid_hash->{$item->{volid
}} = $item;
5943 sub update_disk_config
{
5944 my ($vmid, $conf, $volid_hash) = @_;
5947 my $prefix = "VM $vmid";
5949 # used and unused disks
5950 my $referenced = {};
5952 # Note: it is allowed to define multiple storages with same path (alias), so
5953 # we need to check both 'volid' and real 'path' (two different volid can point
5954 # to the same path).
5956 my $referencedpath = {};
5959 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5960 my ($opt, $drive) = @_;
5962 my $volid = $drive->{file
};
5964 my $volume = $volid_hash->{$volid};
5966 # mark volid as "in-use" for next step
5967 $referenced->{$volid} = 1;
5968 if ($volume && (my $path = $volume->{path
})) {
5969 $referencedpath->{$path} = 1;
5972 return if drive_is_cdrom
($drive);
5975 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
5976 if (defined($updated)) {
5978 $conf->{$opt} = print_drive
($updated);
5979 print "$prefix ($opt): $msg\n";
5983 # remove 'unusedX' entry if volume is used
5984 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
5985 my ($opt, $drive) = @_;
5987 my $volid = $drive->{file
};
5991 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
5992 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5993 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5995 delete $conf->{$opt};
5998 $referenced->{$volid} = 1;
5999 $referencedpath->{$path} = 1 if $path;
6002 foreach my $volid (sort keys %$volid_hash) {
6003 next if $volid =~ m/vm-$vmid-state-/;
6004 next if $referenced->{$volid};
6005 my $path = $volid_hash->{$volid}->{path
};
6006 next if !$path; # just to be sure
6007 next if $referencedpath->{$path};
6009 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6010 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6011 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6018 my ($vmid, $nolock, $dryrun) = @_;
6020 my $cfg = PVE
::Storage
::config
();
6022 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6023 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6024 foreach my $stor (keys %{$cfg->{ids
}}) {
6025 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
6028 print "rescan volumes...\n";
6029 my $volid_hash = scan_volids
($cfg, $vmid);
6031 my $updatefn = sub {
6034 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6036 PVE
::QemuConfig-
>check_lock($conf);
6039 foreach my $volid (keys %$volid_hash) {
6040 my $info = $volid_hash->{$volid};
6041 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6044 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6046 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6049 if (defined($vmid)) {
6053 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6056 my $vmlist = config_list
();
6057 foreach my $vmid (keys %$vmlist) {
6061 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6067 sub restore_proxmox_backup_archive
{
6068 my ($archive, $vmid, $user, $options) = @_;
6070 my $storecfg = PVE
::Storage
::config
();
6072 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6073 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6075 my $fingerprint = $scfg->{fingerprint
};
6076 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6078 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6080 # This is only used for `pbs-restore`!
6081 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6082 local $ENV{PBS_PASSWORD
} = $password;
6083 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6085 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6086 PVE
::Storage
::parse_volname
($storecfg, $archive);
6088 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6090 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6092 my $tmpdir = "/var/tmp/vzdumptmp$$";
6096 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6097 my $tmpfn = "$conffile.$$.tmp";
6098 # disable interrupts (always do cleanups)
6102 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6104 # Note: $oldconf is undef if VM does not exists
6105 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6106 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6108 my $rpcenv = PVE
::RPCEnvironment
::get
();
6117 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6119 my $cfgfn = "$tmpdir/qemu-server.conf";
6120 my $firewall_config_fn = "$tmpdir/fw.conf";
6121 my $index_fn = "$tmpdir/index.json";
6123 my $cmd = "restore";
6125 my $param = [$pbs_backup_name, "index.json", $index_fn];
6126 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6127 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6128 $index = decode_json
($index);
6130 # print Dumper($index);
6131 foreach my $info (@{$index->{files
}}) {
6132 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6134 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6135 $devinfo->{$devname}->{size
} = $1;
6137 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6142 my $is_qemu_server_backup = scalar(
6143 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6145 if (!$is_qemu_server_backup) {
6146 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6148 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6150 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6151 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6153 if ($has_firewall_config) {
6154 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6155 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6157 my $pve_firewall_dir = '/etc/pve/firewall';
6158 mkdir $pve_firewall_dir; # make sure the dir exists
6159 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6162 my $fh = IO
::File-
>new($cfgfn, "r") ||
6163 die "unable to read qemu-server.conf - $!\n";
6165 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6167 # fixme: rate limit?
6169 # create empty/temp config
6170 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6172 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6175 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6177 foreach my $virtdev (sort keys %$virtdev_hash) {
6178 my $d = $virtdev_hash->{$virtdev};
6179 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6181 my $volid = $d->{volid
};
6183 my $path = PVE
::Storage
::path
($storecfg, $volid);
6185 # This is the ONLY user of the PBS_ env vars set on top of this function!
6186 my $pbs_restore_cmd = [
6187 '/usr/bin/pbs-restore',
6188 '--repository', $repo,
6190 "$d->{devname}.img.fidx",
6195 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6196 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6198 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6199 push @$pbs_restore_cmd, '--skip-zero';
6202 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6203 print "restore proxmox backup image: $dbg_cmdstring\n";
6204 run_command
($pbs_restore_cmd);
6207 $fh->seek(0, 0) || die "seek failed - $!\n";
6209 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6211 my $cookie = { netcount
=> 0 };
6212 while (defined(my $line = <$fh>)) {
6213 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique
});
6221 $restore_deactivate_volumes->($storecfg, $devinfo);
6227 $restore_destroy_volumes->($storecfg, $devinfo);
6231 rename($tmpfn, $conffile) ||
6232 die "unable to commit configuration file '$conffile'\n";
6234 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6236 eval { rescan
($vmid, 1); };
6240 sub restore_vma_archive
{
6241 my ($archive, $vmid, $user, $opts, $comp) = @_;
6243 my $readfrom = $archive;
6245 my $cfg = PVE
::Storage
::config
();
6247 my $bwlimit = $opts->{bwlimit
};
6249 my $dbg_cmdstring = '';
6250 my $add_pipe = sub {
6252 push @$commands, $cmd;
6253 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6254 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6259 if ($archive eq '-') {
6262 # If we use a backup from a PVE defined storage we also consider that
6263 # storage's rate limit:
6264 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6265 if (defined($volid)) {
6266 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6267 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6269 print STDERR
"applying read rate limit: $readlimit\n";
6270 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6271 $add_pipe->($cstream);
6277 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6278 my $cmd = $info->{decompressor
};
6279 push @$cmd, $readfrom;
6283 my $tmpdir = "/var/tmp/vzdumptmp$$";
6286 # disable interrupts (always do cleanups)
6290 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6292 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6293 POSIX
::mkfifo
($mapfifo, 0600);
6295 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6297 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6304 my $rpcenv = PVE
::RPCEnvironment
::get
();
6306 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6307 my $tmpfn = "$conffile.$$.tmp";
6309 # Note: $oldconf is undef if VM does not exist
6310 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6311 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6315 my $print_devmap = sub {
6316 my $cfgfn = "$tmpdir/qemu-server.conf";
6318 # we can read the config - that is already extracted
6319 my $fh = IO
::File-
>new($cfgfn, "r") ||
6320 die "unable to read qemu-server.conf - $!\n";
6322 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6324 my $pve_firewall_dir = '/etc/pve/firewall';
6325 mkdir $pve_firewall_dir; # make sure the dir exists
6326 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6329 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6331 foreach my $key (keys %storage_limits) {
6332 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6334 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6335 $storage_limits{$key} = $limit * 1024;
6338 foreach my $devname (keys %$devinfo) {
6339 die "found no device mapping information for device '$devname'\n"
6340 if !$devinfo->{$devname}->{virtdev
};
6343 # create empty/temp config
6345 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6346 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6350 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6352 # print restore information to $fifofh
6353 foreach my $virtdev (sort keys %$virtdev_hash) {
6354 my $d = $virtdev_hash->{$virtdev};
6355 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6357 my $storeid = $d->{storeid
};
6358 my $volid = $d->{volid
};
6361 if (my $limit = $storage_limits{$storeid}) {
6362 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6365 my $write_zeros = 1;
6366 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6370 my $path = PVE
::Storage
::path
($cfg, $volid);
6372 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6374 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6377 $fh->seek(0, 0) || die "seek failed - $!\n";
6379 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6381 my $cookie = { netcount
=> 0 };
6382 while (defined(my $line = <$fh>)) {
6383 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6396 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6397 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6399 $oldtimeout = alarm($timeout);
6406 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6407 my ($dev_id, $size, $devname) = ($1, $2, $3);
6408 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6409 } elsif ($line =~ m/^CTIME: /) {
6410 # we correctly received the vma config, so we can disable
6411 # the timeout now for disk allocation (set to 10 minutes, so
6412 # that we always timeout if something goes wrong)
6415 print $fifofh "done\n";
6416 my $tmp = $oldtimeout || 0;
6417 $oldtimeout = undef;
6424 print "restore vma archive: $dbg_cmdstring\n";
6425 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6429 alarm($oldtimeout) if $oldtimeout;
6431 $restore_deactivate_volumes->($cfg, $devinfo);
6433 close($fifofh) if $fifofh;
6439 $restore_destroy_volumes->($cfg, $devinfo);
6443 rename($tmpfn, $conffile) ||
6444 die "unable to commit configuration file '$conffile'\n";
6446 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6448 eval { rescan
($vmid, 1); };
6452 sub restore_tar_archive
{
6453 my ($archive, $vmid, $user, $opts) = @_;
6455 if ($archive ne '-') {
6456 my $firstfile = tar_archive_read_firstfile
($archive);
6457 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6458 if $firstfile ne 'qemu-server.conf';
6461 my $storecfg = PVE
::Storage
::config
();
6463 # avoid zombie disks when restoring over an existing VM -> cleanup first
6464 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6465 # skiplock=1 because qmrestore has set the 'create' lock itself already
6466 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6467 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6469 my $tocmd = "/usr/lib/qemu-server/qmextract";
6471 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6472 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6473 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6474 $tocmd .= ' --info' if $opts->{info
};
6476 # tar option "xf" does not autodetect compression when read from STDIN,
6477 # so we pipe to zcat
6478 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6479 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6481 my $tmpdir = "/var/tmp/vzdumptmp$$";
6484 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6485 local $ENV{VZDUMP_VMID
} = $vmid;
6486 local $ENV{VZDUMP_USER
} = $user;
6488 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6489 my $tmpfn = "$conffile.$$.tmp";
6491 # disable interrupts (always do cleanups)
6495 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6503 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6505 if ($archive eq '-') {
6506 print "extracting archive from STDIN\n";
6507 run_command
($cmd, input
=> "<&STDIN");
6509 print "extracting archive '$archive'\n";
6513 return if $opts->{info
};
6517 my $statfile = "$tmpdir/qmrestore.stat";
6518 if (my $fd = IO
::File-
>new($statfile, "r")) {
6519 while (defined (my $line = <$fd>)) {
6520 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6521 $map->{$1} = $2 if $1;
6523 print STDERR
"unable to parse line in statfile - $line\n";
6529 my $confsrc = "$tmpdir/qemu-server.conf";
6531 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6533 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6535 my $cookie = { netcount
=> 0 };
6536 while (defined (my $line = <$srcfd>)) {
6537 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6545 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6551 rename $tmpfn, $conffile ||
6552 die "unable to commit configuration file '$conffile'\n";
6554 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6556 eval { rescan
($vmid, 1); };
6560 sub foreach_storage_used_by_vm
{
6561 my ($conf, $func) = @_;
6565 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6566 my ($ds, $drive) = @_;
6567 return if drive_is_cdrom
($drive);
6569 my $volid = $drive->{file
};
6571 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6572 $sidhash->{$sid} = $sid if $sid;
6575 foreach my $sid (sort keys %$sidhash) {
6580 my $qemu_snap_storage = {
6583 sub do_snapshots_with_qemu
{
6584 my ($storecfg, $volid) = @_;
6586 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6587 my $scfg = $storecfg->{ids
}->{$storage_name};
6588 die "could not find storage '$storage_name'\n" if !defined($scfg);
6590 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6594 if ($volid =~ m/\.(qcow2|qed)$/){
6601 sub qga_check_running
{
6602 my ($vmid, $nowarn) = @_;
6604 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6606 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6612 sub template_create
{
6613 my ($vmid, $conf, $disk) = @_;
6615 my $storecfg = PVE
::Storage
::config
();
6617 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6618 my ($ds, $drive) = @_;
6620 return if drive_is_cdrom
($drive);
6621 return if $disk && $ds ne $disk;
6623 my $volid = $drive->{file
};
6624 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6626 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6627 $drive->{file
} = $voliddst;
6628 $conf->{$ds} = print_drive
($drive);
6629 PVE
::QemuConfig-
>write_config($vmid, $conf);
6633 sub convert_iscsi_path
{
6636 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6641 my $initiator_name = get_initiator_name
();
6643 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6644 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6647 die "cannot convert iscsi path '$path', unkown format\n";
6650 sub qemu_img_convert
{
6651 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6653 my $storecfg = PVE
::Storage
::config
();
6654 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6655 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6657 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6661 my $src_is_iscsi = 0;
6665 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6666 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6667 $src_format = qemu_img_format
($src_scfg, $src_volname);
6668 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6669 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6670 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6671 } elsif (-f
$src_volid) {
6672 $src_path = $src_volid;
6673 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6678 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6680 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6681 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6682 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6683 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6686 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6687 push @$cmd, '-l', "snapshot.name=$snapname"
6688 if $snapname && $src_format && $src_format eq "qcow2";
6689 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6690 push @$cmd, '-T', $cachemode if defined($cachemode);
6692 if ($src_is_iscsi) {
6693 push @$cmd, '--image-opts';
6694 $src_path = convert_iscsi_path
($src_path);
6695 } elsif ($src_format) {
6696 push @$cmd, '-f', $src_format;
6699 if ($dst_is_iscsi) {
6700 push @$cmd, '--target-image-opts';
6701 $dst_path = convert_iscsi_path
($dst_path);
6703 push @$cmd, '-O', $dst_format;
6706 push @$cmd, $src_path;
6708 if (!$dst_is_iscsi && $is_zero_initialized) {
6709 push @$cmd, "zeroinit:$dst_path";
6711 push @$cmd, $dst_path;
6716 if($line =~ m/\((\S+)\/100\
%\)/){
6718 my $transferred = int($size * $percent / 100);
6719 my $remaining = $size - $transferred;
6721 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6726 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6728 die "copy failed: $err" if $err;
6731 sub qemu_img_format
{
6732 my ($scfg, $volname) = @_;
6734 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6741 sub qemu_drive_mirror
{
6742 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6744 $jobs = {} if !$jobs;
6748 $jobs->{"drive-$drive"} = {};
6750 if ($dst_volid =~ /^nbd:/) {
6751 $qemu_target = $dst_volid;
6754 my $storecfg = PVE
::Storage
::config
();
6755 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6757 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6759 $format = qemu_img_format
($dst_scfg, $dst_volname);
6761 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6763 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6766 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6767 $opts->{format
} = $format if $format;
6769 if (defined($src_bitmap)) {
6770 $opts->{sync
} = 'incremental';
6771 $opts->{bitmap
} = $src_bitmap;
6772 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6775 if (defined($bwlimit)) {
6776 $opts->{speed
} = $bwlimit * 1024;
6777 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6779 print "drive mirror is starting for drive-$drive\n";
6782 # if a job already runs for this device we get an error, catch it for cleanup
6783 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
6785 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6787 die "mirroring error: $err\n";
6790 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
6793 # $completion can be either
6794 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6795 # 'cancel': wait until all jobs are ready, block-job-cancel them
6796 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6797 sub qemu_drive_mirror_monitor
{
6798 my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6800 $completion //= 'complete';
6803 my $err_complete = 0;
6806 die "storage migration timed out\n" if $err_complete > 300;
6808 my $stats = mon_cmd
($vmid, "query-block-jobs");
6810 my $running_mirror_jobs = {};
6811 foreach my $stat (@$stats) {
6812 next if $stat->{type
} ne 'mirror';
6813 $running_mirror_jobs->{$stat->{device
}} = $stat;
6816 my $readycounter = 0;
6818 foreach my $job (keys %$jobs) {
6820 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6821 print "$job : finished\n";
6822 delete $jobs->{$job};
6826 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6828 my $busy = $running_mirror_jobs->{$job}->{busy
};
6829 my $ready = $running_mirror_jobs->{$job}->{ready
};
6830 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6831 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6832 my $remaining = $total - $transferred;
6833 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6835 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6838 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6841 last if scalar(keys %$jobs) == 0;
6843 if ($readycounter == scalar(keys %$jobs)) {
6844 print "all mirroring jobs are ready \n";
6845 last if $completion eq 'skip'; #do the complete later
6847 if ($vmiddst && $vmiddst != $vmid) {
6848 my $agent_running = $qga && qga_check_running
($vmid);
6849 if ($agent_running) {
6850 print "freeze filesystem\n";
6851 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6853 print "suspend vm\n";
6854 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6857 # if we clone a disk for a new target vm, we don't switch the disk
6858 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6860 if ($agent_running) {
6861 print "unfreeze filesystem\n";
6862 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6864 print "resume vm\n";
6865 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6871 foreach my $job (keys %$jobs) {
6872 # try to switch the disk if source and destination are on the same guest
6873 print "$job: Completing block job...\n";
6876 if ($completion eq 'complete') {
6877 $op = 'block-job-complete';
6878 } elsif ($completion eq 'cancel') {
6879 $op = 'block-job-cancel';
6881 die "invalid completion value: $completion\n";
6883 eval { mon_cmd
($vmid, $op, device
=> $job) };
6884 if ($@ =~ m/cannot be completed/) {
6885 print "$job: Block job cannot be completed, try again.\n";
6888 print "$job: Completed successfully.\n";
6889 $jobs->{$job}->{complete
} = 1;
6900 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6901 die "mirroring error: $err";
6906 sub qemu_blockjobs_cancel
{
6907 my ($vmid, $jobs) = @_;
6909 foreach my $job (keys %$jobs) {
6910 print "$job: Cancelling block job\n";
6911 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6912 $jobs->{$job}->{cancel
} = 1;
6916 my $stats = mon_cmd
($vmid, "query-block-jobs");
6918 my $running_jobs = {};
6919 foreach my $stat (@$stats) {
6920 $running_jobs->{$stat->{device
}} = $stat;
6923 foreach my $job (keys %$jobs) {
6925 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6926 print "$job: Done.\n";
6927 delete $jobs->{$job};
6931 last if scalar(keys %$jobs) == 0;
6938 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6939 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6944 print "create linked clone of drive $drivename ($drive->{file})\n";
6945 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6946 push @$newvollist, $newvolid;
6949 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6950 $storeid = $storage if $storage;
6952 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6954 print "create full clone of drive $drivename ($drive->{file})\n";
6957 if (drive_is_cloudinit
($drive)) {
6958 $name = "vm-$newvmid-cloudinit";
6959 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6960 if ($scfg->{path
}) {
6961 $name .= ".$dst_format";
6964 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
6965 } elsif ($drivename eq 'efidisk0') {
6966 $size = get_efivars_size
($conf);
6968 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
6970 $newvolid = PVE
::Storage
::vdisk_alloc
(
6971 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
6973 push @$newvollist, $newvolid;
6975 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
6977 if (drive_is_cloudinit
($drive)) {
6978 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
6979 # if this is the case, we have to complete any block-jobs still there from
6980 # previous drive-mirrors
6981 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
6982 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
6987 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6988 if (!$running || $snapname) {
6989 # TODO: handle bwlimits
6990 if ($drivename eq 'efidisk0') {
6991 # the relevant data on the efidisk may be smaller than the source
6992 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6993 # that is given by the OVMF_VARS.fd
6994 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
6995 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
6997 # better for Ceph if block size is not too small, see bug #3324
7000 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7001 "if=$src_path", "of=$dst_path"]);
7003 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
7007 my $kvmver = get_running_qemu_version
($vmid);
7008 if (!min_version
($kvmver, 2, 7)) {
7009 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7010 if $drive->{iothread
};
7013 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7014 $completion, $qga, $bwlimit);
7019 my ($size) = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
7022 $disk->{format
} = undef;
7023 $disk->{file
} = $newvolid;
7024 $disk->{size
} = $size if defined($size);
7029 sub get_running_qemu_version
{
7031 my $res = mon_cmd
($vmid, "query-version");
7032 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7035 sub qemu_use_old_bios_files
{
7036 my ($machine_type) = @_;
7038 return if !$machine_type;
7040 my $use_old_bios_files = undef;
7042 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7044 $use_old_bios_files = 1;
7046 my $version = extract_version
($machine_type, kvm_user_version
());
7047 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7048 # load new efi bios files on migration. So this hack is required to allow
7049 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7050 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7051 $use_old_bios_files = !min_version
($version, 2, 4);
7054 return ($use_old_bios_files, $machine_type);
7057 sub get_efivars_size
{
7059 my $arch = get_vm_arch
($conf);
7060 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7061 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
7062 return -s
$ovmf_vars;
7065 sub update_efidisk_size
{
7068 return if !defined($conf->{efidisk0
});
7070 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
7071 $disk->{size
} = get_efivars_size
($conf);
7072 $conf->{efidisk0
} = print_drive
($disk);
7077 sub create_efidisk
($$$$$) {
7078 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7080 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7081 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7083 my $vars_size_b = -s
$ovmf_vars;
7084 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7085 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7086 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7088 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7089 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7091 return ($volid, $size/1024);
7094 sub vm_iothreads_list
{
7097 my $res = mon_cmd
($vmid, 'query-iothreads');
7100 foreach my $iothread (@$res) {
7101 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7108 my ($conf, $drive) = @_;
7112 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7114 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7120 my $controller = int($drive->{index} / $maxdev);
7121 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
7125 return ($maxdev, $controller, $controller_prefix);
7128 sub windows_version
{
7131 return 0 if !$ostype;
7135 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7137 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7139 } elsif ($ostype =~ m/^win(\d+)$/) {
7146 sub resolve_dst_disk_format
{
7147 my ($storecfg, $storeid, $src_volname, $format) = @_;
7148 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7151 # if no target format is specified, use the source disk format as hint
7153 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7154 $format = qemu_img_format
($scfg, $src_volname);
7160 # test if requested format is supported - else use default
7161 my $supported = grep { $_ eq $format } @$validFormats;
7162 $format = $defFormat if !$supported;
7166 # NOTE: if this logic changes, please update docs & possibly gui logic
7167 sub find_vmstate_storage
{
7168 my ($conf, $storecfg) = @_;
7170 # first, return storage from conf if set
7171 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7173 my ($target, $shared, $local);
7175 foreach_storage_used_by_vm
($conf, sub {
7177 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7178 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7179 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7182 # second, use shared storage where VM has at least one disk
7183 # third, use local storage where VM has at least one disk
7184 # fall back to local storage
7185 $target = $shared // $local // 'local';
7191 my ($uuid, $uuid_str);
7192 UUID
::generate
($uuid);
7193 UUID
::unparse
($uuid, $uuid_str);
7197 sub generate_smbios1_uuid
{
7198 return "uuid=".generate_uuid
();
7204 mon_cmd
($vmid, 'nbd-server-stop');
7207 sub create_reboot_request
{
7209 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7210 or die "failed to create reboot trigger file: $!\n";
7214 sub clear_reboot_request
{
7216 my $path = "/run/qemu-server/$vmid.reboot";
7219 $res = unlink($path);
7220 die "could not remove reboot request for $vmid: $!"
7221 if !$res && $! != POSIX
::ENOENT
;
7226 sub bootorder_from_legacy
{
7227 my ($conf, $bootcfg) = @_;
7229 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
7230 my $bootindex_hash = {};
7232 foreach my $o (split(//, $boot)) {
7233 $bootindex_hash->{$o} = $i*100;
7239 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7240 my ($ds, $drive) = @_;
7242 if (drive_is_cdrom
($drive, 1)) {
7243 if ($bootindex_hash->{d
}) {
7244 $bootorder->{$ds} = $bootindex_hash->{d
};
7245 $bootindex_hash->{d
} += 1;
7247 } elsif ($bootindex_hash->{c
}) {
7248 $bootorder->{$ds} = $bootindex_hash->{c
}
7249 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
7250 $bootindex_hash->{c
} += 1;
7254 if ($bootindex_hash->{n
}) {
7255 for (my $i = 0; $i < $MAX_NETS; $i++) {
7256 my $netname = "net$i";
7257 next if !$conf->{$netname};
7258 $bootorder->{$netname} = $bootindex_hash->{n
};
7259 $bootindex_hash->{n
} += 1;
7266 # Generate default device list for 'boot: order=' property. Matches legacy
7267 # default boot order, but with explicit device names. This is important, since
7268 # the fallback for when neither 'order' nor the old format is specified relies
7269 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7270 sub get_default_bootdevices
{
7276 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
7277 push @ret, $first if $first;
7280 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
7281 push @ret, $first if $first;
7284 for (my $i = 0; $i < $MAX_NETS; $i++) {
7285 my $netname = "net$i";
7286 next if !$conf->{$netname};
7287 push @ret, $netname;
7294 sub device_bootorder
{
7297 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
7299 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
7302 if (!defined($boot) || $boot->{legacy
}) {
7303 $bootorder = bootorder_from_legacy
($conf, $boot);
7304 } elsif ($boot->{order
}) {
7305 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7306 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
7307 $bootorder->{$dev} = $i++;
7314 # bash completion helper
7316 sub complete_backup_archives
{
7317 my ($cmdname, $pname, $cvalue) = @_;
7319 my $cfg = PVE
::Storage
::config
();
7323 if ($cvalue =~ m/^([^:]+):/) {
7327 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7330 foreach my $id (keys %$data) {
7331 foreach my $item (@{$data->{$id}}) {
7332 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7333 push @$res, $item->{volid
} if defined($item->{volid
});
7340 my $complete_vmid_full = sub {
7343 my $idlist = vmstatus
();
7347 foreach my $id (keys %$idlist) {
7348 my $d = $idlist->{$id};
7349 if (defined($running)) {
7350 next if $d->{template
};
7351 next if $running && $d->{status
} ne 'running';
7352 next if !$running && $d->{status
} eq 'running';
7361 return &$complete_vmid_full();
7364 sub complete_vmid_stopped
{
7365 return &$complete_vmid_full(0);
7368 sub complete_vmid_running
{
7369 return &$complete_vmid_full(1);
7372 sub complete_storage
{
7374 my $cfg = PVE
::Storage
::config
();
7375 my $ids = $cfg->{ids
};
7378 foreach my $sid (keys %$ids) {
7379 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7380 next if !$ids->{$sid}->{content
}->{images
};
7387 sub complete_migration_storage
{
7388 my ($cmd, $param, $current_value, $all_args) = @_;
7390 my $targetnode = @$all_args[1];
7392 my $cfg = PVE
::Storage
::config
();
7393 my $ids = $cfg->{ids
};
7396 foreach my $sid (keys %$ids) {
7397 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7398 next if !$ids->{$sid}->{content
}->{images
};
7407 my $qmpstatus = eval {
7408 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
7409 mon_cmd
($vmid, "query-status");
7412 return $qmpstatus && $qmpstatus->{status
} eq "paused";