1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
31 use PVE
::DataCenterConfig
;
32 use PVE
::Exception
qw(raise raise_param_exc);
33 use PVE
::Format
qw(render_duration render_bytes);
34 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
36 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
39 use PVE
::RPCEnvironment
;
43 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
47 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
48 use PVE
::QemuServer
::Cloudinit
;
49 use PVE
::QemuServer
::CGroup
;
50 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
51 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
52 use PVE
::QemuServer
::Machine
;
53 use PVE
::QemuServer
::Memory
;
54 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
55 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
56 use PVE
::QemuServer
::USB
qw(parse_usb_device);
60 require PVE
::Network
::SDN
::Zones
;
64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
67 "$EDK2_FW_BASE/OVMF_CODE.fd",
68 "$EDK2_FW_BASE/OVMF_VARS.fd"
71 "$EDK2_FW_BASE/AAVMF_CODE.fd",
72 "$EDK2_FW_BASE/AAVMF_VARS.fd"
76 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
78 # Note about locking: we use flock on the config file protect
79 # against concurent actions.
80 # Aditionaly, we have a 'lock' setting in the config file. This
81 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
82 # allowed when such lock is set. But you can ignore this kind of
83 # lock with the --skiplock flag.
85 cfs_register_file
('/qemu-server/',
89 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
90 description
=> "Some command save/restore state from this location.",
96 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
97 description
=> "Specifies the Qemu machine type.",
99 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
106 my ($map, $source) = @_;
108 return $source if !defined($map);
110 return $map->{entries
}->{$source}
111 if $map->{entries
} && defined($map->{entries
}->{$source});
113 return $map->{default} if $map->{default};
115 # identity (fallback)
119 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
120 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
122 format
=> 'storagepair-list',
126 #no warnings 'redefine';
130 $nodename_cache //= PVE
::INotify
::nodename
();
131 return $nodename_cache;
138 enum
=> [qw(i6300esb ib700)],
139 description
=> "Watchdog type to emulate.",
140 default => 'i6300esb',
145 enum
=> [qw(reset shutdown poweroff pause debug none)],
146 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
150 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
154 description
=> "Enable/disable Qemu GuestAgent.",
159 fstrim_cloned_disks
=> {
160 description
=> "Run fstrim after moving a disk or migrating the VM.",
166 description
=> "Select the agent type",
170 enum
=> [qw(virtio isa)],
176 description
=> "Select the VGA type.",
181 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
184 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
196 description
=> "The size of the file in MB.",
200 pattern
=> '[a-zA-Z0-9\-]+',
202 format_description
=> 'string',
203 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
210 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
211 description
=> "Configure an audio device."
215 enum
=> ['spice', 'none'],
218 description
=> "Driver backend for the audio device."
222 my $spice_enhancements_fmt = {
227 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
231 enum
=> ['off', 'all', 'filter'],
234 description
=> "Enable video streaming. Uses compression for detected video streams."
241 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
243 description
=> "The file on the host to gather entropy from. In most"
244 . " cases /dev/urandom should be preferred over /dev/random"
245 . " to avoid entropy-starvation issues on the host. Using"
246 . " urandom does *not* decrease security in any meaningful"
247 . " way, as it's still seeded from real entropy, and the"
248 . " bytes provided will most likely be mixed with real"
249 . " entropy on the guest as well. /dev/hwrng can be used"
250 . " to pass through a hardware RNG from the host.",
254 description
=> "Maximum bytes of entropy injected into the guest every"
255 . " 'period' milliseconds. Prefer a lower value when using"
256 . " /dev/random as source. Use 0 to disable limiting"
257 . " (potentially dangerous!).",
260 # default is 1 KiB/s, provides enough entropy to the guest to avoid
261 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
262 # of overwhelming the host, provided we're reading from /dev/urandom
267 description
=> "Every 'period' milliseconds the entropy-injection quota"
268 . " is reset, allowing the guest to retrieve another"
269 . " 'max_bytes' of entropy.",
279 description
=> "Specifies whether a VM will be started during system bootup.",
285 description
=> "Automatic restart after crash (currently ignored).",
290 type
=> 'string', format
=> 'pve-hotplug-features',
291 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
292 default => 'network,disk,usb',
297 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
303 description
=> "Lock/unlock the VM.",
304 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
309 description
=> "Limit of CPU usage.",
310 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
318 description
=> "CPU weight for a VM.",
319 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
327 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
334 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
340 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
348 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
349 "It should not be necessary to set it.",
350 enum
=> PVE
::Tools
::kvmkeymaplist
(),
355 type
=> 'string', format
=> 'dns-name',
356 description
=> "Set a name for the VM. Only used on the configuration web interface.",
361 description
=> "SCSI controller model",
362 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
368 description
=> "Description for the VM. Shown in the web-interface VM's summary."
369 ." This is saved as comment inside the configuration file.",
370 maxLength
=> 1024 * 8,
375 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
376 description
=> "Specify guest operating system.",
377 verbose_description
=> <<EODESC,
378 Specify guest operating system. This is used to enable special
379 optimization/features for specific operating systems:
382 other;; unspecified OS
383 wxp;; Microsoft Windows XP
384 w2k;; Microsoft Windows 2000
385 w2k3;; Microsoft Windows 2003
386 w2k8;; Microsoft Windows 2008
387 wvista;; Microsoft Windows Vista
388 win7;; Microsoft Windows 7
389 win8;; Microsoft Windows 8/2012/2012r2
390 win10;; Microsoft Windows 10/2016/2019
391 l24;; Linux 2.4 Kernel
392 l26;; Linux 2.6 - 5.X Kernel
393 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
398 type
=> 'string', format
=> 'pve-qm-boot',
399 description
=> "Specify guest boot order. Use with 'order=', usage with"
400 . " no key or 'legacy=' is deprecated.",
404 type
=> 'string', format
=> 'pve-qm-bootdisk',
405 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
406 pattern
=> '(ide|sata|scsi|virtio)\d+',
411 description
=> "The number of CPUs. Please use option -sockets instead.",
418 description
=> "The number of CPU sockets.",
425 description
=> "The number of cores per socket.",
432 description
=> "Enable/disable NUMA.",
438 description
=> "Enable/disable hugepages memory.",
439 enum
=> [qw(any 2 1024)],
445 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
446 ." after VM shutdown and can be used for subsequent starts.",
451 description
=> "Number of hotplugged vcpus.",
458 description
=> "Enable/disable ACPI.",
463 description
=> "Enable/disable Qemu GuestAgent and its properties.",
465 format
=> $agent_fmt,
470 description
=> "Enable/disable KVM hardware virtualization.",
476 description
=> "Enable/disable time drift fix.",
482 description
=> "Set the real time clock to local time. This is enabled by default if ostype"
483 ." indicates a Microsoft OS.",
488 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
492 type
=> 'string', format
=> $vga_fmt,
493 description
=> "Configure the VGA hardware.",
494 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
495 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
496 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
497 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
498 ." display server. For win* OS you can select how many independent displays you want,"
499 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
500 ." using a serial device as terminal.",
504 type
=> 'string', format
=> 'pve-qm-watchdog',
505 description
=> "Create a virtual hardware watchdog device.",
506 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
507 ." action), the watchdog must be periodically polled by an agent inside the guest or"
508 ." else the watchdog will reset the guest (or execute the respective action specified)",
513 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
514 description
=> "Set the initial date of the real time clock. Valid format for date are:"
515 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
516 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
519 startup
=> get_standard_option
('pve-startup-order'),
523 description
=> "Enable/disable Template.",
529 description
=> "Arbitrary arguments passed to kvm.",
530 verbose_description
=> <<EODESCR,
531 Arbitrary arguments passed to kvm, for example:
533 args: -no-reboot -no-hpet
535 NOTE: this option is for experts only.
542 description
=> "Enable/disable the USB tablet device.",
543 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
544 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
545 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
546 ." may consider disabling this to save some context switches. This is turned off by"
547 ." default if you use spice (`qm set <vmid> --vga qxl`).",
552 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
556 migrate_downtime
=> {
559 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
565 type
=> 'string', format
=> 'pve-qm-ide',
566 typetext
=> '<volume>',
567 description
=> "This is an alias for option -ide2",
571 description
=> "Emulated CPU type.",
573 format
=> 'pve-vm-cpu-conf',
575 parent
=> get_standard_option
('pve-snapshot-name', {
577 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
581 description
=> "Timestamp for snapshots.",
587 type
=> 'string', format
=> 'pve-volume-id',
588 description
=> "Reference to a volume which stores the VM state. This is used internally"
591 vmstatestorage
=> get_standard_option
('pve-storage-id', {
592 description
=> "Default storage for VM state volumes/files.",
595 runningmachine
=> get_standard_option
('pve-qemu-machine', {
596 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
600 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
601 ." internally for snapshots.",
604 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
605 format_description
=> 'QEMU -cpu parameter'
607 machine
=> get_standard_option
('pve-qemu-machine'),
609 description
=> "Virtual processor architecture. Defaults to the host.",
612 enum
=> [qw(x86_64 aarch64)],
615 description
=> "Specify SMBIOS type 1 fields.",
616 type
=> 'string', format
=> 'pve-qm-smbios1',
623 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
624 ." remove disk operations.",
630 enum
=> [ qw(seabios ovmf) ],
631 description
=> "Select BIOS implementation.",
632 default => 'seabios',
636 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
637 format_description
=> 'UUID',
638 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
639 ." to disable explicitly.",
640 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
641 ." value identifier to the guest OS. This allows to notify the guest operating system"
642 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
643 ." execution or creation from a template). The guest operating system notices the"
644 ." change, and is then able to react as appropriate by marking its copies of"
645 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
646 ."Note that auto-creation only works when done through API/CLI create or update methods"
647 .", but not when manually editing the config file.",
648 default => "1 (autogenerated)",
653 format
=> 'pve-volume-id',
655 description
=> "Script that will be executed during various steps in the vms lifetime.",
659 format
=> $ivshmem_fmt,
660 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
666 format
=> $audio_fmt,
667 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
670 spice_enhancements
=> {
672 format
=> $spice_enhancements_fmt,
673 description
=> "Configure additional enhancements for SPICE.",
677 type
=> 'string', format
=> 'pve-tag-list',
678 description
=> 'Tags of the VM. This is only meta information.',
684 description
=> "Configure a VirtIO-based Random Number Generator.",
693 description
=> 'Specify a custom file containing all meta data passed to the VM via"
694 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
695 format
=> 'pve-volume-id',
696 format_description
=> 'volume',
701 description
=> 'Specify a custom file containing all network data passed to the VM via'
703 format
=> 'pve-volume-id',
704 format_description
=> 'volume',
709 description
=> 'Specify a custom file containing all user data passed to the VM via'
711 format
=> 'pve-volume-id',
712 format_description
=> 'volume',
715 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
717 my $confdesc_cloudinit = {
721 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
722 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
723 .' and `configdrive2` for windows.',
724 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
729 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
730 ." image's configured default user.",
735 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
736 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
737 .' support hashed passwords.',
742 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
744 format
=> 'pve-qm-cicustom',
749 description
=> "cloud-init: Sets DNS search domains for a container. Create will'
750 .' automatically use the setting from the host if neither searchdomain nor nameserver'
755 type
=> 'string', format
=> 'address-list',
756 description
=> "cloud-init: Sets DNS server IP address for a container. Create will'
757 .' automatically use the setting from the host if neither searchdomain nor nameserver'
763 format
=> 'urlencoded',
764 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
768 # what about other qemu settings ?
770 #machine => 'string',
783 ##soundhw => 'string',
785 while (my ($k, $v) = each %$confdesc) {
786 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
789 my $MAX_USB_DEVICES = 5;
791 my $MAX_SERIAL_PORTS = 4;
792 my $MAX_PARALLEL_PORTS = 3;
798 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
799 description
=> "CPUs accessing this NUMA node.",
800 format_description
=> "id[-id];...",
804 description
=> "Amount of memory this NUMA node provides.",
809 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
810 description
=> "Host NUMA nodes to use.",
811 format_description
=> "id[-id];...",
816 enum
=> [qw(preferred bind interleave)],
817 description
=> "NUMA allocation policy.",
821 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
824 type
=> 'string', format
=> $numa_fmt,
825 description
=> "NUMA topology.",
827 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
829 for (my $i = 0; $i < $MAX_NUMA; $i++) {
830 $confdesc->{"numa$i"} = $numadesc;
833 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
834 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
835 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
836 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
838 my $net_fmt_bridge_descr = <<__EOD__;
839 Bridge to attach the network device to. The Proxmox VE standard bridge
842 If you do not specify a bridge, we create a kvm user (NATed) network
843 device, which provides DHCP and DNS services. The following addresses
850 The DHCP server assign addresses to the guest starting from 10.0.2.15.
854 macaddr
=> get_standard_option
('mac-addr', {
855 description
=> "MAC address. That address must be unique withing your network. This is"
856 ." automatically generated if not specified.",
860 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
861 ." very low CPU overhead. If your guest does not support this driver, it is usually"
862 ." best to use 'e1000'.",
863 enum
=> $nic_model_list,
866 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
869 description
=> $net_fmt_bridge_descr,
870 format_description
=> 'bridge',
871 pattern
=> '[-_.\w\d]+',
876 minimum
=> 0, maximum
=> 16,
877 description
=> 'Number of packet queues to be used on the device.',
883 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
888 minimum
=> 1, maximum
=> 4094,
889 description
=> 'VLAN tag to apply to packets on this interface.',
894 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
895 description
=> 'VLAN trunks to pass through this interface.',
896 format_description
=> 'vlanid[;vlanid...]',
901 description
=> 'Whether this interface should be protected by the firewall.',
906 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
911 minimum
=> 1, maximum
=> 65520,
912 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
919 type
=> 'string', format
=> $net_fmt,
920 description
=> "Specify network devices.",
923 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
928 format
=> 'pve-ipv4-config',
929 format_description
=> 'IPv4Format/CIDR',
930 description
=> 'IPv4 address in CIDR format.',
937 format_description
=> 'GatewayIPv4',
938 description
=> 'Default gateway for IPv4 traffic.',
944 format
=> 'pve-ipv6-config',
945 format_description
=> 'IPv6Format/CIDR',
946 description
=> 'IPv6 address in CIDR format.',
953 format_description
=> 'GatewayIPv6',
954 description
=> 'Default gateway for IPv6 traffic.',
959 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
962 type
=> 'string', format
=> 'pve-qm-ipconfig',
963 description
=> <<'EODESCR',
964 cloud-init: Specify IP addresses and gateways for the corresponding interface.
966 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
968 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
969 gateway should be provided.
970 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
971 cloud-init 19.4 or newer.
973 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
977 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
979 for (my $i = 0; $i < $MAX_NETS; $i++) {
980 $confdesc->{"net$i"} = $netdesc;
981 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
984 foreach my $key (keys %$confdesc_cloudinit) {
985 $confdesc->{$key} = $confdesc_cloudinit->{$key};
988 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
989 sub verify_volume_id_or_qm_path
{
990 my ($volid, $noerr) = @_;
992 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
996 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
997 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1008 type
=> 'string', format
=> 'pve-qm-usb-device',
1009 format_description
=> 'HOSTUSBDEVICE|spice',
1010 description
=> <<EODESCR,
1011 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1013 'bus-port(.port)*' (decimal numbers) or
1014 'vendor_id:product_id' (hexadeciaml numbers) or
1017 You can use the 'lsusb -t' command to list existing usb devices.
1019 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1020 machines - use with special care.
1022 The value 'spice' can be used to add a usb redirection devices for spice.
1028 description
=> "Specifies whether if given host option is a USB3 device or port.",
1035 type
=> 'string', format
=> $usb_fmt,
1036 description
=> "Configure an USB device (n is 0 to 4).",
1038 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1043 pattern
=> '(/dev/.+|socket)',
1044 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1045 verbose_description
=> <<EODESCR,
1046 Create a serial device inside the VM (n is 0 to 3), and pass through a
1047 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1048 host side (use 'qm terminal' to open a terminal connection).
1050 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1051 use with special care.
1053 CAUTION: Experimental! User reported problems with this option.
1060 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1061 description
=> "Map host parallel devices (n is 0 to 2).",
1062 verbose_description
=> <<EODESCR,
1063 Map host parallel devices (n is 0 to 2).
1065 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1066 machines - use with special care.
1068 CAUTION: Experimental! User reported problems with this option.
1072 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1073 $confdesc->{"parallel$i"} = $paralleldesc;
1076 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1077 $confdesc->{"serial$i"} = $serialdesc;
1080 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1081 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1084 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1085 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1088 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1089 $confdesc->{"usb$i"} = $usbdesc;
1097 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1098 . " Deprecated, use 'order=' instead.",
1099 pattern
=> '[acdn]{1,4}',
1100 format_description
=> "[acdn]{1,4}",
1102 # note: this is also the fallback if boot: is not given at all
1108 format
=> 'pve-qm-bootdev-list',
1109 format_description
=> "device[;device...]",
1110 description
=> <<EODESC,
1111 The guest will attempt to boot from devices in the order they appear here.
1113 Disks, optical drives and passed-through storage USB devices will be directly
1114 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1115 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1117 Note that only devices in this list will be marked as bootable and thus loaded
1118 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1119 (e.g. software-raid), you need to specify all of them here.
1121 Overrides the deprecated 'legacy=[acdn]*' value when given.
1125 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1127 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1128 sub verify_bootdev
{
1129 my ($dev, $noerr) = @_;
1131 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && $dev !~ m/^efidisk/;
1135 return 0 if $dev !~ m/^$base\d+$/;
1136 return 0 if !$confdesc->{$dev};
1140 return $dev if $check->("net");
1141 return $dev if $check->("usb");
1142 return $dev if $check->("hostpci");
1145 die "invalid boot device '$dev'\n";
1148 sub print_bootorder
{
1150 return "" if !@$devs;
1151 my $data = { order
=> join(';', @$devs) };
1152 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1155 my $kvm_api_version = 0;
1158 return $kvm_api_version if $kvm_api_version;
1160 open my $fh, '<', '/dev/kvm' or return;
1162 # 0xae00 => KVM_GET_API_VERSION
1163 $kvm_api_version = ioctl($fh, 0xae00, 0);
1166 return $kvm_api_version;
1169 my $kvm_user_version = {};
1172 sub kvm_user_version
{
1175 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1176 my $st = stat($binary);
1178 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1179 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1180 $cachedmtime == $st->mtime;
1182 $kvm_user_version->{$binary} = 'unknown';
1183 $kvm_mtime->{$binary} = $st->mtime;
1187 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188 $kvm_user_version->{$binary} = $2;
1192 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1195 return $kvm_user_version->{$binary};
1198 my sub extract_version
{
1199 my ($machine_type, $version) = @_;
1200 $version = kvm_user_version
() if !defined($version);
1201 PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1204 sub kernel_has_vhost_net
{
1205 return -c
'/dev/vhost-net';
1210 return defined($confdesc->{$key});
1214 sub get_cdrom_path
{
1216 return $cdrom_path if $cdrom_path;
1218 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1219 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1220 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1224 my ($storecfg, $vmid, $cdrom) = @_;
1226 if ($cdrom eq 'cdrom') {
1227 return get_cdrom_path
();
1228 } elsif ($cdrom eq 'none') {
1230 } elsif ($cdrom =~ m
|^/|) {
1233 return PVE
::Storage
::path
($storecfg, $cdrom);
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id
{
1239 my ($vmid, $file, $media) = @_;
1241 if (!($file eq 'none' || $file eq 'cdrom' ||
1242 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1244 return if $file =~ m
|/|;
1246 if ($media && $media eq 'cdrom') {
1247 $file = "local:iso/$file";
1249 $file = "local:$vmid/$file";
1256 sub verify_media_type
{
1257 my ($opt, $vtype, $media) = @_;
1262 if ($media eq 'disk') {
1264 } elsif ($media eq 'cdrom') {
1267 die "internal error";
1270 return if ($vtype eq $etype);
1272 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1275 sub cleanup_drive_path
{
1276 my ($opt, $storecfg, $drive) = @_;
1278 # try to convert filesystem paths to volume IDs
1280 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1281 ($drive->{file
} !~ m
|^/dev/.+|) &&
1282 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1283 ($drive->{file
} !~ m/^\d+$/)) {
1284 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1285 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1287 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1288 verify_media_type
($opt, $vtype, $drive->{media
});
1289 $drive->{file
} = $volid;
1292 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1295 sub parse_hotplug_features
{
1300 return $res if $data eq '0';
1302 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1304 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1305 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1308 die "invalid hotplug feature '$feature'\n";
1314 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1315 sub pve_verify_hotplug_features
{
1316 my ($value, $noerr) = @_;
1318 return $value if parse_hotplug_features
($value);
1322 die "unable to parse hotplug option\n";
1326 my($fh, $noerr) = @_;
1329 my $SG_GET_VERSION_NUM = 0x2282;
1331 my $versionbuf = "\x00" x
8;
1332 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1334 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1337 my $version = unpack("I", $versionbuf);
1338 if ($version < 30000) {
1339 die "scsi generic interface too old\n" if !$noerr;
1343 my $buf = "\x00" x
36;
1344 my $sensebuf = "\x00" x
8;
1345 my $cmd = pack("C x3 C x1", 0x12, 36);
1347 # see /usr/include/scsi/sg.h
1348 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1350 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351 length($sensebuf), 0, length($buf), $buf,
1352 $cmd, $sensebuf, 6000);
1354 $ret = ioctl($fh, $SG_IO, $packet);
1356 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1360 my @res = unpack($sg_io_hdr_t, $packet);
1361 if ($res[17] || $res[18]) {
1362 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1367 (my $byte0, my $byte1, $res->{vendor
},
1368 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1370 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1371 $res->{type
} = $byte0 & 31;
1379 my $fh = IO
::File-
>new("+<$path") || return;
1380 my $res = scsi_inquiry
($fh, 1);
1386 sub print_tabletdevice_full
{
1387 my ($conf, $arch) = @_;
1389 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1391 # we use uhci for old VMs because tablet driver was buggy in older qemu
1393 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1399 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1402 sub print_keyboarddevice_full
{
1403 my ($conf, $arch, $machine) = @_;
1405 return if $arch ne 'aarch64';
1407 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1410 my sub get_drive_id
{
1412 return "$drive->{interface}$drive->{index}";
1415 sub print_drivedevice_full
{
1416 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1421 my $drive_id = get_drive_id
($drive);
1422 if ($drive->{interface
} eq 'virtio') {
1423 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1424 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1425 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1426 } elsif ($drive->{interface
} eq 'scsi') {
1428 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1429 my $unit = $drive->{index} % $maxdev;
1430 my $devicetype = 'hd';
1432 if (drive_is_cdrom
($drive)) {
1435 if ($drive->{file
} =~ m
|^/|) {
1436 $path = $drive->{file
};
1437 if (my $info = path_is_scsi
($path)) {
1438 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1439 $devicetype = 'block';
1440 } elsif ($info->{type
} == 1) { # tape
1441 $devicetype = 'generic';
1445 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1448 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1449 my $version = extract_version
($machine_type, kvm_user_version
());
1450 if ($path =~ m/^iscsi\:\/\
// &&
1451 !min_version
($version, 4, 1)) {
1452 $devicetype = 'generic';
1456 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1457 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1459 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1460 .",lun=$drive->{index}";
1462 $device .= ",drive=drive-$drive_id,id=$drive_id";
1464 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1465 $device .= ",rotation_rate=1";
1467 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1469 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1470 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1471 my $controller = int($drive->{index} / $maxdev);
1472 my $unit = $drive->{index} % $maxdev;
1473 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1475 $device = "ide-$devicetype";
1476 if ($drive->{interface
} eq 'ide') {
1477 $device .= ",bus=ide.$controller,unit=$unit";
1479 $device .= ",bus=ahci$controller.$unit";
1481 $device .= ",drive=drive-$drive_id,id=$drive_id";
1483 if ($devicetype eq 'hd') {
1484 if (my $model = $drive->{model
}) {
1485 $model = URI
::Escape
::uri_unescape
($model);
1486 $device .= ",model=$model";
1488 if ($drive->{ssd
}) {
1489 $device .= ",rotation_rate=1";
1492 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1493 } elsif ($drive->{interface
} eq 'usb') {
1495 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1497 die "unsupported interface type";
1500 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1502 if (my $serial = $drive->{serial
}) {
1503 $serial = URI
::Escape
::uri_unescape
($serial);
1504 $device .= ",serial=$serial";
1511 sub get_initiator_name
{
1514 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1515 while (defined(my $line = <$fh>)) {
1516 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1525 sub print_drive_commandline_full
{
1526 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1529 my $volid = $drive->{file
};
1530 my $format = $drive->{format
};
1531 my $drive_id = get_drive_id
($drive);
1533 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1534 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1536 if (drive_is_cdrom
($drive)) {
1537 $path = get_iso_path
($storecfg, $vmid, $volid);
1538 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1541 $path = PVE
::Storage
::path
($storecfg, $volid);
1542 $format //= qemu_img_format
($scfg, $volname);
1549 my $is_rbd = $path =~ m/^rbd:/;
1552 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1553 foreach my $o (@qemu_drive_options) {
1554 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1557 # snapshot only accepts on|off
1558 if (defined($drive->{snapshot
})) {
1559 my $v = $drive->{snapshot
} ?
'on' : 'off';
1560 $opts .= ",snapshot=$v";
1563 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1564 my ($dir, $qmpname) = @$type;
1565 if (my $v = $drive->{"mbps$dir"}) {
1566 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1568 if (my $v = $drive->{"mbps${dir}_max"}) {
1569 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1571 if (my $v = $drive->{"bps${dir}_max_length"}) {
1572 $opts .= ",throttling.bps$qmpname-max-length=$v";
1574 if (my $v = $drive->{"iops${dir}"}) {
1575 $opts .= ",throttling.iops$qmpname=$v";
1577 if (my $v = $drive->{"iops${dir}_max"}) {
1578 $opts .= ",throttling.iops$qmpname-max=$v";
1580 if (my $v = $drive->{"iops${dir}_max_length"}) {
1581 $opts .= ",throttling.iops$qmpname-max-length=$v";
1586 $format = "rbd" if $is_rbd;
1587 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1589 $opts .= ",format=alloc-track,file.driver=$format";
1591 $opts .= ",format=$format";
1594 my $cache_direct = 0;
1596 if (my $cache = $drive->{cache
}) {
1597 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1598 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1599 $opts .= ",cache=none";
1603 # io_uring with cache mode writeback or writethrough on krbd will hang...
1604 my $rbd_no_io_uring = $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1606 if (!$drive->{aio
}) {
1607 if ($io_uring && !$rbd_no_io_uring) {
1608 # io_uring supports all cache modes
1609 $opts .= ",aio=io_uring";
1611 # aio native works only with O_DIRECT
1613 $opts .= ",aio=native";
1615 $opts .= ",aio=threads";
1620 if (!drive_is_cdrom
($drive)) {
1622 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1623 $detectzeroes = 'off';
1624 } elsif ($drive->{discard
}) {
1625 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1627 # This used to be our default with discard not being specified:
1628 $detectzeroes = 'on';
1631 # note: 'detect-zeroes' works per blockdev and we want it to persist
1632 # after the alloc-track is removed, so put it on 'file' directly
1633 my $dz_param = $pbs_name ?
"file.detect-zeroes" : "detect-zeroes";
1634 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1638 $opts .= ",backing=$pbs_name";
1639 $opts .= ",auto-remove=on";
1642 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1643 my $file_param = "file";
1645 # non-rbd drivers require the underlying file to be a seperate block
1646 # node, so add a second .file indirection
1647 $file_param .= ".file" if !$is_rbd;
1648 $file_param .= ".filename";
1650 my $pathinfo = $path ?
"$file_param=$path," : '';
1652 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1655 sub print_pbs_blockdev
{
1656 my ($pbs_conf, $pbs_name) = @_;
1657 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1658 $blockdev .= ",repository=$pbs_conf->{repository}";
1659 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1660 $blockdev .= ",archive=$pbs_conf->{archive}";
1661 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1665 sub print_netdevice_full
{
1666 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1668 my $device = $net->{model
};
1669 if ($net->{model
} eq 'virtio') {
1670 $device = 'virtio-net-pci';
1673 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1674 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1675 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1676 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1677 # and out of each queue plus one config interrupt and control vector queue
1678 my $vectors = $net->{queues
} * 2 + 2;
1679 $tmpstr .= ",vectors=$vectors,mq=on";
1681 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1683 if (my $mtu = $net->{mtu
}) {
1684 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1685 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1688 } elsif ($mtu < 576) {
1689 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1690 } elsif ($mtu > $bridge_mtu) {
1691 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1693 $tmpstr .= ",host_mtu=$mtu";
1695 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1699 if ($use_old_bios_files) {
1701 if ($device eq 'virtio-net-pci') {
1702 $romfile = 'pxe-virtio.rom';
1703 } elsif ($device eq 'e1000') {
1704 $romfile = 'pxe-e1000.rom';
1705 } elsif ($device eq 'ne2k') {
1706 $romfile = 'pxe-ne2k_pci.rom';
1707 } elsif ($device eq 'pcnet') {
1708 $romfile = 'pxe-pcnet.rom';
1709 } elsif ($device eq 'rtl8139') {
1710 $romfile = 'pxe-rtl8139.rom';
1712 $tmpstr .= ",romfile=$romfile" if $romfile;
1718 sub print_netdev_full
{
1719 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1722 if ($netid =~ m/^net(\d+)$/) {
1726 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1728 my $ifname = "tap${vmid}i$i";
1730 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1731 die "interface name '$ifname' is too long (max 15 character)\n"
1732 if length($ifname) >= 16;
1734 my $vhostparam = '';
1735 if (is_native
($arch)) {
1736 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1739 my $vmname = $conf->{name
} || "vm$vmid";
1742 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1744 if ($net->{bridge
}) {
1745 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1746 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1748 $netdev = "type=user,id=$netid,hostname=$vmname";
1751 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1757 'cirrus' => 'cirrus-vga',
1759 'vmware' => 'vmware-svga',
1760 'virtio' => 'virtio-vga',
1763 sub print_vga_device
{
1764 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1766 my $type = $vga_map->{$vga->{type
}};
1767 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1768 $type = 'virtio-gpu';
1770 my $vgamem_mb = $vga->{memory
};
1772 my $max_outputs = '';
1774 $type = $id ?
'qxl' : 'qxl-vga';
1776 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1777 # set max outputs so linux can have up to 4 qxl displays with one device
1778 if (min_version
($machine_version, 4, 1)) {
1779 $max_outputs = ",max_outputs=4";
1784 die "no devicetype for $vga->{type}\n" if !$type;
1788 if ($vga->{type
} eq 'virtio') {
1789 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1790 $memory = ",max_hostmem=$bytes";
1792 # from https://www.spice-space.org/multiple-monitors.html
1793 $memory = ",vgamem_mb=$vga->{memory}";
1794 my $ram = $vgamem_mb * 4;
1795 my $vram = $vgamem_mb * 2;
1796 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1798 $memory = ",vgamem_mb=$vga->{memory}";
1800 } elsif ($qxlnum && $id) {
1801 $memory = ",ram_size=67108864,vram_size=33554432";
1805 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1806 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1809 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1810 my $vgaid = "vga" . ($id // '');
1813 if ($q35 && $vgaid eq 'vga') {
1814 # the first display uses pcie.0 bus on q35 machines
1815 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1817 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1820 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1823 sub parse_number_sets
{
1826 foreach my $part (split(/;/, $set)) {
1827 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1828 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1829 push @$res, [ $1, $2 ];
1831 die "invalid range: $part\n";
1840 my $res = parse_property_string
($numa_fmt, $data);
1841 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1842 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1846 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1850 my $res = eval { parse_property_string
($net_fmt, $data) };
1855 if (!defined($res->{macaddr
})) {
1856 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1857 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1862 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1863 sub parse_ipconfig
{
1866 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1872 if ($res->{gw
} && !$res->{ip
}) {
1873 warn 'gateway specified without specifying an IP address';
1876 if ($res->{gw6
} && !$res->{ip6
}) {
1877 warn 'IPv6 gateway specified without specifying an IPv6 address';
1880 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1881 warn 'gateway specified together with DHCP';
1884 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1886 warn "IPv6 gateway specified together with $res->{ip6} address";
1890 if (!$res->{ip
} && !$res->{ip6
}) {
1891 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1900 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1903 sub add_random_macs
{
1904 my ($settings) = @_;
1906 foreach my $opt (keys %$settings) {
1907 next if $opt !~ m/^net(\d+)$/;
1908 my $net = parse_net
($settings->{$opt});
1910 $settings->{$opt} = print_net
($net);
1914 sub vm_is_volid_owner
{
1915 my ($storecfg, $vmid, $volid) = @_;
1917 if ($volid !~ m
|^/|) {
1919 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1920 if ($owner && ($owner == $vmid)) {
1928 sub vmconfig_register_unused_drive
{
1929 my ($storecfg, $vmid, $conf, $drive) = @_;
1931 if (drive_is_cloudinit
($drive)) {
1932 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1934 } elsif (!drive_is_cdrom
($drive)) {
1935 my $volid = $drive->{file
};
1936 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1937 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1942 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1946 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1947 format_description
=> 'UUID',
1948 description
=> "Set SMBIOS1 UUID.",
1953 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1954 format_description
=> 'Base64 encoded string',
1955 description
=> "Set SMBIOS1 version.",
1960 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1961 format_description
=> 'Base64 encoded string',
1962 description
=> "Set SMBIOS1 serial number.",
1967 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1968 format_description
=> 'Base64 encoded string',
1969 description
=> "Set SMBIOS1 manufacturer.",
1974 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1975 format_description
=> 'Base64 encoded string',
1976 description
=> "Set SMBIOS1 product ID.",
1981 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1982 format_description
=> 'Base64 encoded string',
1983 description
=> "Set SMBIOS1 SKU string.",
1988 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1989 format_description
=> 'Base64 encoded string',
1990 description
=> "Set SMBIOS1 family string.",
1995 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2003 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2010 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2013 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2015 sub parse_watchdog
{
2020 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2025 sub parse_guest_agent
{
2028 return {} if !defined($conf->{agent
});
2030 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2033 # if the agent is disabled ignore the other potentially set properties
2034 return {} if !$res->{enabled
};
2039 my ($conf, $key) = @_;
2040 return undef if !defined($conf->{agent
});
2042 my $agent = parse_guest_agent
($conf);
2043 return $agent->{$key};
2049 return {} if !$value;
2050 my $res = eval { parse_property_string
($vga_fmt, $value) };
2060 my $res = eval { parse_property_string
($rng_fmt, $value) };
2065 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2066 sub verify_usb_device
{
2067 my ($value, $noerr) = @_;
2069 return $value if parse_usb_device
($value);
2073 die "unable to parse usb device\n";
2076 # add JSON properties for create and set function
2077 sub json_config_properties
{
2080 foreach my $opt (keys %$confdesc) {
2081 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2082 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2083 $prop->{$opt} = $confdesc->{$opt};
2089 # return copy of $confdesc_cloudinit to generate documentation
2090 sub cloudinit_config_properties
{
2092 return dclone
($confdesc_cloudinit);
2096 my ($key, $value) = @_;
2098 die "unknown setting '$key'\n" if !$confdesc->{$key};
2100 my $type = $confdesc->{$key}->{type
};
2102 if (!defined($value)) {
2103 die "got undefined value\n";
2106 if ($value =~ m/[\n\r]/) {
2107 die "property contains a line feed\n";
2110 if ($type eq 'boolean') {
2111 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2112 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2113 die "type check ('boolean') failed - got '$value'\n";
2114 } elsif ($type eq 'integer') {
2115 return int($1) if $value =~ m/^(\d+)$/;
2116 die "type check ('integer') failed - got '$value'\n";
2117 } elsif ($type eq 'number') {
2118 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2119 die "type check ('number') failed - got '$value'\n";
2120 } elsif ($type eq 'string') {
2121 if (my $fmt = $confdesc->{$key}->{format
}) {
2122 PVE
::JSONSchema
::check_format
($fmt, $value);
2125 $value =~ s/^\"(.*)\"$/$1/;
2128 die "internal error"
2133 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2135 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2137 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2139 if ($conf->{template
}) {
2140 # check if any base image is still used by a linked clone
2141 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2142 my ($ds, $drive) = @_;
2143 return if drive_is_cdrom
($drive);
2145 my $volid = $drive->{file
};
2146 return if !$volid || $volid =~ m
|^/|;
2148 die "base volume '$volid' is still in use by linked cloned\n"
2149 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2154 my $remove_owned_drive = sub {
2155 my ($ds, $drive) = @_;
2156 return if drive_is_cdrom
($drive, 1);
2158 my $volid = $drive->{file
};
2159 return if !$volid || $volid =~ m
|^/|;
2161 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2162 return if !$path || !$owner || ($owner != $vmid);
2164 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2165 warn "Could not remove disk '$volid', check manually: $@" if $@;
2168 # only remove disks owned by this VM (referenced in the config)
2169 my $include_opts = {
2170 include_unused
=> 1,
2171 extra_keys
=> ['vmstate'],
2173 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2175 for my $snap (values %{$conf->{snapshots
}}) {
2176 next if !defined($snap->{vmstate
});
2177 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2178 next if !defined($drive);
2179 $remove_owned_drive->('vmstate', $drive);
2182 if ($purge_unreferenced) { # also remove unreferenced disk
2183 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2184 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2185 my ($volid, $sid, $volname, $d) = @_;
2186 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2191 if (defined $replacement_conf) {
2192 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2194 PVE
::QemuConfig-
>destroy_config($vmid);
2198 sub parse_vm_config
{
2199 my ($filename, $raw) = @_;
2201 return if !defined($raw);
2204 digest
=> Digest
::SHA
::sha1_hex
($raw),
2209 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2210 || die "got strange filename '$filename'";
2218 my @lines = split(/\n/, $raw);
2219 foreach my $line (@lines) {
2220 next if $line =~ m/^\s*$/;
2222 if ($line =~ m/^\[PENDING\]\s*$/i) {
2223 $section = 'pending';
2224 if (defined($descr)) {
2226 $conf->{description
} = $descr;
2229 $conf = $res->{$section} = {};
2232 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2234 if (defined($descr)) {
2236 $conf->{description
} = $descr;
2239 $conf = $res->{snapshots
}->{$section} = {};
2243 if ($line =~ m/^\#(.*)\s*$/) {
2244 $descr = '' if !defined($descr);
2245 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2249 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2250 $descr = '' if !defined($descr);
2251 $descr .= PVE
::Tools
::decode_text
($2);
2252 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2253 $conf->{snapstate
} = $1;
2254 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2257 $conf->{$key} = $value;
2258 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2260 if ($section eq 'pending') {
2261 $conf->{delete} = $value; # we parse this later
2263 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2265 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2268 eval { $value = check_type
($key, $value); };
2270 warn "vm $vmid - unable to parse value of '$key' - $@";
2272 $key = 'ide2' if $key eq 'cdrom';
2273 my $fmt = $confdesc->{$key}->{format
};
2274 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2275 my $v = parse_drive
($key, $value);
2276 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2277 $v->{file
} = $volid;
2278 $value = print_drive
($v);
2280 warn "vm $vmid - unable to parse value of '$key'\n";
2285 $conf->{$key} = $value;
2288 warn "vm $vmid - unable to parse config: $line\n";
2292 if (defined($descr)) {
2294 $conf->{description
} = $descr;
2296 delete $res->{snapstate
}; # just to be sure
2301 sub write_vm_config
{
2302 my ($filename, $conf) = @_;
2304 delete $conf->{snapstate
}; # just to be sure
2306 if ($conf->{cdrom
}) {
2307 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2308 $conf->{ide2
} = $conf->{cdrom
};
2309 delete $conf->{cdrom
};
2312 # we do not use 'smp' any longer
2313 if ($conf->{sockets
}) {
2314 delete $conf->{smp
};
2315 } elsif ($conf->{smp
}) {
2316 $conf->{sockets
} = $conf->{smp
};
2317 delete $conf->{cores
};
2318 delete $conf->{smp
};
2321 my $used_volids = {};
2323 my $cleanup_config = sub {
2324 my ($cref, $pending, $snapname) = @_;
2326 foreach my $key (keys %$cref) {
2327 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2328 $key eq 'snapstate' || $key eq 'pending';
2329 my $value = $cref->{$key};
2330 if ($key eq 'delete') {
2331 die "propertry 'delete' is only allowed in [PENDING]\n"
2333 # fixme: check syntax?
2336 eval { $value = check_type
($key, $value); };
2337 die "unable to parse value of '$key' - $@" if $@;
2339 $cref->{$key} = $value;
2341 if (!$snapname && is_valid_drivename
($key)) {
2342 my $drive = parse_drive
($key, $value);
2343 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2348 &$cleanup_config($conf);
2350 &$cleanup_config($conf->{pending
}, 1);
2352 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2353 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2354 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2357 # remove 'unusedX' settings if we re-add a volume
2358 foreach my $key (keys %$conf) {
2359 my $value = $conf->{$key};
2360 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2361 delete $conf->{$key};
2365 my $generate_raw_config = sub {
2366 my ($conf, $pending) = @_;
2370 # add description as comment to top of file
2371 if (defined(my $descr = $conf->{description
})) {
2373 foreach my $cl (split(/\n/, $descr)) {
2374 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2377 $raw .= "#\n" if $pending;
2381 foreach my $key (sort keys %$conf) {
2382 next if $key =~ /^(digest|description|pending|snapshots)$/;
2383 $raw .= "$key: $conf->{$key}\n";
2388 my $raw = &$generate_raw_config($conf);
2390 if (scalar(keys %{$conf->{pending
}})){
2391 $raw .= "\n[PENDING]\n";
2392 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2395 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2396 $raw .= "\n[$snapname]\n";
2397 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2407 # we use static defaults from our JSON schema configuration
2408 foreach my $key (keys %$confdesc) {
2409 if (defined(my $default = $confdesc->{$key}->{default})) {
2410 $res->{$key} = $default;
2418 my $vmlist = PVE
::Cluster
::get_vmlist
();
2420 return $res if !$vmlist || !$vmlist->{ids
};
2421 my $ids = $vmlist->{ids
};
2422 my $nodename = nodename
();
2424 foreach my $vmid (keys %$ids) {
2425 my $d = $ids->{$vmid};
2426 next if !$d->{node
} || $d->{node
} ne $nodename;
2427 next if !$d->{type
} || $d->{type
} ne 'qemu';
2428 $res->{$vmid}->{exists} = 1;
2433 # test if VM uses local resources (to prevent migration)
2434 sub check_local_resources
{
2435 my ($conf, $noerr) = @_;
2439 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2440 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2442 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2444 foreach my $k (keys %$conf) {
2445 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2446 # sockets are safe: they will recreated be on the target side post-migrate
2447 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2448 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2451 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2456 # check if used storages are available on all nodes (use by migrate)
2457 sub check_storage_availability
{
2458 my ($storecfg, $conf, $node) = @_;
2460 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2461 my ($ds, $drive) = @_;
2463 my $volid = $drive->{file
};
2466 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2469 # check if storage is available on both nodes
2470 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2471 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2473 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2475 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2476 if !$scfg->{content
}->{$vtype};
2480 # list nodes where all VM images are available (used by has_feature API)
2482 my ($conf, $storecfg) = @_;
2484 my $nodelist = PVE
::Cluster
::get_nodelist
();
2485 my $nodehash = { map { $_ => 1 } @$nodelist };
2486 my $nodename = nodename
();
2488 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2489 my ($ds, $drive) = @_;
2491 my $volid = $drive->{file
};
2494 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2496 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2497 if ($scfg->{disable
}) {
2499 } elsif (my $avail = $scfg->{nodes
}) {
2500 foreach my $node (keys %$nodehash) {
2501 delete $nodehash->{$node} if !$avail->{$node};
2503 } elsif (!$scfg->{shared
}) {
2504 foreach my $node (keys %$nodehash) {
2505 delete $nodehash->{$node} if $node ne $nodename
2514 sub check_local_storage_availability
{
2515 my ($conf, $storecfg) = @_;
2517 my $nodelist = PVE
::Cluster
::get_nodelist
();
2518 my $nodehash = { map { $_ => {} } @$nodelist };
2520 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2521 my ($ds, $drive) = @_;
2523 my $volid = $drive->{file
};
2526 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2528 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2530 if ($scfg->{disable
}) {
2531 foreach my $node (keys %$nodehash) {
2532 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2534 } elsif (my $avail = $scfg->{nodes
}) {
2535 foreach my $node (keys %$nodehash) {
2536 if (!$avail->{$node}) {
2537 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2544 foreach my $node (values %$nodehash) {
2545 if (my $unavail = $node->{unavailable_storages
}) {
2546 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2553 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2555 my ($vmid, $nocheck, $node) = @_;
2557 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2558 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2563 my $vzlist = config_list
();
2565 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2567 while (defined(my $de = $fd->read)) {
2568 next if $de !~ m/^(\d+)\.pid$/;
2570 next if !defined($vzlist->{$vmid});
2571 if (my $pid = check_running
($vmid)) {
2572 $vzlist->{$vmid}->{pid
} = $pid;
2579 our $vmstatus_return_properties = {
2580 vmid
=> get_standard_option
('pve-vmid'),
2582 description
=> "Qemu process status.",
2584 enum
=> ['stopped', 'running'],
2587 description
=> "Maximum memory in bytes.",
2590 renderer
=> 'bytes',
2593 description
=> "Root disk size in bytes.",
2596 renderer
=> 'bytes',
2599 description
=> "VM name.",
2604 description
=> "Qemu QMP agent status.",
2609 description
=> "PID of running qemu process.",
2614 description
=> "Uptime.",
2617 renderer
=> 'duration',
2620 description
=> "Maximum usable CPUs.",
2625 description
=> "The current config lock, if any.",
2630 description
=> "The current configured tags, if any",
2634 'running-machine' => {
2635 description
=> "The currently running machine type (if running).",
2640 description
=> "The currently running QEMU version (if running).",
2646 my $last_proc_pid_stat;
2648 # get VM status information
2649 # This must be fast and should not block ($full == false)
2650 # We only query KVM using QMP if $full == true (this can be slow)
2652 my ($opt_vmid, $full) = @_;
2656 my $storecfg = PVE
::Storage
::config
();
2658 my $list = vzlist
();
2659 my $defaults = load_defaults
();
2661 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2663 my $cpucount = $cpuinfo->{cpus
} || 1;
2665 foreach my $vmid (keys %$list) {
2666 next if $opt_vmid && ($vmid ne $opt_vmid);
2668 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2670 my $d = { vmid
=> int($vmid) };
2671 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2673 # fixme: better status?
2674 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2676 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2677 if (defined($size)) {
2678 $d->{disk
} = 0; # no info available
2679 $d->{maxdisk
} = $size;
2685 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2686 * ($conf->{cores
} || $defaults->{cores
});
2687 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2688 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2690 $d->{name
} = $conf->{name
} || "VM $vmid";
2691 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2692 : $defaults->{memory
}*(1024*1024);
2694 if ($conf->{balloon
}) {
2695 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2696 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2697 : $defaults->{shares
};
2708 $d->{diskwrite
} = 0;
2710 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2712 $d->{serial
} = 1 if conf_has_serial
($conf);
2713 $d->{lock} = $conf->{lock} if $conf->{lock};
2714 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2719 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2720 foreach my $dev (keys %$netdev) {
2721 next if $dev !~ m/^tap([1-9]\d*)i/;
2723 my $d = $res->{$vmid};
2726 $d->{netout
} += $netdev->{$dev}->{receive
};
2727 $d->{netin
} += $netdev->{$dev}->{transmit
};
2730 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2731 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2736 my $ctime = gettimeofday
;
2738 foreach my $vmid (keys %$list) {
2740 my $d = $res->{$vmid};
2741 my $pid = $d->{pid
};
2744 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2745 next if !$pstat; # not running
2747 my $used = $pstat->{utime} + $pstat->{stime
};
2749 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2751 if ($pstat->{vsize
}) {
2752 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2755 my $old = $last_proc_pid_stat->{$pid};
2757 $last_proc_pid_stat->{$pid} = {
2765 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2767 if ($dtime > 1000) {
2768 my $dutime = $used - $old->{used
};
2770 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2771 $last_proc_pid_stat->{$pid} = {
2777 $d->{cpu
} = $old->{cpu
};
2781 return $res if !$full;
2783 my $qmpclient = PVE
::QMPClient-
>new();
2785 my $ballooncb = sub {
2786 my ($vmid, $resp) = @_;
2788 my $info = $resp->{'return'};
2789 return if !$info->{max_mem
};
2791 my $d = $res->{$vmid};
2793 # use memory assigned to VM
2794 $d->{maxmem
} = $info->{max_mem
};
2795 $d->{balloon
} = $info->{actual
};
2797 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2798 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2799 $d->{freemem
} = $info->{free_mem
};
2802 $d->{ballooninfo
} = $info;
2805 my $blockstatscb = sub {
2806 my ($vmid, $resp) = @_;
2807 my $data = $resp->{'return'} || [];
2808 my $totalrdbytes = 0;
2809 my $totalwrbytes = 0;
2811 for my $blockstat (@$data) {
2812 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2813 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2815 $blockstat->{device
} =~ s/drive-//;
2816 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2818 $res->{$vmid}->{diskread
} = $totalrdbytes;
2819 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2822 my $machinecb = sub {
2823 my ($vmid, $resp) = @_;
2824 my $data = $resp->{'return'} || [];
2826 $res->{$vmid}->{'running-machine'} =
2827 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
2830 my $versioncb = sub {
2831 my ($vmid, $resp) = @_;
2832 my $data = $resp->{'return'} // {};
2833 my $version = 'unknown';
2835 if (my $v = $data->{qemu
}) {
2836 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
2839 $res->{$vmid}->{'running-qemu'} = $version;
2842 my $statuscb = sub {
2843 my ($vmid, $resp) = @_;
2845 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2846 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2847 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2848 # this fails if ballon driver is not loaded, so this must be
2849 # the last commnand (following command are aborted if this fails).
2850 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2852 my $status = 'unknown';
2853 if (!defined($status = $resp->{'return'}->{status
})) {
2854 warn "unable to get VM status\n";
2858 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2861 foreach my $vmid (keys %$list) {
2862 next if $opt_vmid && ($vmid ne $opt_vmid);
2863 next if !$res->{$vmid}->{pid
}; # not running
2864 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2867 $qmpclient->queue_execute(undef, 2);
2869 foreach my $vmid (keys %$list) {
2870 next if $opt_vmid && ($vmid ne $opt_vmid);
2871 next if !$res->{$vmid}->{pid
}; #not running
2873 # we can't use the $qmpclient since it might have already aborted on
2874 # 'query-balloon', but this might also fail for older versions...
2875 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
2876 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2879 foreach my $vmid (keys %$list) {
2880 next if $opt_vmid && ($vmid ne $opt_vmid);
2881 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2887 sub conf_has_serial
{
2890 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2891 if ($conf->{"serial$i"}) {
2899 sub conf_has_audio
{
2900 my ($conf, $id) = @_;
2903 my $audio = $conf->{"audio$id"};
2904 return if !defined($audio);
2906 my $audioproperties = parse_property_string
($audio_fmt, $audio);
2907 my $audiodriver = $audioproperties->{driver
} // 'spice';
2910 dev
=> $audioproperties->{device
},
2911 dev_id
=> "audiodev$id",
2912 backend
=> $audiodriver,
2913 backend_id
=> "$audiodriver-backend${id}",
2918 my ($audio, $audiopciaddr, $machine_version) = @_;
2922 my $id = $audio->{dev_id
};
2924 if (min_version
($machine_version, 4, 2)) {
2925 $audiodev = ",audiodev=$audio->{backend_id}";
2928 if ($audio->{dev
} eq 'AC97') {
2929 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2930 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
2931 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2932 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2933 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2935 die "unkown audio device '$audio->{dev}', implement me!";
2938 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2943 sub vga_conf_has_spice
{
2946 my $vgaconf = parse_vga
($vga);
2947 my $vgatype = $vgaconf->{type
};
2948 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2955 return get_host_arch
() eq $arch;
2960 return $conf->{arch
} // get_host_arch
();
2963 my $default_machines = {
2968 sub get_installed_machine_version
{
2969 my ($kvmversion) = @_;
2970 $kvmversion = kvm_user_version
() if !defined($kvmversion);
2971 $kvmversion =~ m/^(\d+\.\d+)/;
2975 sub windows_get_pinned_machine_version
{
2976 my ($machine, $base_version, $kvmversion) = @_;
2978 my $pin_version = $base_version;
2979 if (!defined($base_version) ||
2980 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
2982 $pin_version = get_installed_machine_version
($kvmversion);
2984 if (!$machine || $machine eq 'pc') {
2985 $machine = "pc-i440fx-$pin_version";
2986 } elsif ($machine eq 'q35') {
2987 $machine = "pc-q35-$pin_version";
2988 } elsif ($machine eq 'virt') {
2989 $machine = "virt-$pin_version";
2991 warn "unknown machine type '$machine', not touching that!\n";
2997 sub get_vm_machine
{
2998 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3000 my $machine = $forcemachine || $conf->{machine
};
3002 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3003 $kvmversion //= kvm_user_version
();
3004 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3005 # layout which confuses windows quite a bit and may result in various regressions..
3006 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3007 if (windows_version
($conf->{ostype
})) {
3008 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3011 $machine ||= $default_machines->{$arch};
3012 if ($add_pve_version) {
3013 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3014 $machine .= "+pve$pvever";
3018 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3019 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3020 $machine = $1 if $is_pxe;
3022 # for version-pinned machines that do not include a pve-version (e.g.
3023 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3024 $machine .= '+pve0';
3026 $machine .= '.pxe' if $is_pxe;
3032 sub get_ovmf_files
($) {
3035 my $ovmf = $OVMF->{$arch}
3036 or die "no OVMF images known for architecture '$arch'\n";
3042 aarch64
=> '/usr/bin/qemu-system-aarch64',
3043 x86_64
=> '/usr/bin/qemu-system-x86_64',
3045 sub get_command_for_arch
($) {
3047 return '/usr/bin/kvm' if is_native
($arch);
3049 my $cmd = $Arch2Qemu->{$arch}
3050 or die "don't know how to emulate architecture '$arch'\n";
3054 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3055 # to use in a QEMU command line (-cpu element), first array_intersect the result
3056 # of query_supported_ with query_understood_. This is necessary because:
3058 # a) query_understood_ returns flags the host cannot use and
3059 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3060 # flags, but CPU settings - with most of them being flags. Those settings
3061 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3063 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3064 # expensive. If you need the value returned from this, you can get it much
3065 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3066 # $accel being 'kvm' or 'tcg'.
3068 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3069 # changes, automatically populating pmxcfs.
3071 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3072 # since kvm and tcg machines support different flags
3074 sub query_supported_cpu_flags
{
3077 $arch //= get_host_arch
();
3078 my $default_machine = $default_machines->{$arch};
3082 # FIXME: Once this is merged, the code below should work for ARM as well:
3083 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3084 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3087 my $kvm_supported = defined(kvm_version
());
3088 my $qemu_cmd = get_command_for_arch
($arch);
3090 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3092 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3093 my $query_supported_run_qemu = sub {
3099 '-machine', $default_machine,
3101 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3102 '-mon', 'chardev=qmp,mode=control',
3103 '-pidfile', $pidfile,
3108 push @$cmd, '-accel', 'tcg';
3111 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3112 die "QEMU flag querying VM exited with code " . $rc if $rc;
3115 my $cmd_result = mon_cmd
(
3117 'query-cpu-model-expansion',
3119 model
=> { name
=> 'host' }
3122 my $props = $cmd_result->{model
}->{props
};
3123 foreach my $prop (keys %$props) {
3124 next if $props->{$prop} ne '1';
3125 # QEMU returns some flags multiple times, with '_', '.' or '-'
3126 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3127 # We only keep those with underscores, to match /proc/cpuinfo
3128 $prop =~ s/\.|-/_/g;
3129 $flags->{$prop} = 1;
3134 # force stop with 10 sec timeout and 'nocheck'
3135 # always stop, even if QMP failed
3136 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3140 return [ sort keys %$flags ];
3143 # We need to query QEMU twice, since KVM and TCG have different supported flags
3144 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3145 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3146 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3148 if ($kvm_supported) {
3149 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3150 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3157 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3158 my $understood_cpu_flag_dir = "/usr/share/kvm";
3159 sub query_understood_cpu_flags
{
3160 my $arch = get_host_arch
();
3161 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3163 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3166 my $raw = file_get_contents
($filepath);
3167 $raw =~ s/^\s+|\s+$//g;
3168 my @flags = split(/\s+/, $raw);
3173 sub config_to_command
{
3174 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3178 my $globalFlags = [];
3179 my $machineFlags = [];
3184 my $ostype = $conf->{ostype
};
3185 my $winversion = windows_version
($ostype);
3186 my $kvm = $conf->{kvm
};
3187 my $nodename = nodename
();
3189 my $arch = get_vm_arch
($conf);
3190 my $kvm_binary = get_command_for_arch
($arch);
3191 my $kvmver = kvm_user_version
($kvm_binary);
3193 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3194 $kvmver //= "undefined";
3195 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3198 my $add_pve_version = min_version
($kvmver, 4, 1);
3200 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3201 my $machine_version = extract_version
($machine_type, $kvmver);
3202 $kvm //= 1 if is_native
($arch);
3204 $machine_version =~ m/(\d+)\.(\d+)/;
3205 my ($machine_major, $machine_minor) = ($1, $2);
3207 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3208 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3209 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3210 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3211 ." please upgrade node '$nodename'\n"
3212 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3213 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3214 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3215 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3216 ." node '$nodename'\n";
3219 # if a specific +pve version is required for a feature, use $version_guard
3220 # instead of min_version to allow machines to be run with the minimum
3222 my $required_pve_version = 0;
3223 my $version_guard = sub {
3224 my ($major, $minor, $pve) = @_;
3225 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3226 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3227 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3228 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3232 if ($kvm && !defined kvm_version
()) {
3233 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3234 ." or enable in BIOS.\n";
3237 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3238 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3239 my $use_old_bios_files = undef;
3240 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3242 my $cpuunits = defined($conf->{cpuunits
}) ?
3243 $conf->{cpuunits
} : $defaults->{cpuunits
};
3245 push @$cmd, $kvm_binary;
3247 push @$cmd, '-id', $vmid;
3249 my $vmname = $conf->{name
} || "vm$vmid";
3251 push @$cmd, '-name', $vmname;
3253 push @$cmd, '-no-shutdown';
3257 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3258 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3259 push @$cmd, '-mon', "chardev=qmp,mode=control";
3261 if (min_version
($machine_version, 2, 12)) {
3262 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3263 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3266 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3268 push @$cmd, '-daemonize';
3270 if ($conf->{smbios1
}) {
3271 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3272 if ($smbios_conf->{base64
}) {
3273 # Do not pass base64 flag to qemu
3274 delete $smbios_conf->{base64
};
3275 my $smbios_string = "";
3276 foreach my $key (keys %$smbios_conf) {
3278 if ($key eq "uuid") {
3279 $value = $smbios_conf->{uuid
}
3281 $value = decode_base64
($smbios_conf->{$key});
3283 # qemu accepts any binary data, only commas need escaping by double comma
3285 $smbios_string .= "," . $key . "=" . $value if $value;
3287 push @$cmd, '-smbios', "type=1" . $smbios_string;
3289 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3293 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3294 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3295 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3297 my ($path, $format);
3298 my $read_only_str = '';
3299 if (my $efidisk = $conf->{efidisk0
}) {
3300 my $d = parse_drive
('efidisk0', $efidisk);
3301 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3302 $format = $d->{format
};
3304 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3305 if (!defined($format)) {
3306 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3307 $format = qemu_img_format
($scfg, $volname);
3311 die "efidisk format must be specified\n"
3312 if !defined($format);
3315 $read_only_str = ',readonly=on' if drive_is_read_only
($conf, $d);
3317 warn "no efidisk configured! Using temporary efivars disk.\n";
3318 $path = "/tmp/$vmid-ovmf.fd";
3319 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3325 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3326 $size_str = ",size=" . (-s
$ovmf_vars);
3329 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3331 if ($path =~ m/^rbd:/) {
3332 $cache = ',cache=writeback';
3333 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3336 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3337 push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3342 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3343 if (min_version
($machine_version, 4, 0)) {
3344 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3346 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3350 if ($conf->{vmgenid
}) {
3351 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3354 # add usb controllers
3355 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3356 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3357 push @$devices, @usbcontrollers if @usbcontrollers;
3358 my $vga = parse_vga
($conf->{vga
});
3360 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3361 $vga->{type
} = 'qxl' if $qxlnum;
3363 if (!$vga->{type
}) {
3364 if ($arch eq 'aarch64') {
3365 $vga->{type
} = 'virtio';
3366 } elsif (min_version
($machine_version, 2, 9)) {
3367 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3369 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3373 # enable absolute mouse coordinates (needed by vnc)
3375 if (defined($conf->{tablet
})) {
3376 $tablet = $conf->{tablet
};
3378 $tablet = $defaults->{tablet
};
3379 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3380 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3384 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3385 my $kbd = print_keyboarddevice_full
($conf, $arch);
3386 push @$devices, '-device', $kbd if defined($kbd);
3389 my $bootorder = device_bootorder
($conf);
3391 # host pci device passthrough
3392 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3393 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3396 my $usb_dev_features = {};
3397 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3399 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3400 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3401 push @$devices, @usbdevices if @usbdevices;
3404 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3405 if (my $path = $conf->{"serial$i"}) {
3406 if ($path eq 'socket') {
3407 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3408 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3409 # On aarch64, serial0 is the UART device. Qemu only allows
3410 # connecting UART devices via the '-serial' command line, as
3411 # the device has a fixed slot on the hardware...
3412 if ($arch eq 'aarch64' && $i == 0) {
3413 push @$devices, '-serial', "chardev:serial$i";
3415 push @$devices, '-device', "isa-serial,chardev=serial$i";
3418 die "no such serial device\n" if ! -c
$path;
3419 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3420 push @$devices, '-device', "isa-serial,chardev=serial$i";
3426 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3427 if (my $path = $conf->{"parallel$i"}) {
3428 die "no such parallel device\n" if ! -c
$path;
3429 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3430 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3431 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3435 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3436 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3437 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3438 push @$devices, @$audio_devs;
3442 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3443 $sockets = $conf->{sockets
} if $conf->{sockets
};
3445 my $cores = $conf->{cores
} || 1;
3447 my $maxcpus = $sockets * $cores;
3449 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3451 my $allowed_vcpus = $cpuinfo->{cpus
};
3453 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3454 if ($allowed_vcpus < $maxcpus);
3456 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3458 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3459 for (my $i = 2; $i <= $vcpus; $i++) {
3460 my $cpustr = print_cpu_device
($conf,$i);
3461 push @$cmd, '-device', $cpustr;
3466 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3468 push @$cmd, '-nodefaults';
3470 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3472 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3474 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3476 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3477 push @$devices, '-device', print_vga_device
(
3478 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3479 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3480 push @$cmd, '-vnc', "unix:$socket,password=on";
3482 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3483 push @$cmd, '-nographic';
3487 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3488 my $useLocaltime = $conf->{localtime};
3490 if ($winversion >= 5) { # windows
3491 $useLocaltime = 1 if !defined($conf->{localtime});
3493 # use time drift fix when acpi is enabled
3494 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3495 $tdf = 1 if !defined($conf->{tdf
});
3499 if ($winversion >= 6) {
3500 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3501 push @$cmd, '-no-hpet';
3504 push @$rtcFlags, 'driftfix=slew' if $tdf;
3506 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3507 push @$rtcFlags, "base=$conf->{startdate}";
3508 } elsif ($useLocaltime) {
3509 push @$rtcFlags, 'base=localtime';
3513 push @$cmd, '-cpu', $forcecpu;
3515 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3518 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3520 push @$cmd, '-S' if $conf->{freeze
};
3522 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3524 my $guest_agent = parse_guest_agent
($conf);
3526 if ($guest_agent->{enabled
}) {
3527 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3528 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3530 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3531 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3532 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3533 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3534 } elsif ($guest_agent->{type
} eq 'isa') {
3535 push @$devices, '-device', "isa-serial,chardev=qga0";
3539 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3540 if ($rng && $version_guard->(4, 1, 2)) {
3541 check_rng_source
($rng->{source
});
3543 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3544 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3545 my $limiter_str = "";
3547 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3550 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3551 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3552 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3560 for (my $i = 1; $i < $qxlnum; $i++){
3561 push @$devices, '-device', print_vga_device
(
3562 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3565 # assume other OS works like Linux
3566 my ($ram, $vram) = ("134217728", "67108864");
3567 if ($vga->{memory
}) {
3568 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3569 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3571 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3572 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3576 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3578 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3579 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3580 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3582 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3583 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3584 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3586 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3587 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3589 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3590 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3591 if ($spice_enhancement->{foldersharing
}) {
3592 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3593 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3596 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3597 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3598 if $spice_enhancement->{videostreaming
};
3600 push @$devices, '-spice', "$spice_opts";
3603 # enable balloon by default, unless explicitly disabled
3604 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3605 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3606 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3609 if ($conf->{watchdog
}) {
3610 my $wdopts = parse_watchdog
($conf->{watchdog
});
3611 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3612 my $watchdog = $wdopts->{model
} || 'i6300esb';
3613 push @$devices, '-device', "$watchdog$pciaddr";
3614 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3618 my $scsicontroller = {};
3619 my $ahcicontroller = {};
3620 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3622 # Add iscsi initiator name if available
3623 if (my $initiator = get_initiator_name
()) {
3624 push @$devices, '-iscsi', "initiator-name=$initiator";
3627 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3628 my ($ds, $drive) = @_;
3630 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3631 check_volume_storage_type
($storecfg, $drive->{file
});
3632 push @$vollist, $drive->{file
};
3635 # ignore efidisk here, already added in bios/fw handling code above
3636 return if $drive->{interface
} eq 'efidisk';
3638 $use_virtio = 1 if $ds =~ m/^virtio/;
3640 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3642 if ($drive->{interface
} eq 'virtio'){
3643 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3646 if ($drive->{interface
} eq 'scsi') {
3648 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3650 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3651 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3653 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3654 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3657 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3658 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3659 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3660 } elsif ($drive->{iothread
}) {
3661 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3665 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3666 $queues = ",num_queues=$drive->{queues}";
3669 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3670 if !$scsicontroller->{$controller};
3671 $scsicontroller->{$controller}=1;
3674 if ($drive->{interface
} eq 'sata') {
3675 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3676 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3677 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3678 if !$ahcicontroller->{$controller};
3679 $ahcicontroller->{$controller}=1;
3682 my $pbs_conf = $pbs_backing->{$ds};
3683 my $pbs_name = undef;
3685 $pbs_name = "drive-$ds-pbs";
3686 push @$devices, '-blockdev', print_pbs_blockdev
($pbs_conf, $pbs_name);
3689 my $drive_cmd = print_drive_commandline_full
(
3690 $storecfg, $vmid, $drive, $pbs_name, min_version
($kvmver, 6, 0));
3692 # extra protection for templates, but SATA and IDE don't support it..
3693 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
3695 push @$devices, '-drive',$drive_cmd;
3696 push @$devices, '-device', print_drivedevice_full
(
3697 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3700 for (my $i = 0; $i < $MAX_NETS; $i++) {
3701 my $netname = "net$i";
3703 next if !$conf->{$netname};
3704 my $d = parse_net
($conf->{$netname});
3707 $use_virtio = 1 if $d->{model
} eq 'virtio';
3709 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
3711 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
3712 push @$devices, '-netdev', $netdevfull;
3714 my $netdevicefull = print_netdevice_full
(
3715 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3717 push @$devices, '-device', $netdevicefull;
3720 if ($conf->{ivshmem
}) {
3721 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3725 $bus = print_pcie_addr
("ivshmem");
3727 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3730 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3731 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3733 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3734 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3735 .",size=$ivshmem->{size}M";
3738 # pci.4 is nested in pci.1
3739 $bridges->{1} = 1 if $bridges->{4};
3743 if (min_version
($machine_version, 2, 3)) {
3748 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3752 for my $k (sort {$b cmp $a} keys %$bridges) {
3753 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3756 if ($k == 2 && $legacy_igd) {
3759 $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
3761 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3763 # add after -readconfig pve-q35.cfg
3764 splice @$devices, 2, 0, '-device', $devstr;
3766 unshift @$devices, '-device', $devstr if $k > 0;
3771 push @$machineFlags, 'accel=tcg';
3774 my $machine_type_min = $machine_type;
3775 if ($add_pve_version) {
3776 $machine_type_min =~ s/\+pve\d+$//;
3777 $machine_type_min .= "+pve$required_pve_version";
3779 push @$machineFlags, "type=${machine_type_min}";
3781 push @$cmd, @$devices;
3782 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3783 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3784 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3786 if (my $vmstate = $conf->{vmstate
}) {
3787 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3788 push @$vollist, $vmstate;
3789 push @$cmd, '-loadstate', $statepath;
3790 print "activating and using '$vmstate' as vmstate\n";
3793 if (PVE
::QemuConfig-
>is_template($conf)) {
3794 # needed to workaround base volumes being read-only
3795 push @$cmd, '-snapshot';
3799 if ($conf->{args
}) {
3800 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3804 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3807 sub check_rng_source
{
3810 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3811 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3814 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3815 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
3816 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3817 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3818 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3819 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3827 my $res = mon_cmd
($vmid, 'query-spice');
3829 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3832 sub vm_devices_list
{
3835 my $res = mon_cmd
($vmid, 'query-pci');
3836 my $devices_to_check = [];
3838 foreach my $pcibus (@$res) {
3839 push @$devices_to_check, @{$pcibus->{devices
}},
3842 while (@$devices_to_check) {
3844 for my $d (@$devices_to_check) {
3845 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3846 next if !$d->{'pci_bridge'};
3848 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3849 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3851 $devices_to_check = $to_check;
3854 my $resblock = mon_cmd
($vmid, 'query-block');
3855 foreach my $block (@$resblock) {
3856 if($block->{device
} =~ m/^drive-(\S+)/){
3861 my $resmice = mon_cmd
($vmid, 'query-mice');
3862 foreach my $mice (@$resmice) {
3863 if ($mice->{name
} eq 'QEMU HID Tablet') {
3864 $devices->{tablet
} = 1;
3869 # for usb devices there is no query-usb
3870 # but we can iterate over the entries in
3871 # qom-list path=/machine/peripheral
3872 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3873 foreach my $per (@$resperipheral) {
3874 if ($per->{name
} =~ m/^usb\d+$/) {
3875 $devices->{$per->{name
}} = 1;
3883 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3885 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3887 my $devices_list = vm_devices_list
($vmid);
3888 return 1 if defined($devices_list->{$deviceid});
3890 # add PCI bridge if we need it for the device
3891 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3893 if ($deviceid eq 'tablet') {
3895 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
3897 } elsif ($deviceid eq 'keyboard') {
3899 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
3901 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3903 die "usb hotplug currently not reliable\n";
3904 # since we can't reliably hot unplug all added usb devices and usb
3905 # passthrough breaks live migration we disable usb hotplugging for now
3906 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3908 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3910 qemu_iothread_add
($vmid, $deviceid, $device);
3912 qemu_driveadd
($storecfg, $vmid, $device);
3913 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3915 qemu_deviceadd
($vmid, $devicefull);
3916 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3918 eval { qemu_drivedel
($vmid, $deviceid); };
3923 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3926 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3927 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3928 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3930 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3932 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3933 qemu_iothread_add
($vmid, $deviceid, $device);
3934 $devicefull .= ",iothread=iothread-$deviceid";
3937 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3938 $devicefull .= ",num_queues=$device->{queues}";
3941 qemu_deviceadd
($vmid, $devicefull);
3942 qemu_deviceaddverify
($vmid, $deviceid);
3944 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3946 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3947 qemu_driveadd
($storecfg, $vmid, $device);
3949 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3950 eval { qemu_deviceadd
($vmid, $devicefull); };
3952 eval { qemu_drivedel
($vmid, $deviceid); };
3957 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3959 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
3961 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
3962 my $use_old_bios_files = undef;
3963 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3965 my $netdevicefull = print_netdevice_full
(
3966 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3967 qemu_deviceadd
($vmid, $netdevicefull);
3969 qemu_deviceaddverify
($vmid, $deviceid);
3970 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
3973 eval { qemu_netdevdel
($vmid, $deviceid); };
3978 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3981 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3982 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3984 qemu_deviceadd
($vmid, $devicefull);
3985 qemu_deviceaddverify
($vmid, $deviceid);
3988 die "can't hotplug device '$deviceid'\n";
3994 # fixme: this should raise exceptions on error!
3995 sub vm_deviceunplug
{
3996 my ($vmid, $conf, $deviceid) = @_;
3998 my $devices_list = vm_devices_list
($vmid);
3999 return 1 if !defined($devices_list->{$deviceid});
4001 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4002 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4004 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4006 qemu_devicedel
($vmid, $deviceid);
4008 } elsif ($deviceid =~ m/^usb\d+$/) {
4010 die "usb hotplug currently not reliable\n";
4011 # when unplugging usb devices this way, there may be remaining usb
4012 # controllers/hubs so we disable it for now
4013 #qemu_devicedel($vmid, $deviceid);
4014 #qemu_devicedelverify($vmid, $deviceid);
4016 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4018 qemu_devicedel
($vmid, $deviceid);
4019 qemu_devicedelverify
($vmid, $deviceid);
4020 qemu_drivedel
($vmid, $deviceid);
4021 qemu_iothread_del
($conf, $vmid, $deviceid);
4023 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4025 qemu_devicedel
($vmid, $deviceid);
4026 qemu_devicedelverify
($vmid, $deviceid);
4027 qemu_iothread_del
($conf, $vmid, $deviceid);
4029 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4031 qemu_devicedel
($vmid, $deviceid);
4032 qemu_drivedel
($vmid, $deviceid);
4033 qemu_deletescsihw
($conf, $vmid, $deviceid);
4035 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4037 qemu_devicedel
($vmid, $deviceid);
4038 qemu_devicedelverify
($vmid, $deviceid);
4039 qemu_netdevdel
($vmid, $deviceid);
4042 die "can't unplug device '$deviceid'\n";
4048 sub qemu_deviceadd
{
4049 my ($vmid, $devicefull) = @_;
4051 $devicefull = "driver=".$devicefull;
4052 my %options = split(/[=,]/, $devicefull);
4054 mon_cmd
($vmid, "device_add" , %options);
4057 sub qemu_devicedel
{
4058 my ($vmid, $deviceid) = @_;
4060 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4063 sub qemu_iothread_add
{
4064 my($vmid, $deviceid, $device) = @_;
4066 if ($device->{iothread
}) {
4067 my $iothreads = vm_iothreads_list
($vmid);
4068 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4072 sub qemu_iothread_del
{
4073 my($conf, $vmid, $deviceid) = @_;
4075 my $confid = $deviceid;
4076 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4077 $confid = 'scsi' . $1;
4079 my $device = parse_drive
($confid, $conf->{$confid});
4080 if ($device->{iothread
}) {
4081 my $iothreads = vm_iothreads_list
($vmid);
4082 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4086 sub qemu_objectadd
{
4087 my($vmid, $objectid, $qomtype) = @_;
4089 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4094 sub qemu_objectdel
{
4095 my($vmid, $objectid) = @_;
4097 mon_cmd
($vmid, "object-del", id
=> $objectid);
4103 my ($storecfg, $vmid, $device) = @_;
4105 my $kvmver = get_running_qemu_version
($vmid);
4106 my $io_uring = min_version
($kvmver, 6, 0);
4107 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4108 $drive =~ s/\\/\\\\/g;
4109 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4111 # If the command succeeds qemu prints: "OK
"
4112 return 1 if $ret =~ m/OK/s;
4114 die "adding drive failed
: $ret\n";
4118 my($vmid, $deviceid) = @_;
4120 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4123 return 1 if $ret eq "";
4125 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4126 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4128 die "deleting drive
$deviceid failed
: $ret\n";
4131 sub qemu_deviceaddverify {
4132 my ($vmid, $deviceid) = @_;
4134 for (my $i = 0; $i <= 5; $i++) {
4135 my $devices_list = vm_devices_list($vmid);
4136 return 1 if defined($devices_list->{$deviceid});
4140 die "error on hotplug device
'$deviceid'\n";
4144 sub qemu_devicedelverify {
4145 my ($vmid, $deviceid) = @_;
4147 # need to verify that the device is correctly removed as device_del
4148 # is async and empty return is not reliable
4150 for (my $i = 0; $i <= 5; $i++) {
4151 my $devices_list = vm_devices_list($vmid);
4152 return 1 if !defined($devices_list->{$deviceid});
4156 die "error on hot-unplugging device
'$deviceid'\n";
4159 sub qemu_findorcreatescsihw {
4160 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4162 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4164 my $scsihwid="$controller_prefix$controller";
4165 my $devices_list = vm_devices_list($vmid);
4167 if(!defined($devices_list->{$scsihwid})) {
4168 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4174 sub qemu_deletescsihw {
4175 my ($conf, $vmid, $opt) = @_;
4177 my $device = parse_drive($opt, $conf->{$opt});
4179 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4180 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4184 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4186 my $devices_list = vm_devices_list($vmid);
4187 foreach my $opt (keys %{$devices_list}) {
4188 if (is_valid_drivename($opt)) {
4189 my $drive = parse_drive($opt, $conf->{$opt});
4190 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4196 my $scsihwid="scsihw
$controller";
4198 vm_deviceunplug($vmid, $conf, $scsihwid);
4203 sub qemu_add_pci_bridge {
4204 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4210 print_pci_addr($device, $bridges, $arch, $machine_type);
4212 while (my ($k, $v) = each %$bridges) {
4215 return 1 if !defined($bridgeid) || $bridgeid < 1;
4217 my $bridge = "pci
.$bridgeid";
4218 my $devices_list = vm_devices_list($vmid);
4220 if (!defined($devices_list->{$bridge})) {
4221 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4227 sub qemu_set_link_status {
4228 my ($vmid, $device, $up) = @_;
4230 mon_cmd($vmid, "set_link
", name => $device,
4231 up => $up ? JSON::true : JSON::false);
4234 sub qemu_netdevadd {
4235 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4237 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4238 my %options = split(/[=,]/, $netdev);
4240 if (defined(my $vhost = $options{vhost})) {
4241 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4244 if (defined(my $queues = $options{queues})) {
4245 $options{queues} = $queues + 0;
4248 mon_cmd($vmid, "netdev_add
", %options);
4252 sub qemu_netdevdel {
4253 my ($vmid, $deviceid) = @_;
4255 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4258 sub qemu_usb_hotplug {
4259 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4263 # remove the old one first
4264 vm_deviceunplug($vmid, $conf, $deviceid);
4266 # check if xhci controller is necessary and available
4267 if ($device->{usb3}) {
4269 my $devicelist = vm_devices_list($vmid);
4271 if (!$devicelist->{xhci}) {
4272 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4273 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4276 my $d = parse_usb_device($device->{host});
4277 $d->{usb3} = $device->{usb3};
4280 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4283 sub qemu_cpu_hotplug {
4284 my ($vmid, $conf, $vcpus) = @_;
4286 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4289 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4290 $sockets = $conf->{sockets} if $conf->{sockets};
4291 my $cores = $conf->{cores} || 1;
4292 my $maxcpus = $sockets * $cores;
4294 $vcpus = $maxcpus if !$vcpus;
4296 die "you can
't add more vcpus than maxcpus\n"
4297 if $vcpus > $maxcpus;
4299 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4301 if ($vcpus < $currentvcpus) {
4303 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4305 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4306 qemu_devicedel($vmid, "cpu$i");
4308 my $currentrunningvcpus = undef;
4310 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4311 last if scalar(@{$currentrunningvcpus}) == $i-1;
4312 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4316 #update conf after each succesfull cpu unplug
4317 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4318 PVE::QemuConfig->write_config($vmid, $conf);
4321 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4327 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4328 die "vcpus in running vm does not match its configuration\n"
4329 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4331 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4333 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4334 my $cpustr = print_cpu_device($conf, $i);
4335 qemu_deviceadd($vmid, $cpustr);
4338 my $currentrunningvcpus = undef;
4340 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4341 last if scalar(@{$currentrunningvcpus}) == $i;
4342 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4346 #update conf after each succesfull cpu hotplug
4347 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4348 PVE::QemuConfig->write_config($vmid, $conf);
4352 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4353 mon_cmd($vmid, "cpu-add", id => int($i));
4358 sub qemu_block_set_io_throttle {
4359 my ($vmid, $deviceid,
4360 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4361 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4362 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4363 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4365 return if !check_running($vmid) ;
4367 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4369 bps_rd => int($bps_rd),
4370 bps_wr => int($bps_wr),
4372 iops_rd => int($iops_rd),
4373 iops_wr => int($iops_wr),
4374 bps_max => int($bps_max),
4375 bps_rd_max => int($bps_rd_max),
4376 bps_wr_max => int($bps_wr_max),
4377 iops_max => int($iops_max),
4378 iops_rd_max => int($iops_rd_max),
4379 iops_wr_max => int($iops_wr_max),
4380 bps_max_length => int($bps_max_length),
4381 bps_rd_max_length => int($bps_rd_max_length),
4382 bps_wr_max_length => int($bps_wr_max_length),
4383 iops_max_length => int($iops_max_length),
4384 iops_rd_max_length => int($iops_rd_max_length),
4385 iops_wr_max_length => int($iops_wr_max_length),
4390 sub qemu_block_resize {
4391 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4393 my $running = check_running($vmid);
4395 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4397 return if !$running;
4399 my $padding = (1024 - $size % 1024) % 1024;
4400 $size = $size + $padding;
4405 device => $deviceid,
4411 sub qemu_volume_snapshot {
4412 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4414 my $running = check_running($vmid);
4416 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4417 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4419 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4423 sub qemu_volume_snapshot_delete {
4424 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4426 my $running = check_running($vmid);
4431 my $conf = PVE::QemuConfig->load_config($vmid);
4432 PVE::QemuConfig->foreach_volume($conf, sub {
4433 my ($ds, $drive) = @_;
4434 $running = 1 if $drive->{file} eq $volid;
4438 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4439 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4441 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4445 sub set_migration_caps {
4446 my ($vmid, $savevm) = @_;
4448 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4450 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4451 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4456 "auto-converge" => 1,
4458 "x-rdma-pin-all" => 0,
4461 "dirty-bitmaps" => $dirty_bitmaps,
4464 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4466 for my $supported_capability (@$supported_capabilities) {
4468 capability => $supported_capability->{capability},
4469 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4473 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4477 my ($conf, $func, @param) = @_;
4481 my $test_volid = sub {
4482 my ($key, $drive, $snapname) = @_;
4484 my $volid = $drive->{file};
4487 $volhash->{$volid}->{cdrom} //= 1;
4488 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4490 my $replicate = $drive->{replicate} // 1;
4491 $volhash->{$volid}->{replicate} //= 0;
4492 $volhash->{$volid}->{replicate} = 1 if $replicate;
4494 $volhash->{$volid}->{shared} //= 0;
4495 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4497 $volhash->{$volid}->{referenced_in_config} //= 0;
4498 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4500 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4501 if defined($snapname);
4503 my $size = $drive->{size};
4504 $volhash->{$volid}->{size} //= $size if $size;
4506 $volhash->{$volid}->{is_vmstate} //= 0;
4507 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4509 $volhash->{$volid}->{is_unused} //= 0;
4510 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4512 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4515 my $include_opts = {
4516 extra_keys => ['vmstate
'],
4517 include_unused => 1,
4520 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4521 foreach my $snapname (keys %{$conf->{snapshots}}) {
4522 my $snap = $conf->{snapshots}->{$snapname};
4523 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4526 foreach my $volid (keys %$volhash) {
4527 &$func($volid, $volhash->{$volid}, @param);
4531 my $fast_plug_option = {
4539 'vmstatestorage
' => 1,
4544 # hotplug changes in [PENDING]
4545 # $selection hash can be used to only apply specified options, for
4546 # example: { cores => 1 } (only apply changed 'cores
')
4547 # $errors ref is used to return error messages
4548 sub vmconfig_hotplug_pending {
4549 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4551 my $defaults = load_defaults();
4552 my $arch = get_vm_arch($conf);
4553 my $machine_type = get_vm_machine($conf, undef, $arch);
4555 # commit values which do not have any impact on running VM first
4556 # Note: those option cannot raise errors, we we do not care about
4557 # $selection and always apply them.
4559 my $add_error = sub {
4560 my ($opt, $msg) = @_;
4561 $errors->{$opt} = "hotplug problem - $msg";
4565 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4566 if ($fast_plug_option->{$opt}) {
4567 $conf->{$opt} = $conf->{pending}->{$opt};
4568 delete $conf->{pending}->{$opt};
4574 PVE::QemuConfig->write_config($vmid, $conf);
4577 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4579 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4580 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4581 foreach my $opt (sort keys %$pending_delete_hash) {
4582 next if $selection && !$selection->{$opt};
4583 my $force = $pending_delete_hash->{$opt}->{force};
4585 if ($opt eq 'hotplug
') {
4586 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4587 } elsif ($opt eq 'tablet
') {
4588 die "skip\n" if !$hotplug_features->{usb};
4589 if ($defaults->{tablet}) {
4590 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4591 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4592 if $arch eq 'aarch64
';
4594 vm_deviceunplug($vmid, $conf, 'tablet
');
4595 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4597 } elsif ($opt =~ m/^usb\d+/) {
4599 # since we cannot reliably hot unplug usb devices we are disabling it
4600 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4601 #vm_deviceunplug($vmid, $conf, $opt);
4602 } elsif ($opt eq 'vcpus
') {
4603 die "skip\n" if !$hotplug_features->{cpu};
4604 qemu_cpu_hotplug($vmid, $conf, undef);
4605 } elsif ($opt eq 'balloon
') {
4606 # enable balloon device is not hotpluggable
4607 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4608 # here we reset the ballooning value to memory
4609 my $balloon = $conf->{memory} || $defaults->{memory};
4610 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4611 } elsif ($fast_plug_option->{$opt}) {
4613 } elsif ($opt =~ m/^net(\d+)$/) {
4614 die "skip\n" if !$hotplug_features->{network};
4615 vm_deviceunplug($vmid, $conf, $opt);
4616 } elsif (is_valid_drivename($opt)) {
4617 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4618 vm_deviceunplug($vmid, $conf, $opt);
4619 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4620 } elsif ($opt =~ m/^memory$/) {
4621 die "skip\n" if !$hotplug_features->{memory};
4622 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4623 } elsif ($opt eq 'cpuunits
') {
4624 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4625 } elsif ($opt eq 'cpulimit
') {
4626 $cgroup->change_cpu_quota(-1, 100000);
4632 &$add_error($opt, $err) if $err ne "skip\n";
4634 delete $conf->{$opt};
4635 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4639 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4640 $apply_pending_cloudinit = sub {
4641 return if $apply_pending_cloudinit_done; # once is enough
4642 $apply_pending_cloudinit_done = 1; # once is enough
4644 my ($key, $value) = @_;
4646 my @cloudinit_opts = keys %$confdesc_cloudinit;
4647 foreach my $opt (keys %{$conf->{pending}}) {
4648 next if !grep { $_ eq $opt } @cloudinit_opts;
4649 $conf->{$opt} = delete $conf->{pending}->{$opt};
4652 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4653 foreach my $opt (sort keys %$pending_delete_hash) {
4654 next if !grep { $_ eq $opt } @cloudinit_opts;
4655 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4656 delete $conf->{$opt};
4659 my $new_conf = { %$conf };
4660 $new_conf->{$key} = $value;
4661 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4664 foreach my $opt (keys %{$conf->{pending}}) {
4665 next if $selection && !$selection->{$opt};
4666 my $value = $conf->{pending}->{$opt};
4668 if ($opt eq 'hotplug
') {
4669 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4670 } elsif ($opt eq 'tablet
') {
4671 die "skip\n" if !$hotplug_features->{usb};
4673 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4674 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4675 if $arch eq 'aarch64
';
4676 } elsif ($value == 0) {
4677 vm_deviceunplug($vmid, $conf, 'tablet
');
4678 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4680 } elsif ($opt =~ m/^usb\d+$/) {
4682 # since we cannot reliably hot unplug usb devices we disable it for now
4683 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4684 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4685 #die "skip\n" if !$d;
4686 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4687 } elsif ($opt eq 'vcpus
') {
4688 die "skip\n" if !$hotplug_features->{cpu};
4689 qemu_cpu_hotplug($vmid, $conf, $value);
4690 } elsif ($opt eq 'balloon
') {
4691 # enable/disable balloning device is not hotpluggable
4692 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4693 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4694 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4696 # allow manual ballooning if shares is set to zero
4697 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4698 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4699 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4701 } elsif ($opt =~ m/^net(\d+)$/) {
4702 # some changes can be done without hotplug
4703 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4704 $vmid, $opt, $value, $arch, $machine_type);
4705 } elsif (is_valid_drivename($opt)) {
4706 die "skip\n" if $opt eq 'efidisk0
';
4707 # some changes can be done without hotplug
4708 my $drive = parse_drive($opt, $value);
4709 if (drive_is_cloudinit($drive)) {
4710 &$apply_pending_cloudinit($opt, $value);
4712 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4713 $vmid, $opt, $value, $arch, $machine_type);
4714 } elsif ($opt =~ m/^memory$/) { #dimms
4715 die "skip\n" if !$hotplug_features->{memory};
4716 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4717 } elsif ($opt eq 'cpuunits
') {
4718 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4719 } elsif ($opt eq 'cpulimit
') {
4720 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4721 $cgroup->change_cpu_quota($cpulimit, 100000);
4723 die "skip\n"; # skip non-hot-pluggable options
4727 &$add_error($opt, $err) if $err ne "skip\n";
4729 $conf->{$opt} = $value;
4730 delete $conf->{pending}->{$opt};
4734 PVE::QemuConfig->write_config($vmid, $conf);
4737 sub try_deallocate_drive {
4738 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4740 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4741 my $volid = $drive->{file};
4742 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4743 my $sid = PVE::Storage::parse_volume_id($volid);
4744 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4746 # check if the disk is really unused
4747 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4748 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4749 PVE::Storage::vdisk_free($storecfg, $volid);
4752 # If vm is not owner of this disk remove from config
4760 sub vmconfig_delete_or_detach_drive {
4761 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4763 my $drive = parse_drive($opt, $conf->{$opt});
4765 my $rpcenv = PVE::RPCEnvironment::get();
4766 my $authuser = $rpcenv->get_user();
4769 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4770 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4772 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4778 sub vmconfig_apply_pending {
4779 my ($vmid, $conf, $storecfg, $errors) = @_;
4781 my $add_apply_error = sub {
4782 my ($opt, $msg) = @_;
4783 my $err_msg = "unable to apply pending change $opt : $msg";
4784 $errors->{$opt} = $err_msg;
4790 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4791 foreach my $opt (sort keys %$pending_delete_hash) {
4792 my $force = $pending_delete_hash->{$opt}->{force};
4794 if ($opt =~ m/^unused/) {
4795 die "internal error";
4796 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4797 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4801 $add_apply_error->($opt, $err);
4803 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4804 delete $conf->{$opt};
4808 PVE::QemuConfig->cleanup_pending($conf);
4810 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4811 next if $opt eq 'delete'; # just to be sure
4813 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4814 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4818 $add_apply_error->($opt, $err);
4820 $conf->{$opt} = delete $conf->{pending}->{$opt};
4824 # write all changes at once to avoid unnecessary i/o
4825 PVE::QemuConfig->write_config($vmid, $conf);
4828 sub vmconfig_update_net {
4829 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4831 my $newnet = parse_net($value);
4833 if ($conf->{$opt}) {
4834 my $oldnet = parse_net($conf->{$opt});
4836 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4837 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4838 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4839 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4841 # for non online change, we try to hot-unplug
4842 die "skip\n" if !$hotplug;
4843 vm_deviceunplug($vmid, $conf, $opt);
4846 die "internal error" if $opt !~ m/net(\d+)/;
4847 my $iface = "tap${vmid}i$1";
4849 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4850 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4851 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4852 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4853 PVE::Network::tap_unplug($iface);
4856 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4858 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4860 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4861 # Rate can be applied on its own but any change above needs to
4862 # include the rate in tap_plug since OVS resets everything.
4863 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4866 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4867 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4875 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4881 sub vmconfig_update_disk {
4882 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4884 my $drive = parse_drive($opt, $value);
4886 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4887 my $media = $drive->{media} || 'disk
';
4888 my $oldmedia = $old_drive->{media} || 'disk
';
4889 die "unable to change media type\n" if $media ne $oldmedia;
4891 if (!drive_is_cdrom($old_drive)) {
4893 if ($drive->{file} ne $old_drive->{file}) {
4895 die "skip\n" if !$hotplug;
4897 # unplug and register as unused
4898 vm_deviceunplug($vmid, $conf, $opt);
4899 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4902 # update existing disk
4904 # skip non hotpluggable value
4905 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4906 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4907 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4908 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4909 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4914 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4915 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4916 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4917 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4918 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4919 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4920 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4921 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4922 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4923 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4924 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4925 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4926 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4927 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4928 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4929 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4930 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4931 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4933 qemu_block_set_io_throttle(
4935 ($drive->{mbps} || 0)*1024*1024,
4936 ($drive->{mbps_rd} || 0)*1024*1024,
4937 ($drive->{mbps_wr} || 0)*1024*1024,
4938 $drive->{iops} || 0,
4939 $drive->{iops_rd} || 0,
4940 $drive->{iops_wr} || 0,
4941 ($drive->{mbps_max} || 0)*1024*1024,
4942 ($drive->{mbps_rd_max} || 0)*1024*1024,
4943 ($drive->{mbps_wr_max} || 0)*1024*1024,
4944 $drive->{iops_max} || 0,
4945 $drive->{iops_rd_max} || 0,
4946 $drive->{iops_wr_max} || 0,
4947 $drive->{bps_max_length} || 1,
4948 $drive->{bps_rd_max_length} || 1,
4949 $drive->{bps_wr_max_length} || 1,
4950 $drive->{iops_max_length} || 1,
4951 $drive->{iops_rd_max_length} || 1,
4952 $drive->{iops_wr_max_length} || 1,
4962 if ($drive->{file} eq 'none
') {
4963 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4964 if (drive_is_cloudinit($old_drive)) {
4965 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4968 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4970 # force eject if locked
4971 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4974 mon_cmd($vmid, "blockdev-change-medium",
4975 id => "$opt", filename => "$path");
4983 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4985 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4986 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4989 # called in locked context by incoming migration
4990 sub vm_migrate_get_nbd_disks {
4991 my ($storecfg, $conf, $replicated_volumes) = @_;
4993 my $local_volumes = {};
4994 PVE::QemuConfig->foreach_volume($conf, sub {
4995 my ($ds, $drive) = @_;
4997 return if drive_is_cdrom($drive);
4999 my $volid = $drive->{file};
5003 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5005 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5006 return if $scfg->{shared};
5008 # replicated disks re-use existing state via bitmap
5009 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5010 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5012 return $local_volumes;
5015 # called in locked context by incoming migration
5016 sub vm_migrate_alloc_nbd_disks {
5017 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5022 foreach my $opt (sort keys %$source_volumes) {
5023 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5025 if ($use_existing) {
5026 $nbd->{$opt}->{drivestr} = print_drive($drive);
5027 $nbd->{$opt}->{volid} = $volid;
5028 $nbd->{$opt}->{replicated} = 1;
5032 # If a remote storage is specified and the format of the original
5033 # volume is not available there, fall back to the default format.
5034 # Otherwise use the same format as the original.
5035 if (!$storagemap->{identity}) {
5036 $storeid = map_storage($storagemap, $storeid);
5037 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5038 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5039 my $fileFormat = qemu_img_format($scfg, $volname);
5040 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5042 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5043 $format = qemu_img_format($scfg, $volname);
5046 my $size = $drive->{size} / 1024;
5047 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5048 my $newdrive = $drive;
5049 $newdrive->{format} = $format;
5050 $newdrive->{file} = $newvolid;
5051 my $drivestr = print_drive($newdrive);
5052 $nbd->{$opt}->{drivestr} = $drivestr;
5053 $nbd->{$opt}->{volid} = $newvolid;
5059 # see vm_start_nolock for parameters, additionally:
5061 # storagemap = parsed storage map for allocating NBD disks
5063 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5065 return PVE::QemuConfig->lock_config($vmid, sub {
5066 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5068 die "you can't start a vm
if it
's a template\n"
5069 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5071 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended
');
5072 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup
');
5074 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5076 if ($has_backup_lock && $running) {
5077 # a backup is currently running, attempt to start the guest in the
5078 # existing QEMU instance
5079 return vm_resume($vmid);
5082 PVE::QemuConfig->check_lock($conf)
5083 if !($params->{skiplock} || $has_suspended_lock);
5085 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5087 die "VM $vmid already running\n" if $running;
5089 if (my $storagemap = $migrate_opts->{storagemap}) {
5090 my $replicated = $migrate_opts->{replicated_volumes};
5091 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5092 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5094 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5095 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5099 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5105 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
5106 # skiplock => 0/1, skip checking for config lock
5107 # skiptemplate => 0/1, skip checking whether VM is template
5108 # forcemachine => to force Qemu machine (rollback/migration)
5109 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
5110 # timeout => in seconds
5111 # paused => start VM in paused state (backup)
5112 # resume => resume from hibernation
5123 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5124 # migratedfrom => source node
5125 # spice_ticket => used for spice migration, passed via tunnel/stdin
5126 # network => CIDR of migration network
5127 # type => secure/insecure - tunnel over encrypted connection or plain-text
5128 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5129 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5130 sub vm_start_nolock {
5131 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5133 my $statefile = $params->{statefile};
5134 my $resume = $params->{resume};
5136 my $migratedfrom = $migrate_opts->{migratedfrom};
5137 my $migration_type = $migrate_opts->{type};
5141 # clean up leftover reboot request files
5142 eval { clear_reboot_request($vmid); };
5145 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5146 vmconfig_apply_pending($vmid, $conf, $storecfg);
5147 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5150 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5152 my $defaults = load_defaults();
5154 # set environment variable useful inside network script
5155 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5157 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
5159 my $forcemachine = $params->{forcemachine};
5160 my $forcecpu = $params->{forcecpu};
5162 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5163 $forcemachine = $conf->{runningmachine};
5164 $forcecpu = $conf->{runningcpu};
5165 print "Resuming suspended VM\n";
5168 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5169 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing
'});
5172 my $get_migration_ip = sub {
5173 my ($nodename) = @_;
5175 return $migration_ip if defined($migration_ip);
5177 my $cidr = $migrate_opts->{network};
5179 if (!defined($cidr)) {
5180 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5181 $cidr = $dc_conf->{migration}->{network};
5184 if (defined($cidr)) {
5185 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5187 die "could not get IP: no address configured on local " .
5188 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5190 die "could not get IP: multiple addresses configured on local " .
5191 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5193 $migration_ip = @$ips[0];
5196 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5197 if !defined($migration_ip);
5199 return $migration_ip;
5204 if ($statefile eq 'tcp
') {
5205 my $localip = "localhost";
5206 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5207 my $nodename = nodename();
5209 if (!defined($migration_type)) {
5210 if (defined($datacenterconf->{migration}->{type})) {
5211 $migration_type = $datacenterconf->{migration}->{type};
5213 $migration_type = 'secure
';
5217 if ($migration_type eq 'insecure
') {
5218 $localip = $get_migration_ip->($nodename);
5219 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5222 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5223 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5224 $migrate_uri = "tcp:${localip}:${migrate_port}";
5225 push @$cmd, '-incoming
', $migrate_uri;
5228 } elsif ($statefile eq 'unix
') {
5229 # should be default for secure migrations as a ssh TCP forward
5230 # tunnel is not deterministic reliable ready and fails regurarly
5231 # to set up in time, so use UNIX socket forwards
5232 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5233 unlink $socket_addr;
5235 $migrate_uri = "unix:$socket_addr";
5237 push @$cmd, '-incoming
', $migrate_uri;
5240 } elsif (-e $statefile) {
5241 push @$cmd, '-loadstate
', $statefile;
5243 my $statepath = PVE::Storage::path($storecfg, $statefile);
5244 push @$vollist, $statefile;
5245 push @$cmd, '-loadstate
', $statepath;
5247 } elsif ($params->{paused}) {
5252 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
5253 my $d = parse_hostpci($conf->{"hostpci$i"});
5255 my $pcidevices = $d->{pciid};
5256 foreach my $pcidevice (@$pcidevices) {
5257 my $pciid = $pcidevice->{id};
5259 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5260 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5261 die "no pci device info for device '$pciid'\n" if !$info;
5264 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5265 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5267 die "can't unbind
/bind PCI group to VFIO
'$pciid'\n"
5268 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5269 die "can
't reset PCI device '$pciid'\n"
5270 if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5275 PVE::Storage::activate_volumes($storecfg, $vollist);
5278 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5279 outfunc => sub {}, errfunc => sub {});
5281 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5282 # timeout should be more than enough here...
5283 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5285 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5286 : $defaults->{cpuunits};
5288 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5290 timeout => $statefile ? undef : $start_timeout,
5295 # when migrating, prefix QEMU output so other side can pick up any
5296 # errors that might occur and show the user
5297 if ($migratedfrom) {
5298 $run_params{quiet} = 1;
5299 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5303 Slice => 'qemu
.slice
',
5304 KillMode => 'process
',
5306 TimeoutStopUSec => ULONG_MAX, # infinity
5309 if (PVE::CGroup::cgroup_mode() == 2) {
5310 $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5311 $properties{CPUWeight} = $cpuunits;
5313 $properties{CPUShares} = $cpuunits;
5316 if (my $cpulimit = $conf->{cpulimit}) {
5317 $properties{CPUQuota} = int($cpulimit * 100);
5319 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5321 my $run_qemu = sub {
5322 PVE::Tools::run_fork sub {
5323 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5325 my $exitcode = run_command($cmd, %run_params);
5326 die "QEMU exited with code $exitcode\n" if $exitcode;
5330 if ($conf->{hugepages}) {
5333 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5334 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5336 PVE::QemuServer::Memory::hugepages_mount();
5337 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5339 eval { $run_qemu->() };
5341 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5342 if !$conf->{keephugepages};
5346 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5347 if !$conf->{keephugepages};
5349 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5352 eval { $run_qemu->() };
5356 # deactivate volumes if start fails
5357 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5358 die "start failed: $err";
5361 print "migration listens on $migrate_uri\n" if $migrate_uri;
5362 $res->{migrate_uri} = $migrate_uri;
5364 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5365 eval { mon_cmd($vmid, "cont"); };
5369 #start nbd server for storage migration
5370 if (my $nbd = $migrate_opts->{nbd}) {
5371 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5373 my $migrate_storage_uri;
5374 # nbd_protocol_version > 0 for unix socket support
5375 if ($nbd_protocol_version > 0 && $migration_type eq 'secure
') {
5376 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5377 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix
', data => { path => $socket_path } } );
5378 $migrate_storage_uri = "nbd:unix:$socket_path";
5380 my $nodename = nodename();
5381 my $localip = $get_migration_ip->($nodename);
5382 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5383 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5385 mon_cmd($vmid, "nbd-server-start", addr => {
5388 host => "${localip}",
5389 port => "${storage_migrate_port}",
5392 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5393 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5396 $res->{migrate_storage_uri} = $migrate_storage_uri;
5398 foreach my $opt (sort keys %$nbd) {
5399 my $drivestr = $nbd->{$opt}->{drivestr};
5400 my $volid = $nbd->{$opt}->{volid};
5401 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5402 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5403 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5404 print "re-using replicated volume: $opt - $volid\n"
5405 if $nbd->{$opt}->{replicated};
5407 $res->{drives}->{$opt} = $nbd->{$opt};
5408 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5412 if ($migratedfrom) {
5414 set_migration_caps($vmid);
5419 print "spice listens on port $spice_port\n";
5420 $res->{spice_port} = $spice_port;
5421 if ($migrate_opts->{spice_ticket}) {
5422 mon_cmd($vmid, "set_password", protocol => 'spice
', password =>
5423 $migrate_opts->{spice_ticket});
5424 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5429 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5430 if !$statefile && $conf->{balloon};
5432 foreach my $opt (keys %$conf) {
5433 next if $opt !~ m/^net\d+$/;
5434 my $nicconf = parse_net($conf->{$opt});
5435 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5439 mon_cmd($vmid, 'qom-set
',
5440 path => "machine/peripheral/balloon0",
5441 property => "guest-stats-polling-interval",
5442 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5445 print "Resumed VM, removing state\n";
5446 if (my $vmstate = $conf->{vmstate}) {
5447 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5448 PVE::Storage::vdisk_free($storecfg, $vmstate);
5450 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5451 PVE
::QemuConfig-
>write_config($vmid, $conf);
5454 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5459 sub vm_commandline
{
5460 my ($storecfg, $vmid, $snapname) = @_;
5462 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5467 my $snapshot = $conf->{snapshots
}->{$snapname};
5468 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5470 # check for machine or CPU overrides in snapshot
5471 $forcemachine = $snapshot->{runningmachine
};
5472 $forcecpu = $snapshot->{runningcpu
};
5474 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5479 my $defaults = load_defaults
();
5481 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults,
5482 $forcemachine, $forcecpu);
5484 return PVE
::Tools
::cmd2string
($cmd);
5488 my ($vmid, $skiplock) = @_;
5490 PVE
::QemuConfig-
>lock_config($vmid, sub {
5492 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5494 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5496 mon_cmd
($vmid, "system_reset");
5500 sub get_vm_volumes
{
5504 foreach_volid
($conf, sub {
5505 my ($volid, $attr) = @_;
5507 return if $volid =~ m
|^/|;
5509 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5512 push @$vollist, $volid;
5518 sub vm_stop_cleanup
{
5519 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5524 my $vollist = get_vm_volumes
($conf);
5525 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5528 foreach my $ext (qw(mon qmp pid vnc qga)) {
5529 unlink "/var/run/qemu-server/${vmid}.$ext";
5532 if ($conf->{ivshmem
}) {
5533 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5534 # just delete it for now, VMs which have this already open do not
5535 # are affected, but new VMs will get a separated one. If this
5536 # becomes an issue we either add some sort of ref-counting or just
5537 # add a "don't delete on stop" flag to the ivshmem format.
5538 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5541 foreach my $key (keys %$conf) {
5542 next if $key !~ m/^hostpci(\d+)$/;
5543 my $hostpciindex = $1;
5544 my $d = parse_hostpci
($conf->{$key});
5545 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5547 foreach my $pci (@{$d->{pciid
}}) {
5548 my $pciid = $pci->{id
};
5549 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5553 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5555 warn $@ if $@; # avoid errors - just warn
5558 # call only in locked context
5560 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5562 my $pid = check_running
($vmid, $nocheck);
5567 $conf = PVE
::QemuConfig-
>load_config($vmid);
5568 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5569 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5570 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5571 $timeout = $opts->{down
} if $opts->{down
};
5573 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5578 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
5579 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5581 mon_cmd
($vmid, "system_powerdown");
5584 mon_cmd
($vmid, "quit");
5590 $timeout = 60 if !defined($timeout);
5593 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5598 if ($count >= $timeout) {
5600 warn "VM still running - terminating now with SIGTERM\n";
5603 die "VM quit/powerdown failed - got timeout\n";
5606 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5610 if (!check_running
($vmid, $nocheck)) {
5611 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5615 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5618 die "VM quit/powerdown failed\n";
5626 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5631 if ($count >= $timeout) {
5632 warn "VM still running - terminating now with SIGKILL\n";
5637 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5640 # Note: use $nocheck to skip tests if VM configuration file exists.
5641 # We need that when migration VMs to other nodes (files already moved)
5642 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5644 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5646 $force = 1 if !defined($force) && !$shutdown;
5649 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5650 kill 15, $pid if $pid;
5651 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5652 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5656 PVE
::QemuConfig-
>lock_config($vmid, sub {
5657 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5662 my ($vmid, $timeout) = @_;
5664 PVE
::QemuConfig-
>lock_config($vmid, sub {
5667 # only reboot if running, as qmeventd starts it again on a stop event
5668 return if !check_running
($vmid);
5670 create_reboot_request
($vmid);
5672 my $storecfg = PVE
::Storage
::config
();
5673 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5677 # avoid that the next normal shutdown will be confused for a reboot
5678 clear_reboot_request
($vmid);
5684 # note: if using the statestorage parameter, the caller has to check privileges
5686 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5693 PVE
::QemuConfig-
>lock_config($vmid, sub {
5695 $conf = PVE
::QemuConfig-
>load_config($vmid);
5697 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5698 PVE
::QemuConfig-
>check_lock($conf)
5699 if !($skiplock || $is_backing_up);
5701 die "cannot suspend to disk during backup\n"
5702 if $is_backing_up && $includestate;
5704 if ($includestate) {
5705 $conf->{lock} = 'suspending';
5706 my $date = strftime
("%Y-%m-%d", localtime(time()));
5707 $storecfg = PVE
::Storage
::config
();
5708 if (!$statestorage) {
5709 $statestorage = find_vmstate_storage
($conf, $storecfg);
5710 # check permissions for the storage
5711 my $rpcenv = PVE
::RPCEnvironment
::get
();
5712 if ($rpcenv->{type
} ne 'cli') {
5713 my $authuser = $rpcenv->get_user();
5714 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5719 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
5720 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5721 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5722 PVE
::QemuConfig-
>write_config($vmid, $conf);
5724 mon_cmd
($vmid, "stop");
5728 if ($includestate) {
5730 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5733 set_migration_caps
($vmid, 1);
5734 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5736 my $state = mon_cmd
($vmid, "query-savevm");
5737 if (!$state->{status
}) {
5738 die "savevm not active\n";
5739 } elsif ($state->{status
} eq 'active') {
5742 } elsif ($state->{status
} eq 'completed') {
5743 print "State saved, quitting\n";
5745 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5746 die "query-savevm failed with error '$state->{error}'\n"
5748 die "query-savevm returned status '$state->{status}'\n";
5754 PVE
::QemuConfig-
>lock_config($vmid, sub {
5755 $conf = PVE
::QemuConfig-
>load_config($vmid);
5757 # cleanup, but leave suspending lock, to indicate something went wrong
5759 mon_cmd
($vmid, "savevm-end");
5760 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5761 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5762 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5763 PVE
::QemuConfig-
>write_config($vmid, $conf);
5769 die "lock changed unexpectedly\n"
5770 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5772 mon_cmd
($vmid, "quit");
5773 $conf->{lock} = 'suspended';
5774 PVE
::QemuConfig-
>write_config($vmid, $conf);
5780 my ($vmid, $skiplock, $nocheck) = @_;
5782 PVE
::QemuConfig-
>lock_config($vmid, sub {
5783 my $res = mon_cmd
($vmid, 'query-status');
5784 my $resume_cmd = 'cont';
5787 if ($res->{status
}) {
5788 return if $res->{status
} eq 'running'; # job done, go home
5789 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
5790 $reset = 1 if $res->{status
} eq 'shutdown';
5795 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5797 PVE
::QemuConfig-
>check_lock($conf)
5798 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5802 # required if a VM shuts down during a backup and we get a resume
5803 # request before the backup finishes for example
5804 mon_cmd
($vmid, "system_reset");
5806 mon_cmd
($vmid, $resume_cmd);
5811 my ($vmid, $skiplock, $key) = @_;
5813 PVE
::QemuConfig-
>lock_config($vmid, sub {
5815 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5817 # there is no qmp command, so we use the human monitor command
5818 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5819 die $res if $res ne '';
5823 # vzdump restore implementaion
5825 sub tar_archive_read_firstfile
{
5826 my $archive = shift;
5828 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5830 # try to detect archive type first
5831 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5832 die "unable to open file '$archive'\n";
5833 my $firstfile = <$fh>;
5837 die "ERROR: archive contaions no data\n" if !$firstfile;
5843 sub tar_restore_cleanup
{
5844 my ($storecfg, $statfile) = @_;
5846 print STDERR
"starting cleanup\n";
5848 if (my $fd = IO
::File-
>new($statfile, "r")) {
5849 while (defined(my $line = <$fd>)) {
5850 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5853 if ($volid =~ m
|^/|) {
5854 unlink $volid || die 'unlink failed\n';
5856 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5858 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5860 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5862 print STDERR
"unable to parse line in statfile - $line";
5869 sub restore_file_archive
{
5870 my ($archive, $vmid, $user, $opts) = @_;
5872 return restore_vma_archive
($archive, $vmid, $user, $opts)
5875 my $info = PVE
::Storage
::archive_info
($archive);
5876 my $format = $opts->{format
} // $info->{format
};
5877 my $comp = $info->{compression
};
5879 # try to detect archive format
5880 if ($format eq 'tar') {
5881 return restore_tar_archive
($archive, $vmid, $user, $opts);
5883 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5887 # hepler to remove disks that will not be used after restore
5888 my $restore_cleanup_oldconf = sub {
5889 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5891 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
5892 my ($ds, $drive) = @_;
5894 return if drive_is_cdrom
($drive, 1);
5896 my $volid = $drive->{file
};
5897 return if !$volid || $volid =~ m
|^/|;
5899 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
5900 return if !$path || !$owner || ($owner != $vmid);
5902 # Note: only delete disk we want to restore
5903 # other volumes will become unused
5904 if ($virtdev_hash->{$ds}) {
5905 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
5912 # delete vmstate files, after the restore we have no snapshots anymore
5913 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5914 my $snap = $oldconf->{snapshots
}->{$snapname};
5915 if ($snap->{vmstate
}) {
5916 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
5924 # Helper to parse vzdump backup device hints
5926 # $rpcenv: Environment, used to ckeck storage permissions
5927 # $user: User ID, to check storage permissions
5928 # $storecfg: Storage configuration
5929 # $fh: the file handle for reading the configuration
5930 # $devinfo: should contain device sizes for all backu-up'ed devices
5931 # $options: backup options (pool, default storage)
5933 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5934 my $parse_backup_hints = sub {
5935 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5937 my $virtdev_hash = {};
5939 while (defined(my $line = <$fh>)) {
5940 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5941 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5942 die "archive does not contain data for drive '$virtdev'\n"
5943 if !$devinfo->{$devname};
5945 if (defined($options->{storage
})) {
5946 $storeid = $options->{storage
} || 'local';
5947 } elsif (!$storeid) {
5950 $format = 'raw' if !$format;
5951 $devinfo->{$devname}->{devname
} = $devname;
5952 $devinfo->{$devname}->{virtdev
} = $virtdev;
5953 $devinfo->{$devname}->{format
} = $format;
5954 $devinfo->{$devname}->{storeid
} = $storeid;
5956 # check permission on storage
5957 my $pool = $options->{pool
}; # todo: do we need that?
5958 if ($user ne 'root@pam') {
5959 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5962 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5963 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5965 my $drive = parse_drive
($virtdev, $2);
5966 if (drive_is_cloudinit
($drive)) {
5967 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
5968 $storeid = $options->{storage
} if defined ($options->{storage
});
5969 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5970 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
5972 $virtdev_hash->{$virtdev} = {
5974 storeid
=> $storeid,
5975 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
5982 return $virtdev_hash;
5985 # Helper to allocate and activate all volumes required for a restore
5987 # $storecfg: Storage configuration
5988 # $virtdev_hash: as returned by parse_backup_hints()
5990 # Returns: { $virtdev => $volid }
5991 my $restore_allocate_devices = sub {
5992 my ($storecfg, $virtdev_hash, $vmid) = @_;
5995 foreach my $virtdev (sort keys %$virtdev_hash) {
5996 my $d = $virtdev_hash->{$virtdev};
5997 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5998 my $storeid = $d->{storeid
};
5999 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6001 # test if requested format is supported
6002 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6003 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6004 $d->{format
} = $defFormat if !$supported;
6007 if ($d->{is_cloudinit
}) {
6008 $name = "vm-$vmid-cloudinit";
6009 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6010 if ($scfg->{path
}) {
6011 $name .= ".$d->{format}";
6015 my $volid = PVE
::Storage
::vdisk_alloc
(
6016 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6018 print STDERR
"new volume ID is '$volid'\n";
6019 $d->{volid
} = $volid;
6021 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6023 $map->{$virtdev} = $volid;
6029 sub restore_update_config_line
{
6030 my ($cookie, $map, $line, $unique) = @_;
6032 return '' if $line =~ m/^\#qmdump\#/;
6033 return '' if $line =~ m/^\#vzdump\#/;
6034 return '' if $line =~ m/^lock:/;
6035 return '' if $line =~ m/^unused\d+:/;
6036 return '' if $line =~ m/^parent:/;
6040 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6041 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6042 # try to convert old 1.X settings
6043 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6044 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6045 my ($model, $macaddr) = split(/\=/, $devconfig);
6046 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6049 bridge
=> "vmbr$ind",
6050 macaddr
=> $macaddr,
6052 my $netstr = print_net
($net);
6054 $res .= "net$cookie->{netcount}: $netstr\n";
6055 $cookie->{netcount
}++;
6057 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6058 my ($id, $netstr) = ($1, $2);
6059 my $net = parse_net
($netstr);
6060 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6061 $netstr = print_net
($net);
6062 $res .= "$id: $netstr\n";
6063 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6066 my $di = parse_drive
($virtdev, $value);
6067 if (defined($di->{backup
}) && !$di->{backup
}) {
6069 } elsif ($map->{$virtdev}) {
6070 delete $di->{format
}; # format can change on restore
6071 $di->{file
} = $map->{$virtdev};
6072 $value = print_drive
($di);
6073 $res .= "$virtdev: $value\n";
6077 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6079 if ($vmgenid ne '0') {
6080 # always generate a new vmgenid if there was a valid one setup
6081 $vmgenid = generate_uuid
();
6083 $res .= "vmgenid: $vmgenid\n";
6084 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6085 my ($uuid, $uuid_str);
6086 UUID
::generate
($uuid);
6087 UUID
::unparse
($uuid, $uuid_str);
6088 my $smbios1 = parse_smbios1
($2);
6089 $smbios1->{uuid
} = $uuid_str;
6090 $res .= $1.print_smbios1
($smbios1)."\n";
6098 my $restore_deactivate_volumes = sub {
6099 my ($storecfg, $devinfo) = @_;
6102 foreach my $devname (keys %$devinfo) {
6103 my $volid = $devinfo->{$devname}->{volid
};
6104 push @$vollist, $volid if $volid;
6107 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6110 my $restore_destroy_volumes = sub {
6111 my ($storecfg, $devinfo) = @_;
6113 foreach my $devname (keys %$devinfo) {
6114 my $volid = $devinfo->{$devname}->{volid
};
6117 if ($volid =~ m
|^/|) {
6118 unlink $volid || die 'unlink failed\n';
6120 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6122 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6124 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6129 my ($cfg, $vmid) = @_;
6131 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6133 my $volid_hash = {};
6134 foreach my $storeid (keys %$info) {
6135 foreach my $item (@{$info->{$storeid}}) {
6136 next if !($item->{volid
} && $item->{size
});
6137 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6138 $volid_hash->{$item->{volid
}} = $item;
6145 sub update_disk_config
{
6146 my ($vmid, $conf, $volid_hash) = @_;
6149 my $prefix = "VM $vmid";
6151 # used and unused disks
6152 my $referenced = {};
6154 # Note: it is allowed to define multiple storages with same path (alias), so
6155 # we need to check both 'volid' and real 'path' (two different volid can point
6156 # to the same path).
6158 my $referencedpath = {};
6161 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6162 my ($opt, $drive) = @_;
6164 my $volid = $drive->{file
};
6166 my $volume = $volid_hash->{$volid};
6168 # mark volid as "in-use" for next step
6169 $referenced->{$volid} = 1;
6170 if ($volume && (my $path = $volume->{path
})) {
6171 $referencedpath->{$path} = 1;
6174 return if drive_is_cdrom
($drive);
6177 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6178 if (defined($updated)) {
6180 $conf->{$opt} = print_drive
($updated);
6181 print "$prefix ($opt): $msg\n";
6185 # remove 'unusedX' entry if volume is used
6186 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6187 my ($opt, $drive) = @_;
6189 my $volid = $drive->{file
};
6193 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6194 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6195 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6197 delete $conf->{$opt};
6200 $referenced->{$volid} = 1;
6201 $referencedpath->{$path} = 1 if $path;
6204 foreach my $volid (sort keys %$volid_hash) {
6205 next if $volid =~ m/vm-$vmid-state-/;
6206 next if $referenced->{$volid};
6207 my $path = $volid_hash->{$volid}->{path
};
6208 next if !$path; # just to be sure
6209 next if $referencedpath->{$path};
6211 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6212 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6213 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6220 my ($vmid, $nolock, $dryrun) = @_;
6222 my $cfg = PVE
::Storage
::config
();
6224 print "rescan volumes...\n";
6225 my $volid_hash = scan_volids
($cfg, $vmid);
6227 my $updatefn = sub {
6230 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6232 PVE
::QemuConfig-
>check_lock($conf);
6235 foreach my $volid (keys %$volid_hash) {
6236 my $info = $volid_hash->{$volid};
6237 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6240 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6242 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6245 if (defined($vmid)) {
6249 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6252 my $vmlist = config_list
();
6253 foreach my $vmid (keys %$vmlist) {
6257 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6263 sub restore_proxmox_backup_archive
{
6264 my ($archive, $vmid, $user, $options) = @_;
6266 my $storecfg = PVE
::Storage
::config
();
6268 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6269 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6271 my $fingerprint = $scfg->{fingerprint
};
6272 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6274 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6276 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6277 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6278 local $ENV{PBS_PASSWORD
} = $password;
6279 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6281 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6282 PVE
::Storage
::parse_volname
($storecfg, $archive);
6284 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6286 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6288 my $tmpdir = "/var/tmp/vzdumptmp$$";
6292 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6293 # disable interrupts (always do cleanups)
6297 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6299 # Note: $oldconf is undef if VM does not exists
6300 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6301 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6302 my $new_conf_raw = '';
6304 my $rpcenv = PVE
::RPCEnvironment
::get
();
6313 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6315 my $cfgfn = "$tmpdir/qemu-server.conf";
6316 my $firewall_config_fn = "$tmpdir/fw.conf";
6317 my $index_fn = "$tmpdir/index.json";
6319 my $cmd = "restore";
6321 my $param = [$pbs_backup_name, "index.json", $index_fn];
6322 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6323 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6324 $index = decode_json
($index);
6326 # print Dumper($index);
6327 foreach my $info (@{$index->{files
}}) {
6328 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6330 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6331 $devinfo->{$devname}->{size
} = $1;
6333 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6338 my $is_qemu_server_backup = scalar(
6339 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6341 if (!$is_qemu_server_backup) {
6342 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6344 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6346 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6347 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6349 if ($has_firewall_config) {
6350 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6351 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6353 my $pve_firewall_dir = '/etc/pve/firewall';
6354 mkdir $pve_firewall_dir; # make sure the dir exists
6355 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6358 my $fh = IO
::File-
>new($cfgfn, "r") ||
6359 die "unable to read qemu-server.conf - $!\n";
6361 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6363 # fixme: rate limit?
6365 # create empty/temp config
6366 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6368 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6371 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6373 foreach my $virtdev (sort keys %$virtdev_hash) {
6374 my $d = $virtdev_hash->{$virtdev};
6375 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6377 # for live-restore we only want to preload the efidisk
6378 next if $options->{live
} && $virtdev ne 'efidisk0';
6380 my $volid = $d->{volid
};
6382 my $path = PVE
::Storage
::path
($storecfg, $volid);
6384 my $pbs_restore_cmd = [
6385 '/usr/bin/pbs-restore',
6386 '--repository', $repo,
6388 "$d->{devname}.img.fidx",
6393 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6394 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6396 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6397 push @$pbs_restore_cmd, '--skip-zero';
6400 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6401 print "restore proxmox backup image: $dbg_cmdstring\n";
6402 run_command
($pbs_restore_cmd);
6405 $fh->seek(0, 0) || die "seek failed - $!\n";
6407 my $cookie = { netcount
=> 0 };
6408 while (defined(my $line = <$fh>)) {
6409 $new_conf_raw .= restore_update_config_line
(
6421 if ($err || !$options->{live
}) {
6422 $restore_deactivate_volumes->($storecfg, $devinfo);
6428 $restore_destroy_volumes->($storecfg, $devinfo);
6432 if ($options->{live
}) {
6433 # keep lock during live-restore
6434 $new_conf_raw .= "\nlock: create";
6437 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6439 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6441 eval { rescan
($vmid, 1); };
6444 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
6446 if ($options->{live
}) {
6452 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
6454 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6455 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
6457 delete $devinfo->{'drive-efidisk0'}; # this special drive is already restored before start
6458 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6460 PVE
::QemuConfig-
>remove_lock($vmid, "create");
6464 sub pbs_live_restore
{
6465 my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6467 print "starting VM for live-restore\n";
6468 print "repository: '$repo', snapshot: '$snap'\n";
6470 my $pbs_backing = {};
6471 for my $ds (keys %$restored_disks) {
6472 $ds =~ m/^drive-(.*)$/;
6474 $pbs_backing->{$confname} = {
6475 repository
=> $repo,
6477 archive
=> "$ds.img.fidx",
6479 $pbs_backing->{$confname}->{keyfile
} = $keyfile if -e
$keyfile;
6481 my $drive = parse_drive
($confname, $conf->{$confname});
6482 print "restoring '$ds' to '$drive->{file}'\n";
6485 my $drives_streamed = 0;
6487 # make sure HA doesn't interrupt our restore by stopping the VM
6488 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
6489 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
6492 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6493 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6494 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'pbs-backing' => $pbs_backing}, {});
6496 my $qmeventd_fd = register_qmeventd_handle
($vmid);
6498 # begin streaming, i.e. data copy from PBS to target disk for every vol,
6499 # this will effectively collapse the backing image chain consisting of
6500 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6501 # removes itself once all backing images vanish with 'auto-remove=on')
6503 for my $ds (sort keys %$restored_disks) {
6504 my $job_id = "restore-$ds";
6505 mon_cmd
($vmid, 'block-stream',
6506 'job-id' => $job_id,
6509 $jobs->{$job_id} = {};
6512 mon_cmd
($vmid, 'cont');
6513 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
6515 print "restore-drive jobs finished successfully, removing all tracking block devices"
6516 ." to disconnect from Proxmox Backup Server\n";
6518 for my $ds (sort keys %$restored_disks) {
6519 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6522 close($qmeventd_fd);
6528 warn "An error occured during live-restore: $err\n";
6529 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
6530 die "live-restore failed\n";
6534 sub restore_vma_archive
{
6535 my ($archive, $vmid, $user, $opts, $comp) = @_;
6537 my $readfrom = $archive;
6539 my $cfg = PVE
::Storage
::config
();
6541 my $bwlimit = $opts->{bwlimit
};
6543 my $dbg_cmdstring = '';
6544 my $add_pipe = sub {
6546 push @$commands, $cmd;
6547 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6548 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6553 if ($archive eq '-') {
6556 # If we use a backup from a PVE defined storage we also consider that
6557 # storage's rate limit:
6558 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6559 if (defined($volid)) {
6560 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6561 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6563 print STDERR
"applying read rate limit: $readlimit\n";
6564 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6565 $add_pipe->($cstream);
6571 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6572 my $cmd = $info->{decompressor
};
6573 push @$cmd, $readfrom;
6577 my $tmpdir = "/var/tmp/vzdumptmp$$";
6580 # disable interrupts (always do cleanups)
6584 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6586 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6587 POSIX
::mkfifo
($mapfifo, 0600);
6589 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6591 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6598 my $rpcenv = PVE
::RPCEnvironment
::get
();
6600 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6602 # Note: $oldconf is undef if VM does not exist
6603 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6604 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6605 my $new_conf_raw = '';
6609 my $print_devmap = sub {
6610 my $cfgfn = "$tmpdir/qemu-server.conf";
6612 # we can read the config - that is already extracted
6613 my $fh = IO
::File-
>new($cfgfn, "r") ||
6614 die "unable to read qemu-server.conf - $!\n";
6616 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6618 my $pve_firewall_dir = '/etc/pve/firewall';
6619 mkdir $pve_firewall_dir; # make sure the dir exists
6620 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6623 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6625 foreach my $info (values %{$virtdev_hash}) {
6626 my $storeid = $info->{storeid
};
6627 next if defined($storage_limits{$storeid});
6629 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
6630 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6631 $storage_limits{$storeid} = $limit * 1024;
6634 foreach my $devname (keys %$devinfo) {
6635 die "found no device mapping information for device '$devname'\n"
6636 if !$devinfo->{$devname}->{virtdev
};
6639 # create empty/temp config
6641 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6642 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6646 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6648 # print restore information to $fifofh
6649 foreach my $virtdev (sort keys %$virtdev_hash) {
6650 my $d = $virtdev_hash->{$virtdev};
6651 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6653 my $storeid = $d->{storeid
};
6654 my $volid = $d->{volid
};
6657 if (my $limit = $storage_limits{$storeid}) {
6658 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6661 my $write_zeros = 1;
6662 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6666 my $path = PVE
::Storage
::path
($cfg, $volid);
6668 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6670 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6673 $fh->seek(0, 0) || die "seek failed - $!\n";
6675 my $cookie = { netcount
=> 0 };
6676 while (defined(my $line = <$fh>)) {
6677 $new_conf_raw .= restore_update_config_line
(
6694 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6695 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6697 $oldtimeout = alarm($timeout);
6704 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6705 my ($dev_id, $size, $devname) = ($1, $2, $3);
6706 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6707 } elsif ($line =~ m/^CTIME: /) {
6708 # we correctly received the vma config, so we can disable
6709 # the timeout now for disk allocation (set to 10 minutes, so
6710 # that we always timeout if something goes wrong)
6713 print $fifofh "done\n";
6714 my $tmp = $oldtimeout || 0;
6715 $oldtimeout = undef;
6722 print "restore vma archive: $dbg_cmdstring\n";
6723 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6727 alarm($oldtimeout) if $oldtimeout;
6729 $restore_deactivate_volumes->($cfg, $devinfo);
6731 close($fifofh) if $fifofh;
6736 $restore_destroy_volumes->($cfg, $devinfo);
6740 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6742 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6744 eval { rescan
($vmid, 1); };
6747 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
6750 sub restore_tar_archive
{
6751 my ($archive, $vmid, $user, $opts) = @_;
6753 if ($archive ne '-') {
6754 my $firstfile = tar_archive_read_firstfile
($archive);
6755 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6756 if $firstfile ne 'qemu-server.conf';
6759 my $storecfg = PVE
::Storage
::config
();
6761 # avoid zombie disks when restoring over an existing VM -> cleanup first
6762 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6763 # skiplock=1 because qmrestore has set the 'create' lock itself already
6764 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6765 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6767 my $tocmd = "/usr/lib/qemu-server/qmextract";
6769 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6770 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6771 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6772 $tocmd .= ' --info' if $opts->{info
};
6774 # tar option "xf" does not autodetect compression when read from STDIN,
6775 # so we pipe to zcat
6776 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6777 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6779 my $tmpdir = "/var/tmp/vzdumptmp$$";
6782 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6783 local $ENV{VZDUMP_VMID
} = $vmid;
6784 local $ENV{VZDUMP_USER
} = $user;
6786 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6787 my $new_conf_raw = '';
6789 # disable interrupts (always do cleanups)
6793 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6801 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6803 if ($archive eq '-') {
6804 print "extracting archive from STDIN\n";
6805 run_command
($cmd, input
=> "<&STDIN");
6807 print "extracting archive '$archive'\n";
6811 return if $opts->{info
};
6815 my $statfile = "$tmpdir/qmrestore.stat";
6816 if (my $fd = IO
::File-
>new($statfile, "r")) {
6817 while (defined (my $line = <$fd>)) {
6818 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6819 $map->{$1} = $2 if $1;
6821 print STDERR
"unable to parse line in statfile - $line\n";
6827 my $confsrc = "$tmpdir/qemu-server.conf";
6829 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6831 my $cookie = { netcount
=> 0 };
6832 while (defined (my $line = <$srcfd>)) {
6833 $new_conf_raw .= restore_update_config_line
(
6844 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6850 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6852 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6854 eval { rescan
($vmid, 1); };
6858 sub foreach_storage_used_by_vm
{
6859 my ($conf, $func) = @_;
6863 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6864 my ($ds, $drive) = @_;
6865 return if drive_is_cdrom
($drive);
6867 my $volid = $drive->{file
};
6869 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6870 $sidhash->{$sid} = $sid if $sid;
6873 foreach my $sid (sort keys %$sidhash) {
6878 my $qemu_snap_storage = {
6881 sub do_snapshots_with_qemu
{
6882 my ($storecfg, $volid) = @_;
6884 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6885 my $scfg = $storecfg->{ids
}->{$storage_name};
6886 die "could not find storage '$storage_name'\n" if !defined($scfg);
6888 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6892 if ($volid =~ m/\.(qcow2|qed)$/){
6899 sub qga_check_running
{
6900 my ($vmid, $nowarn) = @_;
6902 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6904 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6910 sub template_create
{
6911 my ($vmid, $conf, $disk) = @_;
6913 my $storecfg = PVE
::Storage
::config
();
6915 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6916 my ($ds, $drive) = @_;
6918 return if drive_is_cdrom
($drive);
6919 return if $disk && $ds ne $disk;
6921 my $volid = $drive->{file
};
6922 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6924 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6925 $drive->{file
} = $voliddst;
6926 $conf->{$ds} = print_drive
($drive);
6927 PVE
::QemuConfig-
>write_config($vmid, $conf);
6931 sub convert_iscsi_path
{
6934 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6939 my $initiator_name = get_initiator_name
();
6941 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6942 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6945 die "cannot convert iscsi path '$path', unkown format\n";
6948 sub qemu_img_convert
{
6949 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6951 my $storecfg = PVE
::Storage
::config
();
6952 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6953 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6955 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6959 my $src_is_iscsi = 0;
6963 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6964 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6965 $src_format = qemu_img_format
($src_scfg, $src_volname);
6966 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6967 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6968 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6969 } elsif (-f
$src_volid) {
6970 $src_path = $src_volid;
6971 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6976 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6978 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6979 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6980 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6981 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6984 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6985 push @$cmd, '-l', "snapshot.name=$snapname"
6986 if $snapname && $src_format && $src_format eq "qcow2";
6987 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6988 push @$cmd, '-T', $cachemode if defined($cachemode);
6990 if ($src_is_iscsi) {
6991 push @$cmd, '--image-opts';
6992 $src_path = convert_iscsi_path
($src_path);
6993 } elsif ($src_format) {
6994 push @$cmd, '-f', $src_format;
6997 if ($dst_is_iscsi) {
6998 push @$cmd, '--target-image-opts';
6999 $dst_path = convert_iscsi_path
($dst_path);
7001 push @$cmd, '-O', $dst_format;
7004 push @$cmd, $src_path;
7006 if (!$dst_is_iscsi && $is_zero_initialized) {
7007 push @$cmd, "zeroinit:$dst_path";
7009 push @$cmd, $dst_path;
7014 if($line =~ m/\((\S+)\/100\
%\)/){
7016 my $transferred = int($size * $percent / 100);
7017 my $total_h = render_bytes
($size, 1);
7018 my $transferred_h = render_bytes
($transferred, 1);
7020 print "transferred $transferred_h of $total_h ($percent%)\n";
7025 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7027 die "copy failed: $err" if $err;
7030 sub qemu_img_format
{
7031 my ($scfg, $volname) = @_;
7033 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7040 sub qemu_drive_mirror
{
7041 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7043 $jobs = {} if !$jobs;
7047 $jobs->{"drive-$drive"} = {};
7049 if ($dst_volid =~ /^nbd:/) {
7050 $qemu_target = $dst_volid;
7053 my $storecfg = PVE
::Storage
::config
();
7054 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7056 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7058 $format = qemu_img_format
($dst_scfg, $dst_volname);
7060 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7062 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7065 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7066 $opts->{format
} = $format if $format;
7068 if (defined($src_bitmap)) {
7069 $opts->{sync
} = 'incremental';
7070 $opts->{bitmap
} = $src_bitmap;
7071 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7074 if (defined($bwlimit)) {
7075 $opts->{speed
} = $bwlimit * 1024;
7076 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7078 print "drive mirror is starting for drive-$drive\n";
7081 # if a job already runs for this device we get an error, catch it for cleanup
7082 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7084 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7086 die "mirroring error: $err\n";
7089 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7092 # $completion can be either
7093 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7094 # 'cancel': wait until all jobs are ready, block-job-cancel them
7095 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7096 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7097 sub qemu_drive_mirror_monitor
{
7098 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7100 $completion //= 'complete';
7104 my $err_complete = 0;
7106 my $starttime = time ();
7108 die "block job ('$op') timed out\n" if $err_complete > 300;
7110 my $stats = mon_cmd
($vmid, "query-block-jobs");
7113 my $running_jobs = {};
7114 for my $stat (@$stats) {
7115 next if $stat->{type
} ne $op;
7116 $running_jobs->{$stat->{device
}} = $stat;
7119 my $readycounter = 0;
7121 for my $job_id (sort keys %$jobs) {
7122 my $job = $running_jobs->{$job_id};
7124 my $vanished = !defined($job);
7125 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7126 if($complete || ($vanished && $completion eq 'auto')) {
7127 print "$job_id: $op-job finished\n";
7128 delete $jobs->{$job_id};
7132 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7134 my $busy = $job->{busy
};
7135 my $ready = $job->{ready
};
7136 if (my $total = $job->{len
}) {
7137 my $transferred = $job->{offset
} || 0;
7138 my $remaining = $total - $transferred;
7139 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7141 my $duration = $ctime - $starttime;
7142 my $total_h = render_bytes
($total, 1);
7143 my $transferred_h = render_bytes
($transferred, 1);
7145 my $status = sprintf(
7146 "transferred $transferred_h of $total_h ($percent%%) in %s",
7147 render_duration
($duration),
7152 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7154 $status .= ", ready";
7157 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
7158 $jobs->{$job_id}->{ready
} = $ready;
7161 $readycounter++ if $job->{ready
};
7164 last if scalar(keys %$jobs) == 0;
7166 if ($readycounter == scalar(keys %$jobs)) {
7167 print "all '$op' jobs are ready\n";
7169 # do the complete later (or has already been done)
7170 last if $completion eq 'skip' || $completion eq 'auto';
7172 if ($vmiddst && $vmiddst != $vmid) {
7173 my $agent_running = $qga && qga_check_running
($vmid);
7174 if ($agent_running) {
7175 print "freeze filesystem\n";
7176 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
7178 print "suspend vm\n";
7179 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
7182 # if we clone a disk for a new target vm, we don't switch the disk
7183 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
7185 if ($agent_running) {
7186 print "unfreeze filesystem\n";
7187 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
7189 print "resume vm\n";
7190 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
7196 for my $job_id (sort keys %$jobs) {
7197 # try to switch the disk if source and destination are on the same guest
7198 print "$job_id: Completing block job_id...\n";
7201 if ($completion eq 'complete') {
7202 $op = 'block-job-complete';
7203 } elsif ($completion eq 'cancel') {
7204 $op = 'block-job-cancel';
7206 die "invalid completion value: $completion\n";
7208 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
7209 if ($@ =~ m/cannot be completed/) {
7210 print "$job_id: block job cannot be completed, trying again.\n";
7213 print "$job_id: Completed successfully.\n";
7214 $jobs->{$job_id}->{complete
} = 1;
7225 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7226 die "block job ($op) error: $err";
7230 sub qemu_blockjobs_cancel
{
7231 my ($vmid, $jobs) = @_;
7233 foreach my $job (keys %$jobs) {
7234 print "$job: Cancelling block job\n";
7235 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
7236 $jobs->{$job}->{cancel
} = 1;
7240 my $stats = mon_cmd
($vmid, "query-block-jobs");
7242 my $running_jobs = {};
7243 foreach my $stat (@$stats) {
7244 $running_jobs->{$stat->{device
}} = $stat;
7247 foreach my $job (keys %$jobs) {
7249 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
7250 print "$job: Done.\n";
7251 delete $jobs->{$job};
7255 last if scalar(keys %$jobs) == 0;
7262 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7263 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7268 print "create linked clone of drive $drivename ($drive->{file})\n";
7269 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
7270 push @$newvollist, $newvolid;
7273 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
7274 $storeid = $storage if $storage;
7276 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
7278 print "create full clone of drive $drivename ($drive->{file})\n";
7281 if (drive_is_cloudinit
($drive)) {
7282 $name = "vm-$newvmid-cloudinit";
7283 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7284 if ($scfg->{path
}) {
7285 $name .= ".$dst_format";
7288 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
7289 } elsif ($drivename eq 'efidisk0') {
7290 $size = get_efivars_size
($conf);
7292 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
7294 $newvolid = PVE
::Storage
::vdisk_alloc
(
7295 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7297 push @$newvollist, $newvolid;
7299 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
7301 if (drive_is_cloudinit
($drive)) {
7302 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7303 # if this is the case, we have to complete any block-jobs still there from
7304 # previous drive-mirrors
7305 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7306 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
7311 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
7312 if (!$running || $snapname) {
7313 # TODO: handle bwlimits
7314 if ($drivename eq 'efidisk0') {
7315 # the relevant data on the efidisk may be smaller than the source
7316 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7317 # that is given by the OVMF_VARS.fd
7318 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
7319 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
7321 # better for Ceph if block size is not too small, see bug #3324
7324 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7325 "if=$src_path", "of=$dst_path"]);
7327 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
7331 my $kvmver = get_running_qemu_version
($vmid);
7332 if (!min_version
($kvmver, 2, 7)) {
7333 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7334 if $drive->{iothread
};
7337 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7338 $completion, $qga, $bwlimit);
7343 my ($size) = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
7346 $disk->{format
} = undef;
7347 $disk->{file
} = $newvolid;
7348 $disk->{size
} = $size if defined($size);
7353 sub get_running_qemu_version
{
7355 my $res = mon_cmd
($vmid, "query-version");
7356 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7359 sub qemu_use_old_bios_files
{
7360 my ($machine_type) = @_;
7362 return if !$machine_type;
7364 my $use_old_bios_files = undef;
7366 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7368 $use_old_bios_files = 1;
7370 my $version = extract_version
($machine_type, kvm_user_version
());
7371 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7372 # load new efi bios files on migration. So this hack is required to allow
7373 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7374 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7375 $use_old_bios_files = !min_version
($version, 2, 4);
7378 return ($use_old_bios_files, $machine_type);
7381 sub get_efivars_size
{
7383 my $arch = get_vm_arch
($conf);
7384 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7385 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
7386 return -s
$ovmf_vars;
7389 sub update_efidisk_size
{
7392 return if !defined($conf->{efidisk0
});
7394 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
7395 $disk->{size
} = get_efivars_size
($conf);
7396 $conf->{efidisk0
} = print_drive
($disk);
7401 sub create_efidisk
($$$$$) {
7402 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7404 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7405 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7407 my $vars_size_b = -s
$ovmf_vars;
7408 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7409 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7410 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7412 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7413 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7415 return ($volid, $size/1024);
7418 sub vm_iothreads_list
{
7421 my $res = mon_cmd
($vmid, 'query-iothreads');
7424 foreach my $iothread (@$res) {
7425 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7432 my ($conf, $drive) = @_;
7436 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7438 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7444 my $controller = int($drive->{index} / $maxdev);
7445 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
7449 return ($maxdev, $controller, $controller_prefix);
7452 sub windows_version
{
7455 return 0 if !$ostype;
7459 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7461 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7463 } elsif ($ostype =~ m/^win(\d+)$/) {
7470 sub resolve_dst_disk_format
{
7471 my ($storecfg, $storeid, $src_volname, $format) = @_;
7472 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7475 # if no target format is specified, use the source disk format as hint
7477 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7478 $format = qemu_img_format
($scfg, $src_volname);
7484 # test if requested format is supported - else use default
7485 my $supported = grep { $_ eq $format } @$validFormats;
7486 $format = $defFormat if !$supported;
7490 # NOTE: if this logic changes, please update docs & possibly gui logic
7491 sub find_vmstate_storage
{
7492 my ($conf, $storecfg) = @_;
7494 # first, return storage from conf if set
7495 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7497 my ($target, $shared, $local);
7499 foreach_storage_used_by_vm
($conf, sub {
7501 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7502 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7503 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7506 # second, use shared storage where VM has at least one disk
7507 # third, use local storage where VM has at least one disk
7508 # fall back to local storage
7509 $target = $shared // $local // 'local';
7515 my ($uuid, $uuid_str);
7516 UUID
::generate
($uuid);
7517 UUID
::unparse
($uuid, $uuid_str);
7521 sub generate_smbios1_uuid
{
7522 return "uuid=".generate_uuid
();
7528 mon_cmd
($vmid, 'nbd-server-stop');
7531 sub create_reboot_request
{
7533 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7534 or die "failed to create reboot trigger file: $!\n";
7538 sub clear_reboot_request
{
7540 my $path = "/run/qemu-server/$vmid.reboot";
7543 $res = unlink($path);
7544 die "could not remove reboot request for $vmid: $!"
7545 if !$res && $! != POSIX
::ENOENT
;
7550 sub bootorder_from_legacy
{
7551 my ($conf, $bootcfg) = @_;
7553 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
7554 my $bootindex_hash = {};
7556 foreach my $o (split(//, $boot)) {
7557 $bootindex_hash->{$o} = $i*100;
7563 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7564 my ($ds, $drive) = @_;
7566 if (drive_is_cdrom
($drive, 1)) {
7567 if ($bootindex_hash->{d
}) {
7568 $bootorder->{$ds} = $bootindex_hash->{d
};
7569 $bootindex_hash->{d
} += 1;
7571 } elsif ($bootindex_hash->{c
}) {
7572 $bootorder->{$ds} = $bootindex_hash->{c
}
7573 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
7574 $bootindex_hash->{c
} += 1;
7578 if ($bootindex_hash->{n
}) {
7579 for (my $i = 0; $i < $MAX_NETS; $i++) {
7580 my $netname = "net$i";
7581 next if !$conf->{$netname};
7582 $bootorder->{$netname} = $bootindex_hash->{n
};
7583 $bootindex_hash->{n
} += 1;
7590 # Generate default device list for 'boot: order=' property. Matches legacy
7591 # default boot order, but with explicit device names. This is important, since
7592 # the fallback for when neither 'order' nor the old format is specified relies
7593 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7594 sub get_default_bootdevices
{
7600 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
7601 push @ret, $first if $first;
7604 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
7605 push @ret, $first if $first;
7608 for (my $i = 0; $i < $MAX_NETS; $i++) {
7609 my $netname = "net$i";
7610 next if !$conf->{$netname};
7611 push @ret, $netname;
7618 sub device_bootorder
{
7621 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
7623 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
7626 if (!defined($boot) || $boot->{legacy
}) {
7627 $bootorder = bootorder_from_legacy
($conf, $boot);
7628 } elsif ($boot->{order
}) {
7629 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7630 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
7631 $bootorder->{$dev} = $i++;
7638 sub register_qmeventd_handle
{
7642 my $peer = "/var/run/qmeventd.sock";
7647 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
7649 if ($! != EINTR
&& $! != EAGAIN
) {
7650 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7653 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7654 . "after $count retries\n";
7659 # send handshake to mark VM as backing up
7660 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
7662 # return handle to be closed later when inhibit is no longer required
7666 # bash completion helper
7668 sub complete_backup_archives
{
7669 my ($cmdname, $pname, $cvalue) = @_;
7671 my $cfg = PVE
::Storage
::config
();
7675 if ($cvalue =~ m/^([^:]+):/) {
7679 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7682 foreach my $id (keys %$data) {
7683 foreach my $item (@{$data->{$id}}) {
7684 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7685 push @$res, $item->{volid
} if defined($item->{volid
});
7692 my $complete_vmid_full = sub {
7695 my $idlist = vmstatus
();
7699 foreach my $id (keys %$idlist) {
7700 my $d = $idlist->{$id};
7701 if (defined($running)) {
7702 next if $d->{template
};
7703 next if $running && $d->{status
} ne 'running';
7704 next if !$running && $d->{status
} eq 'running';
7713 return &$complete_vmid_full();
7716 sub complete_vmid_stopped
{
7717 return &$complete_vmid_full(0);
7720 sub complete_vmid_running
{
7721 return &$complete_vmid_full(1);
7724 sub complete_storage
{
7726 my $cfg = PVE
::Storage
::config
();
7727 my $ids = $cfg->{ids
};
7730 foreach my $sid (keys %$ids) {
7731 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7732 next if !$ids->{$sid}->{content
}->{images
};
7739 sub complete_migration_storage
{
7740 my ($cmd, $param, $current_value, $all_args) = @_;
7742 my $targetnode = @$all_args[1];
7744 my $cfg = PVE
::Storage
::config
();
7745 my $ids = $cfg->{ids
};
7748 foreach my $sid (keys %$ids) {
7749 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7750 next if !$ids->{$sid}->{content
}->{images
};
7759 my $qmpstatus = eval {
7760 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
7761 mon_cmd
($vmid, "query-status");
7764 return $qmpstatus && $qmpstatus->{status
} eq "paused";
7767 sub check_volume_storage_type
{
7768 my ($storecfg, $vol) = @_;
7770 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
7771 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7772 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
7774 die "storage '$storeid' does not support content-type '$vtype'\n"
7775 if !$scfg->{content
}->{$vtype};