1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday usleep);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
31 use PVE
::DataCenterConfig
;
32 use PVE
::Exception
qw(raise raise_param_exc);
33 use PVE
::Format
qw(render_duration render_bytes);
34 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
36 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
39 use PVE
::RPCEnvironment
;
43 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
47 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
48 use PVE
::QemuServer
::Cloudinit
;
49 use PVE
::QemuServer
::CGroup
;
50 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
51 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
52 use PVE
::QemuServer
::Machine
;
53 use PVE
::QemuServer
::Memory
;
54 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
55 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
56 use PVE
::QemuServer
::USB
qw(parse_usb_device);
60 require PVE
::Network
::SDN
::Zones
;
64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
68 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
69 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
72 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
73 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
76 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
77 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
80 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
81 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
84 "$EDK2_FW_BASE/OVMF_CODE.fd",
85 "$EDK2_FW_BASE/OVMF_VARS.fd",
90 "$EDK2_FW_BASE/AAVMF_CODE.fd",
91 "$EDK2_FW_BASE/AAVMF_VARS.fd",
96 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
98 # Note about locking: we use flock on the config file protect against concurent actions.
99 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
101 # But you can ignore this kind of lock with the --skiplock flag.
103 cfs_register_file
('/qemu-server/',
107 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
108 description
=> "Some command save/restore state from this location.",
114 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
115 description
=> "Specifies the Qemu machine type.",
117 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
124 my ($map, $source) = @_;
126 return $source if !defined($map);
128 return $map->{entries
}->{$source}
129 if $map->{entries
} && defined($map->{entries
}->{$source});
131 return $map->{default} if $map->{default};
133 # identity (fallback)
137 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
138 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
140 format
=> 'storagepair-list',
144 #no warnings 'redefine';
148 $nodename_cache //= PVE
::INotify
::nodename
();
149 return $nodename_cache;
156 enum
=> [qw(i6300esb ib700)],
157 description
=> "Watchdog type to emulate.",
158 default => 'i6300esb',
163 enum
=> [qw(reset shutdown poweroff pause debug none)],
164 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
168 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
172 description
=> "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
177 fstrim_cloned_disks
=> {
178 description
=> "Run fstrim after moving a disk or migrating the VM.",
184 description
=> "Select the agent type",
188 enum
=> [qw(virtio isa)],
194 description
=> "Select the VGA type.",
199 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
202 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
214 description
=> "The size of the file in MB.",
218 pattern
=> '[a-zA-Z0-9\-]+',
220 format_description
=> 'string',
221 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
228 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
229 description
=> "Configure an audio device."
233 enum
=> ['spice', 'none'],
236 description
=> "Driver backend for the audio device."
240 my $spice_enhancements_fmt = {
245 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
249 enum
=> ['off', 'all', 'filter'],
252 description
=> "Enable video streaming. Uses compression for detected video streams."
259 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
261 description
=> "The file on the host to gather entropy from. In most cases '/dev/urandom'"
262 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
263 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
264 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
265 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
266 ." a hardware RNG from the host.",
270 description
=> "Maximum bytes of entropy allowed to get injected into the guest every"
271 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
272 ." `0` to disable limiting (potentially dangerous!).",
275 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
276 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
277 # reading from /dev/urandom
282 description
=> "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
283 ." the guest to retrieve another 'max_bytes' of entropy.",
289 my $meta_info_fmt = {
292 description
=> "The guest creation timestamp as UNIX epoch time",
298 description
=> "The QEMU (machine) version from the time this VM was created.",
299 pattern
=> '\d+(\.\d+)+',
308 description
=> "Specifies whether a VM will be started during system bootup.",
314 description
=> "Automatic restart after crash (currently ignored).",
319 type
=> 'string', format
=> 'pve-hotplug-features',
320 description
=> "Selectively enable hotplug features. This is a comma separated list of"
321 ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
322 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
323 default => 'network,disk,usb',
328 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
334 description
=> "Lock/unlock the VM.",
335 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
340 description
=> "Limit of CPU usage.",
341 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
342 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
350 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
351 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
352 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
353 ." weights of all the other running VMs.",
356 default => 'cgroup v1: 1024, cgroup v2: 100',
361 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when"
362 ." you use the balloon device.",
369 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
375 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the"
376 ." more memory this VM gets. Number is relative to weights of all other running VMs."
377 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
385 description
=> "Keyboard layout for VNC server. The default is read from the"
386 ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
387 enum
=> PVE
::Tools
::kvmkeymaplist
(),
392 type
=> 'string', format
=> 'dns-name',
393 description
=> "Set a name for the VM. Only used on the configuration web interface.",
398 description
=> "SCSI controller model",
399 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
405 description
=> "Description for the VM. Shown in the web-interface VM's summary."
406 ." This is saved as comment inside the configuration file.",
407 maxLength
=> 1024 * 8,
412 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
413 description
=> "Specify guest operating system.",
414 verbose_description
=> <<EODESC,
415 Specify guest operating system. This is used to enable special
416 optimization/features for specific operating systems:
419 other;; unspecified OS
420 wxp;; Microsoft Windows XP
421 w2k;; Microsoft Windows 2000
422 w2k3;; Microsoft Windows 2003
423 w2k8;; Microsoft Windows 2008
424 wvista;; Microsoft Windows Vista
425 win7;; Microsoft Windows 7
426 win8;; Microsoft Windows 8/2012/2012r2
427 win10;; Microsoft Windows 10/2016/2019
428 win11;; Microsoft Windows 11/2022
429 l24;; Linux 2.4 Kernel
430 l26;; Linux 2.6 - 5.X Kernel
431 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
436 type
=> 'string', format
=> 'pve-qm-boot',
437 description
=> "Specify guest boot order. Use the 'order=' sub-property as usage with no"
438 ." key or 'legacy=' is deprecated.",
442 type
=> 'string', format
=> 'pve-qm-bootdisk',
443 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
444 pattern
=> '(ide|sata|scsi|virtio)\d+',
449 description
=> "The number of CPUs. Please use option -sockets instead.",
456 description
=> "The number of CPU sockets.",
463 description
=> "The number of cores per socket.",
470 description
=> "Enable/disable NUMA.",
476 description
=> "Enable/disable hugepages memory.",
477 enum
=> [qw(any 2 1024)],
483 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
484 ." after VM shutdown and can be used for subsequent starts.",
489 description
=> "Number of hotplugged vcpus.",
496 description
=> "Enable/disable ACPI.",
501 description
=> "Enable/disable communication with the Qemu Guest Agent and its properties.",
503 format
=> $agent_fmt,
508 description
=> "Enable/disable KVM hardware virtualization.",
514 description
=> "Enable/disable time drift fix.",
520 description
=> "Set the real time clock (RTC) to local time. This is enabled by default if"
521 ." the `ostype` indicates a Microsoft Windows OS.",
526 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
530 type
=> 'string', format
=> $vga_fmt,
531 description
=> "Configure the VGA hardware.",
532 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
533 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
534 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
535 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
536 ." display server. For win* OS you can select how many independent displays you want,"
537 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
538 ." using a serial device as terminal.",
542 type
=> 'string', format
=> 'pve-qm-watchdog',
543 description
=> "Create a virtual hardware watchdog device.",
544 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
545 ." action), the watchdog must be periodically polled by an agent inside the guest or"
546 ." else the watchdog will reset the guest (or execute the respective action specified)",
551 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
552 description
=> "Set the initial date of the real time clock. Valid format for date are:"
553 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
554 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
557 startup
=> get_standard_option
('pve-startup-order'),
561 description
=> "Enable/disable Template.",
567 description
=> "Arbitrary arguments passed to kvm.",
568 verbose_description
=> <<EODESCR,
569 Arbitrary arguments passed to kvm, for example:
571 args: -no-reboot -no-hpet
573 NOTE: this option is for experts only.
580 description
=> "Enable/disable the USB tablet device.",
581 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
582 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
583 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
584 ." may consider disabling this to save some context switches. This is turned off by"
585 ." default if you use spice (`qm set <vmid> --vga qxl`).",
590 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
594 migrate_downtime
=> {
597 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
603 type
=> 'string', format
=> 'pve-qm-ide',
604 typetext
=> '<volume>',
605 description
=> "This is an alias for option -ide2",
609 description
=> "Emulated CPU type.",
611 format
=> 'pve-vm-cpu-conf',
613 parent
=> get_standard_option
('pve-snapshot-name', {
615 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
619 description
=> "Timestamp for snapshots.",
625 type
=> 'string', format
=> 'pve-volume-id',
626 description
=> "Reference to a volume which stores the VM state. This is used internally"
629 vmstatestorage
=> get_standard_option
('pve-storage-id', {
630 description
=> "Default storage for VM state volumes/files.",
633 runningmachine
=> get_standard_option
('pve-qemu-machine', {
634 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
638 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
639 ." internally for snapshots.",
642 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
643 format_description
=> 'QEMU -cpu parameter'
645 machine
=> get_standard_option
('pve-qemu-machine'),
647 description
=> "Virtual processor architecture. Defaults to the host.",
650 enum
=> [qw(x86_64 aarch64)],
653 description
=> "Specify SMBIOS type 1 fields.",
654 type
=> 'string', format
=> 'pve-qm-smbios1',
661 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
662 ." remove disk operations.",
668 enum
=> [ qw(seabios ovmf) ],
669 description
=> "Select BIOS implementation.",
670 default => 'seabios',
674 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
675 format_description
=> 'UUID',
676 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
677 ." to disable explicitly.",
678 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
679 ." value identifier to the guest OS. This allows to notify the guest operating system"
680 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
681 ." execution or creation from a template). The guest operating system notices the"
682 ." change, and is then able to react as appropriate by marking its copies of"
683 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
684 ."Note that auto-creation only works when done through API/CLI create or update methods"
685 .", but not when manually editing the config file.",
686 default => "1 (autogenerated)",
691 format
=> 'pve-volume-id',
693 description
=> "Script that will be executed during various steps in the vms lifetime.",
697 format
=> $ivshmem_fmt,
698 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
704 format
=> $audio_fmt,
705 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
708 spice_enhancements
=> {
710 format
=> $spice_enhancements_fmt,
711 description
=> "Configure additional enhancements for SPICE.",
715 type
=> 'string', format
=> 'pve-tag-list',
716 description
=> 'Tags of the VM. This is only meta information.',
722 description
=> "Configure a VirtIO-based Random Number Generator.",
727 format
=> $meta_info_fmt,
728 description
=> "Some (read-only) meta-information about this guest.",
737 description
=> 'Specify a custom file containing all meta data passed to the VM via"
738 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
739 format
=> 'pve-volume-id',
740 format_description
=> 'volume',
745 description
=> 'Specify a custom file containing all network data passed to the VM via'
747 format
=> 'pve-volume-id',
748 format_description
=> 'volume',
753 description
=> 'Specify a custom file containing all user data passed to the VM via'
755 format
=> 'pve-volume-id',
756 format_description
=> 'volume',
761 description
=> 'Specify a custom file containing all vendor data passed to the VM via'
763 format
=> 'pve-volume-id',
764 format_description
=> 'volume',
767 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
769 my $confdesc_cloudinit = {
773 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
774 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
775 .' and `configdrive2` for windows.',
776 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
781 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
782 ." image's configured default user.",
787 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
788 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
789 .' support hashed passwords.',
794 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
796 format
=> 'pve-qm-cicustom',
801 description
=> "cloud-init: Sets DNS search domains for a container. Create will'
802 .' automatically use the setting from the host if neither searchdomain nor nameserver'
807 type
=> 'string', format
=> 'address-list',
808 description
=> "cloud-init: Sets DNS server IP address for a container. Create will'
809 .' automatically use the setting from the host if neither searchdomain nor nameserver'
815 format
=> 'urlencoded',
816 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
820 # what about other qemu settings ?
822 #machine => 'string',
835 ##soundhw => 'string',
837 while (my ($k, $v) = each %$confdesc) {
838 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
841 my $MAX_USB_DEVICES = 5;
843 my $MAX_SERIAL_PORTS = 4;
844 my $MAX_PARALLEL_PORTS = 3;
850 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
851 description
=> "CPUs accessing this NUMA node.",
852 format_description
=> "id[-id];...",
856 description
=> "Amount of memory this NUMA node provides.",
861 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
862 description
=> "Host NUMA nodes to use.",
863 format_description
=> "id[-id];...",
868 enum
=> [qw(preferred bind interleave)],
869 description
=> "NUMA allocation policy.",
873 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
876 type
=> 'string', format
=> $numa_fmt,
877 description
=> "NUMA topology.",
879 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
881 for (my $i = 0; $i < $MAX_NUMA; $i++) {
882 $confdesc->{"numa$i"} = $numadesc;
885 my $nic_model_list = [
901 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
903 my $net_fmt_bridge_descr = <<__EOD__;
904 Bridge to attach the network device to. The Proxmox VE standard bridge
907 If you do not specify a bridge, we create a kvm user (NATed) network
908 device, which provides DHCP and DNS services. The following addresses
915 The DHCP server assign addresses to the guest starting from 10.0.2.15.
919 macaddr
=> get_standard_option
('mac-addr', {
920 description
=> "MAC address. That address must be unique withing your network. This is"
921 ." automatically generated if not specified.",
925 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
926 ." very low CPU overhead. If your guest does not support this driver, it is usually"
927 ." best to use 'e1000'.",
928 enum
=> $nic_model_list,
931 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
934 description
=> $net_fmt_bridge_descr,
935 format_description
=> 'bridge',
936 pattern
=> '[-_.\w\d]+',
941 minimum
=> 0, maximum
=> 16,
942 description
=> 'Number of packet queues to be used on the device.',
948 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
953 minimum
=> 1, maximum
=> 4094,
954 description
=> 'VLAN tag to apply to packets on this interface.',
959 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
960 description
=> 'VLAN trunks to pass through this interface.',
961 format_description
=> 'vlanid[;vlanid...]',
966 description
=> 'Whether this interface should be protected by the firewall.',
971 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
976 minimum
=> 1, maximum
=> 65520,
977 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
984 type
=> 'string', format
=> $net_fmt,
985 description
=> "Specify network devices.",
988 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
993 format
=> 'pve-ipv4-config',
994 format_description
=> 'IPv4Format/CIDR',
995 description
=> 'IPv4 address in CIDR format.',
1002 format_description
=> 'GatewayIPv4',
1003 description
=> 'Default gateway for IPv4 traffic.',
1009 format
=> 'pve-ipv6-config',
1010 format_description
=> 'IPv6Format/CIDR',
1011 description
=> 'IPv6 address in CIDR format.',
1018 format_description
=> 'GatewayIPv6',
1019 description
=> 'Default gateway for IPv6 traffic.',
1024 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
1025 my $ipconfigdesc = {
1027 type
=> 'string', format
=> 'pve-qm-ipconfig',
1028 description
=> <<'EODESCR',
1029 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1031 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1033 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1034 gateway should be provided.
1035 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1036 cloud-init 19.4 or newer.
1038 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1042 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
1044 for (my $i = 0; $i < $MAX_NETS; $i++) {
1045 $confdesc->{"net$i"} = $netdesc;
1046 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1049 foreach my $key (keys %$confdesc_cloudinit) {
1050 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1053 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1054 sub verify_volume_id_or_qm_path
{
1055 my ($volid, $noerr) = @_;
1057 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
1061 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1062 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1073 type
=> 'string', format
=> 'pve-qm-usb-device',
1074 format_description
=> 'HOSTUSBDEVICE|spice',
1075 description
=> <<EODESCR,
1076 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1078 'bus-port(.port)*' (decimal numbers) or
1079 'vendor_id:product_id' (hexadeciaml numbers) or
1082 You can use the 'lsusb -t' command to list existing usb devices.
1084 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1085 machines - use with special care.
1087 The value 'spice' can be used to add a usb redirection devices for spice.
1093 description
=> "Specifies whether if given host option is a USB3 device or port.",
1100 type
=> 'string', format
=> $usb_fmt,
1101 description
=> "Configure an USB device (n is 0 to 4).",
1103 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1108 pattern
=> '(/dev/.+|socket)',
1109 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1110 verbose_description
=> <<EODESCR,
1111 Create a serial device inside the VM (n is 0 to 3), and pass through a
1112 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1113 host side (use 'qm terminal' to open a terminal connection).
1115 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1116 use with special care.
1118 CAUTION: Experimental! User reported problems with this option.
1125 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1126 description
=> "Map host parallel devices (n is 0 to 2).",
1127 verbose_description
=> <<EODESCR,
1128 Map host parallel devices (n is 0 to 2).
1130 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1131 machines - use with special care.
1133 CAUTION: Experimental! User reported problems with this option.
1137 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1138 $confdesc->{"parallel$i"} = $paralleldesc;
1141 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1142 $confdesc->{"serial$i"} = $serialdesc;
1145 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1146 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1149 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1150 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1153 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1154 $confdesc->{"usb$i"} = $usbdesc;
1162 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1163 . " Deprecated, use 'order=' instead.",
1164 pattern
=> '[acdn]{1,4}',
1165 format_description
=> "[acdn]{1,4}",
1167 # note: this is also the fallback if boot: is not given at all
1173 format
=> 'pve-qm-bootdev-list',
1174 format_description
=> "device[;device...]",
1175 description
=> <<EODESC,
1176 The guest will attempt to boot from devices in the order they appear here.
1178 Disks, optical drives and passed-through storage USB devices will be directly
1179 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1180 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1182 Note that only devices in this list will be marked as bootable and thus loaded
1183 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1184 (e.g. software-raid), you need to specify all of them here.
1186 Overrides the deprecated 'legacy=[acdn]*' value when given.
1190 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1192 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1193 sub verify_bootdev
{
1194 my ($dev, $noerr) = @_;
1196 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1197 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && !$special;
1201 return 0 if $dev !~ m/^$base\d+$/;
1202 return 0 if !$confdesc->{$dev};
1206 return $dev if $check->("net");
1207 return $dev if $check->("usb");
1208 return $dev if $check->("hostpci");
1211 die "invalid boot device '$dev'\n";
1214 sub print_bootorder
{
1216 return "" if !@$devs;
1217 my $data = { order
=> join(';', @$devs) };
1218 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1221 my $kvm_api_version = 0;
1224 return $kvm_api_version if $kvm_api_version;
1226 open my $fh, '<', '/dev/kvm' or return;
1228 # 0xae00 => KVM_GET_API_VERSION
1229 $kvm_api_version = ioctl($fh, 0xae00, 0);
1232 return $kvm_api_version;
1235 my $kvm_user_version = {};
1238 sub kvm_user_version
{
1241 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1242 my $st = stat($binary);
1244 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1245 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1246 $cachedmtime == $st->mtime;
1248 $kvm_user_version->{$binary} = 'unknown';
1249 $kvm_mtime->{$binary} = $st->mtime;
1253 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1254 $kvm_user_version->{$binary} = $2;
1258 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1261 return $kvm_user_version->{$binary};
1264 my sub extract_version
{
1265 my ($machine_type, $version) = @_;
1266 $version = kvm_user_version
() if !defined($version);
1267 return PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1270 sub kernel_has_vhost_net
{
1271 return -c
'/dev/vhost-net';
1276 return defined($confdesc->{$key});
1280 sub get_cdrom_path
{
1282 return $cdrom_path if $cdrom_path;
1284 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1285 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1286 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1290 my ($storecfg, $vmid, $cdrom) = @_;
1292 if ($cdrom eq 'cdrom') {
1293 return get_cdrom_path
();
1294 } elsif ($cdrom eq 'none') {
1296 } elsif ($cdrom =~ m
|^/|) {
1299 return PVE
::Storage
::path
($storecfg, $cdrom);
1303 # try to convert old style file names to volume IDs
1304 sub filename_to_volume_id
{
1305 my ($vmid, $file, $media) = @_;
1307 if (!($file eq 'none' || $file eq 'cdrom' ||
1308 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1310 return if $file =~ m
|/|;
1312 if ($media && $media eq 'cdrom') {
1313 $file = "local:iso/$file";
1315 $file = "local:$vmid/$file";
1322 sub verify_media_type
{
1323 my ($opt, $vtype, $media) = @_;
1328 if ($media eq 'disk') {
1330 } elsif ($media eq 'cdrom') {
1333 die "internal error";
1336 return if ($vtype eq $etype);
1338 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1341 sub cleanup_drive_path
{
1342 my ($opt, $storecfg, $drive) = @_;
1344 # try to convert filesystem paths to volume IDs
1346 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1347 ($drive->{file
} !~ m
|^/dev/.+|) &&
1348 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1349 ($drive->{file
} !~ m/^\d+$/)) {
1350 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1351 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1353 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1354 verify_media_type
($opt, $vtype, $drive->{media
});
1355 $drive->{file
} = $volid;
1358 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1361 sub parse_hotplug_features
{
1366 return $res if $data eq '0';
1368 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1370 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1371 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1374 die "invalid hotplug feature '$feature'\n";
1380 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1381 sub pve_verify_hotplug_features
{
1382 my ($value, $noerr) = @_;
1384 return $value if parse_hotplug_features
($value);
1388 die "unable to parse hotplug option\n";
1392 my($fh, $noerr) = @_;
1395 my $SG_GET_VERSION_NUM = 0x2282;
1397 my $versionbuf = "\x00" x
8;
1398 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1400 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1403 my $version = unpack("I", $versionbuf);
1404 if ($version < 30000) {
1405 die "scsi generic interface too old\n" if !$noerr;
1409 my $buf = "\x00" x
36;
1410 my $sensebuf = "\x00" x
8;
1411 my $cmd = pack("C x3 C x1", 0x12, 36);
1413 # see /usr/include/scsi/sg.h
1414 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1417 $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1420 $ret = ioctl($fh, $SG_IO, $packet);
1422 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1426 my @res = unpack($sg_io_hdr_t, $packet);
1427 if ($res[17] || $res[18]) {
1428 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1433 $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1435 $res->{removable
} = $res->{removable
} & 128 ?
1 : 0;
1436 $res->{type
} &= 0x1F;
1444 my $fh = IO
::File-
>new("+<$path") || return;
1445 my $res = scsi_inquiry
($fh, 1);
1451 sub print_tabletdevice_full
{
1452 my ($conf, $arch) = @_;
1454 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1456 # we use uhci for old VMs because tablet driver was buggy in older qemu
1458 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1464 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1467 sub print_keyboarddevice_full
{
1468 my ($conf, $arch) = @_;
1470 return if $arch ne 'aarch64';
1472 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1475 my sub get_drive_id
{
1477 return "$drive->{interface}$drive->{index}";
1480 sub print_drivedevice_full
{
1481 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1486 my $drive_id = get_drive_id
($drive);
1487 if ($drive->{interface
} eq 'virtio') {
1488 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1489 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1490 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1491 } elsif ($drive->{interface
} eq 'scsi') {
1493 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1494 my $unit = $drive->{index} % $maxdev;
1495 my $devicetype = 'hd';
1497 if (drive_is_cdrom
($drive)) {
1500 if ($drive->{file
} =~ m
|^/|) {
1501 $path = $drive->{file
};
1502 if (my $info = path_is_scsi
($path)) {
1503 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1504 $devicetype = 'block';
1505 } elsif ($info->{type
} == 1) { # tape
1506 $devicetype = 'generic';
1510 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1513 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1514 my $version = extract_version
($machine_type, kvm_user_version
());
1515 if ($path =~ m/^iscsi\:\/\
// &&
1516 !min_version
($version, 4, 1)) {
1517 $devicetype = 'generic';
1521 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1522 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1524 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1525 .",lun=$drive->{index}";
1527 $device .= ",drive=drive-$drive_id,id=$drive_id";
1529 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1530 $device .= ",rotation_rate=1";
1532 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1534 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1535 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1536 my $controller = int($drive->{index} / $maxdev);
1537 my $unit = $drive->{index} % $maxdev;
1538 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1540 $device = "ide-$devicetype";
1541 if ($drive->{interface
} eq 'ide') {
1542 $device .= ",bus=ide.$controller,unit=$unit";
1544 $device .= ",bus=ahci$controller.$unit";
1546 $device .= ",drive=drive-$drive_id,id=$drive_id";
1548 if ($devicetype eq 'hd') {
1549 if (my $model = $drive->{model
}) {
1550 $model = URI
::Escape
::uri_unescape
($model);
1551 $device .= ",model=$model";
1553 if ($drive->{ssd
}) {
1554 $device .= ",rotation_rate=1";
1557 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1558 } elsif ($drive->{interface
} eq 'usb') {
1560 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1562 die "unsupported interface type";
1565 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1567 if (my $serial = $drive->{serial
}) {
1568 $serial = URI
::Escape
::uri_unescape
($serial);
1569 $device .= ",serial=$serial";
1576 sub get_initiator_name
{
1579 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1580 while (defined(my $line = <$fh>)) {
1581 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1590 sub print_drive_commandline_full
{
1591 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1594 my $volid = $drive->{file
};
1595 my $format = $drive->{format
};
1596 my $drive_id = get_drive_id
($drive);
1598 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1599 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1601 if (drive_is_cdrom
($drive)) {
1602 $path = get_iso_path
($storecfg, $vmid, $volid);
1603 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1606 $path = PVE
::Storage
::path
($storecfg, $volid);
1607 $format //= qemu_img_format
($scfg, $volname);
1614 my $is_rbd = $path =~ m/^rbd:/;
1617 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1618 foreach my $o (@qemu_drive_options) {
1619 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1622 # snapshot only accepts on|off
1623 if (defined($drive->{snapshot
})) {
1624 my $v = $drive->{snapshot
} ?
'on' : 'off';
1625 $opts .= ",snapshot=$v";
1628 if (defined($drive->{ro
})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1629 $opts .= ",readonly=" . ($drive->{ro
} ?
'on' : 'off');
1632 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1633 my ($dir, $qmpname) = @$type;
1634 if (my $v = $drive->{"mbps$dir"}) {
1635 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1637 if (my $v = $drive->{"mbps${dir}_max"}) {
1638 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1640 if (my $v = $drive->{"bps${dir}_max_length"}) {
1641 $opts .= ",throttling.bps$qmpname-max-length=$v";
1643 if (my $v = $drive->{"iops${dir}"}) {
1644 $opts .= ",throttling.iops$qmpname=$v";
1646 if (my $v = $drive->{"iops${dir}_max"}) {
1647 $opts .= ",throttling.iops$qmpname-max=$v";
1649 if (my $v = $drive->{"iops${dir}_max_length"}) {
1650 $opts .= ",throttling.iops$qmpname-max-length=$v";
1655 $format = "rbd" if $is_rbd;
1656 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1658 $opts .= ",format=alloc-track,file.driver=$format";
1660 $opts .= ",format=$format";
1663 my $cache_direct = 0;
1665 if (my $cache = $drive->{cache
}) {
1666 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1667 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1668 $opts .= ",cache=none";
1672 # io_uring with cache mode writeback or writethrough on krbd will hang...
1673 my $rbd_no_io_uring = $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1675 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1676 # sometimes, just plain disable...
1677 my $lvm_no_io_uring = $scfg && $scfg->{type
} eq 'lvm';
1679 if (!$drive->{aio
}) {
1680 if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1681 # io_uring supports all cache modes
1682 $opts .= ",aio=io_uring";
1684 # aio native works only with O_DIRECT
1686 $opts .= ",aio=native";
1688 $opts .= ",aio=threads";
1693 if (!drive_is_cdrom
($drive)) {
1695 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1696 $detectzeroes = 'off';
1697 } elsif ($drive->{discard
}) {
1698 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1700 # This used to be our default with discard not being specified:
1701 $detectzeroes = 'on';
1704 # note: 'detect-zeroes' works per blockdev and we want it to persist
1705 # after the alloc-track is removed, so put it on 'file' directly
1706 my $dz_param = $pbs_name ?
"file.detect-zeroes" : "detect-zeroes";
1707 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1711 $opts .= ",backing=$pbs_name";
1712 $opts .= ",auto-remove=on";
1715 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1716 my $file_param = "file";
1718 # non-rbd drivers require the underlying file to be a seperate block
1719 # node, so add a second .file indirection
1720 $file_param .= ".file" if !$is_rbd;
1721 $file_param .= ".filename";
1723 my $pathinfo = $path ?
"$file_param=$path," : '';
1725 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1728 sub print_pbs_blockdev
{
1729 my ($pbs_conf, $pbs_name) = @_;
1730 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1731 $blockdev .= ",repository=$pbs_conf->{repository}";
1732 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1733 $blockdev .= ",archive=$pbs_conf->{archive}";
1734 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1738 sub print_netdevice_full
{
1739 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1741 my $device = $net->{model
};
1742 if ($net->{model
} eq 'virtio') {
1743 $device = 'virtio-net-pci';
1746 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1747 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1748 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1749 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1750 # and out of each queue plus one config interrupt and control vector queue
1751 my $vectors = $net->{queues
} * 2 + 2;
1752 $tmpstr .= ",vectors=$vectors,mq=on";
1754 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1756 if (my $mtu = $net->{mtu
}) {
1757 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1758 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1761 } elsif ($mtu < 576) {
1762 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1763 } elsif ($mtu > $bridge_mtu) {
1764 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1766 $tmpstr .= ",host_mtu=$mtu";
1768 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1772 if ($use_old_bios_files) {
1774 if ($device eq 'virtio-net-pci') {
1775 $romfile = 'pxe-virtio.rom';
1776 } elsif ($device eq 'e1000') {
1777 $romfile = 'pxe-e1000.rom';
1778 } elsif ($device eq 'e1000e') {
1779 $romfile = 'pxe-e1000e.rom';
1780 } elsif ($device eq 'ne2k') {
1781 $romfile = 'pxe-ne2k_pci.rom';
1782 } elsif ($device eq 'pcnet') {
1783 $romfile = 'pxe-pcnet.rom';
1784 } elsif ($device eq 'rtl8139') {
1785 $romfile = 'pxe-rtl8139.rom';
1787 $tmpstr .= ",romfile=$romfile" if $romfile;
1793 sub print_netdev_full
{
1794 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1797 if ($netid =~ m/^net(\d+)$/) {
1801 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1803 my $ifname = "tap${vmid}i$i";
1805 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1806 die "interface name '$ifname' is too long (max 15 character)\n"
1807 if length($ifname) >= 16;
1809 my $vhostparam = '';
1810 if (is_native
($arch)) {
1811 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1814 my $vmname = $conf->{name
} || "vm$vmid";
1817 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1819 if ($net->{bridge
}) {
1820 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1821 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1823 $netdev = "type=user,id=$netid,hostname=$vmname";
1826 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1832 'cirrus' => 'cirrus-vga',
1834 'vmware' => 'vmware-svga',
1835 'virtio' => 'virtio-vga',
1838 sub print_vga_device
{
1839 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1841 my $type = $vga_map->{$vga->{type
}};
1842 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1843 $type = 'virtio-gpu';
1845 my $vgamem_mb = $vga->{memory
};
1847 my $max_outputs = '';
1849 $type = $id ?
'qxl' : 'qxl-vga';
1851 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1852 # set max outputs so linux can have up to 4 qxl displays with one device
1853 if (min_version
($machine_version, 4, 1)) {
1854 $max_outputs = ",max_outputs=4";
1859 die "no devicetype for $vga->{type}\n" if !$type;
1863 if ($vga->{type
} eq 'virtio') {
1864 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1865 $memory = ",max_hostmem=$bytes";
1867 # from https://www.spice-space.org/multiple-monitors.html
1868 $memory = ",vgamem_mb=$vga->{memory}";
1869 my $ram = $vgamem_mb * 4;
1870 my $vram = $vgamem_mb * 2;
1871 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1873 $memory = ",vgamem_mb=$vga->{memory}";
1875 } elsif ($qxlnum && $id) {
1876 $memory = ",ram_size=67108864,vram_size=33554432";
1880 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1881 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1884 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1885 my $vgaid = "vga" . ($id // '');
1887 if ($q35 && $vgaid eq 'vga') {
1888 # the first display uses pcie.0 bus on q35 machines
1889 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1891 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1894 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1897 sub parse_number_sets
{
1900 foreach my $part (split(/;/, $set)) {
1901 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1902 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1903 push @$res, [ $1, $2 ];
1905 die "invalid range: $part\n";
1914 my $res = parse_property_string
($numa_fmt, $data);
1915 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1916 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1920 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1924 my $res = eval { parse_property_string
($net_fmt, $data) };
1929 if (!defined($res->{macaddr
})) {
1930 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1931 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1936 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1937 sub parse_ipconfig
{
1940 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1946 if ($res->{gw
} && !$res->{ip
}) {
1947 warn 'gateway specified without specifying an IP address';
1950 if ($res->{gw6
} && !$res->{ip6
}) {
1951 warn 'IPv6 gateway specified without specifying an IPv6 address';
1954 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1955 warn 'gateway specified together with DHCP';
1958 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1960 warn "IPv6 gateway specified together with $res->{ip6} address";
1964 if (!$res->{ip
} && !$res->{ip6
}) {
1965 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1974 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1977 sub add_random_macs
{
1978 my ($settings) = @_;
1980 foreach my $opt (keys %$settings) {
1981 next if $opt !~ m/^net(\d+)$/;
1982 my $net = parse_net
($settings->{$opt});
1984 $settings->{$opt} = print_net
($net);
1988 sub vm_is_volid_owner
{
1989 my ($storecfg, $vmid, $volid) = @_;
1991 if ($volid !~ m
|^/|) {
1993 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1994 if ($owner && ($owner == $vmid)) {
2002 sub vmconfig_register_unused_drive
{
2003 my ($storecfg, $vmid, $conf, $drive) = @_;
2005 if (drive_is_cloudinit
($drive)) {
2006 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2008 } elsif (!drive_is_cdrom
($drive)) {
2009 my $volid = $drive->{file
};
2010 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2011 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2016 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2020 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2021 format_description
=> 'UUID',
2022 description
=> "Set SMBIOS1 UUID.",
2027 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2028 format_description
=> 'Base64 encoded string',
2029 description
=> "Set SMBIOS1 version.",
2034 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2035 format_description
=> 'Base64 encoded string',
2036 description
=> "Set SMBIOS1 serial number.",
2041 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2042 format_description
=> 'Base64 encoded string',
2043 description
=> "Set SMBIOS1 manufacturer.",
2048 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2049 format_description
=> 'Base64 encoded string',
2050 description
=> "Set SMBIOS1 product ID.",
2055 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2056 format_description
=> 'Base64 encoded string',
2057 description
=> "Set SMBIOS1 SKU string.",
2062 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2063 format_description
=> 'Base64 encoded string',
2064 description
=> "Set SMBIOS1 family string.",
2069 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2077 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2084 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2087 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2089 sub parse_watchdog
{
2094 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2099 sub parse_guest_agent
{
2102 return {} if !defined($conf->{agent
});
2104 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2107 # if the agent is disabled ignore the other potentially set properties
2108 return {} if !$res->{enabled
};
2113 my ($conf, $key) = @_;
2114 return undef if !defined($conf->{agent
});
2116 my $agent = parse_guest_agent
($conf);
2117 return $agent->{$key};
2123 return {} if !$value;
2124 my $res = eval { parse_property_string
($vga_fmt, $value) };
2134 my $res = eval { parse_property_string
($rng_fmt, $value) };
2139 sub parse_meta_info
{
2144 my $res = eval { parse_property_string
($meta_info_fmt, $value) };
2149 sub new_meta_info_string
{
2150 my () = @_; # for now do not allow to override any value
2152 return PVE
::JSONSchema
::print_property_string
(
2154 'creation-qemu' => kvm_user_version
(),
2155 ctime
=> "". int(time()),
2161 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2162 sub verify_usb_device
{
2163 my ($value, $noerr) = @_;
2165 return $value if parse_usb_device
($value);
2169 die "unable to parse usb device\n";
2172 # add JSON properties for create and set function
2173 sub json_config_properties
{
2176 my $skip_json_config_opts = {
2180 runningmachine
=> 1,
2185 foreach my $opt (keys %$confdesc) {
2186 next if $skip_json_config_opts->{$opt};
2187 $prop->{$opt} = $confdesc->{$opt};
2193 # return copy of $confdesc_cloudinit to generate documentation
2194 sub cloudinit_config_properties
{
2196 return dclone
($confdesc_cloudinit);
2200 my ($key, $value) = @_;
2202 die "unknown setting '$key'\n" if !$confdesc->{$key};
2204 my $type = $confdesc->{$key}->{type
};
2206 if (!defined($value)) {
2207 die "got undefined value\n";
2210 if ($value =~ m/[\n\r]/) {
2211 die "property contains a line feed\n";
2214 if ($type eq 'boolean') {
2215 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2216 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2217 die "type check ('boolean') failed - got '$value'\n";
2218 } elsif ($type eq 'integer') {
2219 return int($1) if $value =~ m/^(\d+)$/;
2220 die "type check ('integer') failed - got '$value'\n";
2221 } elsif ($type eq 'number') {
2222 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2223 die "type check ('number') failed - got '$value'\n";
2224 } elsif ($type eq 'string') {
2225 if (my $fmt = $confdesc->{$key}->{format
}) {
2226 PVE
::JSONSchema
::check_format
($fmt, $value);
2229 $value =~ s/^\"(.*)\"$/$1/;
2232 die "internal error"
2237 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2239 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2241 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2243 if ($conf->{template
}) {
2244 # check if any base image is still used by a linked clone
2245 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2246 my ($ds, $drive) = @_;
2247 return if drive_is_cdrom
($drive);
2249 my $volid = $drive->{file
};
2250 return if !$volid || $volid =~ m
|^/|;
2252 die "base volume '$volid' is still in use by linked cloned\n"
2253 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2259 my $remove_owned_drive = sub {
2260 my ($ds, $drive) = @_;
2261 return if drive_is_cdrom
($drive, 1);
2263 my $volid = $drive->{file
};
2264 return if !$volid || $volid =~ m
|^/|;
2265 return if $volids->{$volid};
2267 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2268 return if !$path || !$owner || ($owner != $vmid);
2270 $volids->{$volid} = 1;
2271 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2272 warn "Could not remove disk '$volid', check manually: $@" if $@;
2275 # only remove disks owned by this VM (referenced in the config)
2276 my $include_opts = {
2277 include_unused
=> 1,
2278 extra_keys
=> ['vmstate'],
2280 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2282 for my $snap (values %{$conf->{snapshots
}}) {
2283 next if !defined($snap->{vmstate
});
2284 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2285 next if !defined($drive);
2286 $remove_owned_drive->('vmstate', $drive);
2289 PVE
::QemuConfig-
>foreach_volume_full($conf->{pending
}, $include_opts, $remove_owned_drive);
2291 if ($purge_unreferenced) { # also remove unreferenced disk
2292 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2293 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2294 my ($volid, $sid, $volname, $d) = @_;
2295 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2300 if (defined $replacement_conf) {
2301 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2303 PVE
::QemuConfig-
>destroy_config($vmid);
2307 sub parse_vm_config
{
2308 my ($filename, $raw) = @_;
2310 return if !defined($raw);
2313 digest
=> Digest
::SHA
::sha1_hex
($raw),
2318 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2319 || die "got strange filename '$filename'";
2327 my @lines = split(/\n/, $raw);
2328 foreach my $line (@lines) {
2329 next if $line =~ m/^\s*$/;
2331 if ($line =~ m/^\[PENDING\]\s*$/i) {
2332 $section = 'pending';
2333 if (defined($descr)) {
2335 $conf->{description
} = $descr;
2338 $conf = $res->{$section} = {};
2341 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2343 if (defined($descr)) {
2345 $conf->{description
} = $descr;
2348 $conf = $res->{snapshots
}->{$section} = {};
2352 if ($line =~ m/^\#(.*)\s*$/) {
2353 $descr = '' if !defined($descr);
2354 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2358 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2359 $descr = '' if !defined($descr);
2360 $descr .= PVE
::Tools
::decode_text
($2);
2361 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2362 $conf->{snapstate
} = $1;
2363 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2366 $conf->{$key} = $value;
2367 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2369 if ($section eq 'pending') {
2370 $conf->{delete} = $value; # we parse this later
2372 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2374 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2377 eval { $value = check_type
($key, $value); };
2379 warn "vm $vmid - unable to parse value of '$key' - $@";
2381 $key = 'ide2' if $key eq 'cdrom';
2382 my $fmt = $confdesc->{$key}->{format
};
2383 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2384 my $v = parse_drive
($key, $value);
2385 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2386 $v->{file
} = $volid;
2387 $value = print_drive
($v);
2389 warn "vm $vmid - unable to parse value of '$key'\n";
2394 $conf->{$key} = $value;
2397 warn "vm $vmid - unable to parse config: $line\n";
2401 if (defined($descr)) {
2403 $conf->{description
} = $descr;
2405 delete $res->{snapstate
}; # just to be sure
2410 sub write_vm_config
{
2411 my ($filename, $conf) = @_;
2413 delete $conf->{snapstate
}; # just to be sure
2415 if ($conf->{cdrom
}) {
2416 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2417 $conf->{ide2
} = $conf->{cdrom
};
2418 delete $conf->{cdrom
};
2421 # we do not use 'smp' any longer
2422 if ($conf->{sockets
}) {
2423 delete $conf->{smp
};
2424 } elsif ($conf->{smp
}) {
2425 $conf->{sockets
} = $conf->{smp
};
2426 delete $conf->{cores
};
2427 delete $conf->{smp
};
2430 my $used_volids = {};
2432 my $cleanup_config = sub {
2433 my ($cref, $pending, $snapname) = @_;
2435 foreach my $key (keys %$cref) {
2436 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2437 $key eq 'snapstate' || $key eq 'pending';
2438 my $value = $cref->{$key};
2439 if ($key eq 'delete') {
2440 die "propertry 'delete' is only allowed in [PENDING]\n"
2442 # fixme: check syntax?
2445 eval { $value = check_type
($key, $value); };
2446 die "unable to parse value of '$key' - $@" if $@;
2448 $cref->{$key} = $value;
2450 if (!$snapname && is_valid_drivename
($key)) {
2451 my $drive = parse_drive
($key, $value);
2452 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2457 &$cleanup_config($conf);
2459 &$cleanup_config($conf->{pending
}, 1);
2461 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2462 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2463 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2466 # remove 'unusedX' settings if we re-add a volume
2467 foreach my $key (keys %$conf) {
2468 my $value = $conf->{$key};
2469 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2470 delete $conf->{$key};
2474 my $generate_raw_config = sub {
2475 my ($conf, $pending) = @_;
2479 # add description as comment to top of file
2480 if (defined(my $descr = $conf->{description
})) {
2482 foreach my $cl (split(/\n/, $descr)) {
2483 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2486 $raw .= "#\n" if $pending;
2490 foreach my $key (sort keys %$conf) {
2491 next if $key =~ /^(digest|description|pending|snapshots)$/;
2492 $raw .= "$key: $conf->{$key}\n";
2497 my $raw = &$generate_raw_config($conf);
2499 if (scalar(keys %{$conf->{pending
}})){
2500 $raw .= "\n[PENDING]\n";
2501 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2504 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2505 $raw .= "\n[$snapname]\n";
2506 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2516 # we use static defaults from our JSON schema configuration
2517 foreach my $key (keys %$confdesc) {
2518 if (defined(my $default = $confdesc->{$key}->{default})) {
2519 $res->{$key} = $default;
2527 my $vmlist = PVE
::Cluster
::get_vmlist
();
2529 return $res if !$vmlist || !$vmlist->{ids
};
2530 my $ids = $vmlist->{ids
};
2531 my $nodename = nodename
();
2533 foreach my $vmid (keys %$ids) {
2534 my $d = $ids->{$vmid};
2535 next if !$d->{node
} || $d->{node
} ne $nodename;
2536 next if !$d->{type
} || $d->{type
} ne 'qemu';
2537 $res->{$vmid}->{exists} = 1;
2542 # test if VM uses local resources (to prevent migration)
2543 sub check_local_resources
{
2544 my ($conf, $noerr) = @_;
2548 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2549 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2551 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2553 foreach my $k (keys %$conf) {
2554 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2555 # sockets are safe: they will recreated be on the target side post-migrate
2556 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2557 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2560 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2565 # check if used storages are available on all nodes (use by migrate)
2566 sub check_storage_availability
{
2567 my ($storecfg, $conf, $node) = @_;
2569 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2570 my ($ds, $drive) = @_;
2572 my $volid = $drive->{file
};
2575 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2578 # check if storage is available on both nodes
2579 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2580 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2582 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2584 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2585 if !$scfg->{content
}->{$vtype};
2589 # list nodes where all VM images are available (used by has_feature API)
2591 my ($conf, $storecfg) = @_;
2593 my $nodelist = PVE
::Cluster
::get_nodelist
();
2594 my $nodehash = { map { $_ => 1 } @$nodelist };
2595 my $nodename = nodename
();
2597 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2598 my ($ds, $drive) = @_;
2600 my $volid = $drive->{file
};
2603 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2605 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2606 if ($scfg->{disable
}) {
2608 } elsif (my $avail = $scfg->{nodes
}) {
2609 foreach my $node (keys %$nodehash) {
2610 delete $nodehash->{$node} if !$avail->{$node};
2612 } elsif (!$scfg->{shared
}) {
2613 foreach my $node (keys %$nodehash) {
2614 delete $nodehash->{$node} if $node ne $nodename
2623 sub check_local_storage_availability
{
2624 my ($conf, $storecfg) = @_;
2626 my $nodelist = PVE
::Cluster
::get_nodelist
();
2627 my $nodehash = { map { $_ => {} } @$nodelist };
2629 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2630 my ($ds, $drive) = @_;
2632 my $volid = $drive->{file
};
2635 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2637 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2639 if ($scfg->{disable
}) {
2640 foreach my $node (keys %$nodehash) {
2641 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2643 } elsif (my $avail = $scfg->{nodes
}) {
2644 foreach my $node (keys %$nodehash) {
2645 if (!$avail->{$node}) {
2646 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2653 foreach my $node (values %$nodehash) {
2654 if (my $unavail = $node->{unavailable_storages
}) {
2655 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2662 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2664 my ($vmid, $nocheck, $node) = @_;
2666 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2667 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2672 my $vzlist = config_list
();
2674 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2676 while (defined(my $de = $fd->read)) {
2677 next if $de !~ m/^(\d+)\.pid$/;
2679 next if !defined($vzlist->{$vmid});
2680 if (my $pid = check_running
($vmid)) {
2681 $vzlist->{$vmid}->{pid
} = $pid;
2688 our $vmstatus_return_properties = {
2689 vmid
=> get_standard_option
('pve-vmid'),
2691 description
=> "Qemu process status.",
2693 enum
=> ['stopped', 'running'],
2696 description
=> "Maximum memory in bytes.",
2699 renderer
=> 'bytes',
2702 description
=> "Root disk size in bytes.",
2705 renderer
=> 'bytes',
2708 description
=> "VM name.",
2713 description
=> "Qemu QMP agent status.",
2718 description
=> "PID of running qemu process.",
2723 description
=> "Uptime.",
2726 renderer
=> 'duration',
2729 description
=> "Maximum usable CPUs.",
2734 description
=> "The current config lock, if any.",
2739 description
=> "The current configured tags, if any",
2743 'running-machine' => {
2744 description
=> "The currently running machine type (if running).",
2749 description
=> "The currently running QEMU version (if running).",
2755 my $last_proc_pid_stat;
2757 # get VM status information
2758 # This must be fast and should not block ($full == false)
2759 # We only query KVM using QMP if $full == true (this can be slow)
2761 my ($opt_vmid, $full) = @_;
2765 my $storecfg = PVE
::Storage
::config
();
2767 my $list = vzlist
();
2768 my $defaults = load_defaults
();
2770 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2772 my $cpucount = $cpuinfo->{cpus
} || 1;
2774 foreach my $vmid (keys %$list) {
2775 next if $opt_vmid && ($vmid ne $opt_vmid);
2777 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2779 my $d = { vmid
=> int($vmid) };
2780 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2782 # fixme: better status?
2783 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2785 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2786 if (defined($size)) {
2787 $d->{disk
} = 0; # no info available
2788 $d->{maxdisk
} = $size;
2794 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2795 * ($conf->{cores
} || $defaults->{cores
});
2796 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2797 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2799 $d->{name
} = $conf->{name
} || "VM $vmid";
2800 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2801 : $defaults->{memory
}*(1024*1024);
2803 if ($conf->{balloon
}) {
2804 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2805 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2806 : $defaults->{shares
};
2817 $d->{diskwrite
} = 0;
2819 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2821 $d->{serial
} = 1 if conf_has_serial
($conf);
2822 $d->{lock} = $conf->{lock} if $conf->{lock};
2823 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2828 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2829 foreach my $dev (keys %$netdev) {
2830 next if $dev !~ m/^tap([1-9]\d*)i/;
2832 my $d = $res->{$vmid};
2835 $d->{netout
} += $netdev->{$dev}->{receive
};
2836 $d->{netin
} += $netdev->{$dev}->{transmit
};
2839 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2840 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2845 my $ctime = gettimeofday
;
2847 foreach my $vmid (keys %$list) {
2849 my $d = $res->{$vmid};
2850 my $pid = $d->{pid
};
2853 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2854 next if !$pstat; # not running
2856 my $used = $pstat->{utime} + $pstat->{stime
};
2858 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2860 if ($pstat->{vsize
}) {
2861 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2864 my $old = $last_proc_pid_stat->{$pid};
2866 $last_proc_pid_stat->{$pid} = {
2874 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2876 if ($dtime > 1000) {
2877 my $dutime = $used - $old->{used
};
2879 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2880 $last_proc_pid_stat->{$pid} = {
2886 $d->{cpu
} = $old->{cpu
};
2890 return $res if !$full;
2892 my $qmpclient = PVE
::QMPClient-
>new();
2894 my $ballooncb = sub {
2895 my ($vmid, $resp) = @_;
2897 my $info = $resp->{'return'};
2898 return if !$info->{max_mem
};
2900 my $d = $res->{$vmid};
2902 # use memory assigned to VM
2903 $d->{maxmem
} = $info->{max_mem
};
2904 $d->{balloon
} = $info->{actual
};
2906 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2907 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2908 $d->{freemem
} = $info->{free_mem
};
2911 $d->{ballooninfo
} = $info;
2914 my $blockstatscb = sub {
2915 my ($vmid, $resp) = @_;
2916 my $data = $resp->{'return'} || [];
2917 my $totalrdbytes = 0;
2918 my $totalwrbytes = 0;
2920 for my $blockstat (@$data) {
2921 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2922 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2924 $blockstat->{device
} =~ s/drive-//;
2925 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2927 $res->{$vmid}->{diskread
} = $totalrdbytes;
2928 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2931 my $machinecb = sub {
2932 my ($vmid, $resp) = @_;
2933 my $data = $resp->{'return'} || [];
2935 $res->{$vmid}->{'running-machine'} =
2936 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
2939 my $versioncb = sub {
2940 my ($vmid, $resp) = @_;
2941 my $data = $resp->{'return'} // {};
2942 my $version = 'unknown';
2944 if (my $v = $data->{qemu
}) {
2945 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
2948 $res->{$vmid}->{'running-qemu'} = $version;
2951 my $statuscb = sub {
2952 my ($vmid, $resp) = @_;
2954 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2955 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2956 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2957 # this fails if ballon driver is not loaded, so this must be
2958 # the last commnand (following command are aborted if this fails).
2959 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2961 my $status = 'unknown';
2962 if (!defined($status = $resp->{'return'}->{status
})) {
2963 warn "unable to get VM status\n";
2967 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2970 foreach my $vmid (keys %$list) {
2971 next if $opt_vmid && ($vmid ne $opt_vmid);
2972 next if !$res->{$vmid}->{pid
}; # not running
2973 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2976 $qmpclient->queue_execute(undef, 2);
2978 foreach my $vmid (keys %$list) {
2979 next if $opt_vmid && ($vmid ne $opt_vmid);
2980 next if !$res->{$vmid}->{pid
}; #not running
2982 # we can't use the $qmpclient since it might have already aborted on
2983 # 'query-balloon', but this might also fail for older versions...
2984 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
2985 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2988 foreach my $vmid (keys %$list) {
2989 next if $opt_vmid && ($vmid ne $opt_vmid);
2990 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2996 sub conf_has_serial
{
2999 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3000 if ($conf->{"serial$i"}) {
3008 sub conf_has_audio
{
3009 my ($conf, $id) = @_;
3012 my $audio = $conf->{"audio$id"};
3013 return if !defined($audio);
3015 my $audioproperties = parse_property_string
($audio_fmt, $audio);
3016 my $audiodriver = $audioproperties->{driver
} // 'spice';
3019 dev
=> $audioproperties->{device
},
3020 dev_id
=> "audiodev$id",
3021 backend
=> $audiodriver,
3022 backend_id
=> "$audiodriver-backend${id}",
3027 my ($audio, $audiopciaddr, $machine_version) = @_;
3031 my $id = $audio->{dev_id
};
3033 if (min_version
($machine_version, 4, 2)) {
3034 $audiodev = ",audiodev=$audio->{backend_id}";
3037 if ($audio->{dev
} eq 'AC97') {
3038 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3039 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3040 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3041 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3042 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3044 die "unkown audio device '$audio->{dev}', implement me!";
3047 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3055 socket => "/var/run/qemu-server/$vmid.swtpm",
3056 pid
=> "/var/run/qemu-server/$vmid.swtpm.pid",
3060 sub add_tpm_device
{
3061 my ($vmid, $devices, $conf) = @_;
3063 return if !$conf->{tpmstate0
};
3065 my $paths = get_tpm_paths
($vmid);
3067 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3068 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3069 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3073 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3075 return if !$tpmdrive;
3078 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
3079 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
3081 $state = PVE
::Storage
::map_volume
($storecfg, $tpm->{file
});
3083 $state = $tpm->{file
};
3086 my $paths = get_tpm_paths
($vmid);
3088 # during migration, we will get state from remote
3091 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3098 "--create-platform-cert",
3101 "/etc/swtpm_setup.conf", # do not use XDG configs
3103 "0", # force creation as root, error if not possible
3104 "--not-overwrite", # ignore existing state, do not modify
3107 push @$setup_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3108 # TPM 2.0 supports ECC crypto, use if possible
3109 push @$setup_cmd, "--ecc" if $tpm->{version
} eq 'v2.0';
3111 run_command
($setup_cmd, outfunc
=> sub {
3112 print "swtpm_setup: $1\n";
3116 my $emulator_cmd = [
3120 "backend-uri=file://$state,mode=0600",
3122 "type=unixio,path=$paths->{socket},mode=0600",
3124 "file=$paths->{pid}",
3125 "--terminate", # terminate on QEMU disconnect
3128 push @$emulator_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3129 run_command
($emulator_cmd, outfunc
=> sub { print $1; });
3131 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3132 while (! -e
$paths->{pid
}) {
3133 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3137 # return untainted PID of swtpm daemon so it can be killed on error
3138 file_read_firstline
($paths->{pid
}) =~ m/(\d+)/;
3142 sub vga_conf_has_spice
{
3145 my $vgaconf = parse_vga
($vga);
3146 my $vgatype = $vgaconf->{type
};
3147 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3154 return get_host_arch
() eq $arch;
3159 return $conf->{arch
} // get_host_arch
();
3162 my $default_machines = {
3167 sub get_installed_machine_version
{
3168 my ($kvmversion) = @_;
3169 $kvmversion = kvm_user_version
() if !defined($kvmversion);
3170 $kvmversion =~ m/^(\d+\.\d+)/;
3174 sub windows_get_pinned_machine_version
{
3175 my ($machine, $base_version, $kvmversion) = @_;
3177 my $pin_version = $base_version;
3178 if (!defined($base_version) ||
3179 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3181 $pin_version = get_installed_machine_version
($kvmversion);
3183 if (!$machine || $machine eq 'pc') {
3184 $machine = "pc-i440fx-$pin_version";
3185 } elsif ($machine eq 'q35') {
3186 $machine = "pc-q35-$pin_version";
3187 } elsif ($machine eq 'virt') {
3188 $machine = "virt-$pin_version";
3190 warn "unknown machine type '$machine', not touching that!\n";
3196 sub get_vm_machine
{
3197 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3199 my $machine = $forcemachine || $conf->{machine
};
3201 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3202 $kvmversion //= kvm_user_version
();
3203 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3204 # layout which confuses windows quite a bit and may result in various regressions..
3205 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3206 if (windows_version
($conf->{ostype
})) {
3207 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3210 $machine ||= $default_machines->{$arch};
3211 if ($add_pve_version) {
3212 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3213 $machine .= "+pve$pvever";
3217 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3218 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3219 $machine = $1 if $is_pxe;
3221 # for version-pinned machines that do not include a pve-version (e.g.
3222 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3223 $machine .= '+pve0';
3225 $machine .= '.pxe' if $is_pxe;
3231 sub get_ovmf_files
($$$) {
3232 my ($arch, $efidisk, $smm) = @_;
3234 my $types = $OVMF->{$arch}
3235 or die "no OVMF images known for architecture '$arch'\n";
3237 my $type = 'default';
3238 if (defined($efidisk->{efitype
}) && $efidisk->{efitype
} eq '4m') {
3239 $type = $smm ?
"4m" : "4m-no-smm";
3240 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3243 return $types->{$type}->@*;
3247 aarch64
=> '/usr/bin/qemu-system-aarch64',
3248 x86_64
=> '/usr/bin/qemu-system-x86_64',
3250 sub get_command_for_arch
($) {
3252 return '/usr/bin/kvm' if is_native
($arch);
3254 my $cmd = $Arch2Qemu->{$arch}
3255 or die "don't know how to emulate architecture '$arch'\n";
3259 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3260 # to use in a QEMU command line (-cpu element), first array_intersect the result
3261 # of query_supported_ with query_understood_. This is necessary because:
3263 # a) query_understood_ returns flags the host cannot use and
3264 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3265 # flags, but CPU settings - with most of them being flags. Those settings
3266 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3268 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3269 # expensive. If you need the value returned from this, you can get it much
3270 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3271 # $accel being 'kvm' or 'tcg'.
3273 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3274 # changes, automatically populating pmxcfs.
3276 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3277 # since kvm and tcg machines support different flags
3279 sub query_supported_cpu_flags
{
3282 $arch //= get_host_arch
();
3283 my $default_machine = $default_machines->{$arch};
3287 # FIXME: Once this is merged, the code below should work for ARM as well:
3288 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3289 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3292 my $kvm_supported = defined(kvm_version
());
3293 my $qemu_cmd = get_command_for_arch
($arch);
3295 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3297 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3298 my $query_supported_run_qemu = sub {
3304 '-machine', $default_machine,
3306 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3307 '-mon', 'chardev=qmp,mode=control',
3308 '-pidfile', $pidfile,
3313 push @$cmd, '-accel', 'tcg';
3316 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3317 die "QEMU flag querying VM exited with code " . $rc if $rc;
3320 my $cmd_result = mon_cmd
(
3322 'query-cpu-model-expansion',
3324 model
=> { name
=> 'host' }
3327 my $props = $cmd_result->{model
}->{props
};
3328 foreach my $prop (keys %$props) {
3329 next if $props->{$prop} ne '1';
3330 # QEMU returns some flags multiple times, with '_', '.' or '-'
3331 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3332 # We only keep those with underscores, to match /proc/cpuinfo
3333 $prop =~ s/\.|-/_/g;
3334 $flags->{$prop} = 1;
3339 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3340 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3344 return [ sort keys %$flags ];
3347 # We need to query QEMU twice, since KVM and TCG have different supported flags
3348 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3349 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3350 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3352 if ($kvm_supported) {
3353 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3354 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3361 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3362 my $understood_cpu_flag_dir = "/usr/share/kvm";
3363 sub query_understood_cpu_flags
{
3364 my $arch = get_host_arch
();
3365 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3367 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3370 my $raw = file_get_contents
($filepath);
3371 $raw =~ s/^\s+|\s+$//g;
3372 my @flags = split(/\s+/, $raw);
3377 my sub get_cpuunits
{
3379 return $conf->{cpuunits
} // (PVE
::CGroup
::cgroup_mode
() == 2 ?
100 : 1024);
3381 sub config_to_command
{
3382 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3386 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3389 my $ostype = $conf->{ostype
};
3390 my $winversion = windows_version
($ostype);
3391 my $kvm = $conf->{kvm
};
3392 my $nodename = nodename
();
3394 my $arch = get_vm_arch
($conf);
3395 my $kvm_binary = get_command_for_arch
($arch);
3396 my $kvmver = kvm_user_version
($kvm_binary);
3398 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3399 $kvmver //= "undefined";
3400 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3403 my $add_pve_version = min_version
($kvmver, 4, 1);
3405 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3406 my $machine_version = extract_version
($machine_type, $kvmver);
3407 $kvm //= 1 if is_native
($arch);
3409 $machine_version =~ m/(\d+)\.(\d+)/;
3410 my ($machine_major, $machine_minor) = ($1, $2);
3412 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3413 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3414 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3415 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3416 ." please upgrade node '$nodename'\n"
3417 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3418 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3419 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3420 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3421 ." node '$nodename'\n";
3424 # if a specific +pve version is required for a feature, use $version_guard
3425 # instead of min_version to allow machines to be run with the minimum
3427 my $required_pve_version = 0;
3428 my $version_guard = sub {
3429 my ($major, $minor, $pve) = @_;
3430 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3431 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3432 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3433 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3437 if ($kvm && !defined kvm_version
()) {
3438 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3439 ." or enable in BIOS.\n";
3442 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3443 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3444 my $use_old_bios_files = undef;
3445 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3447 my $cpuunits = get_cpuunits
($conf);
3449 push @$cmd, $kvm_binary;
3451 push @$cmd, '-id', $vmid;
3453 my $vmname = $conf->{name
} || "vm$vmid";
3455 push @$cmd, '-name', $vmname;
3457 push @$cmd, '-no-shutdown';
3461 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3462 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3463 push @$cmd, '-mon', "chardev=qmp,mode=control";
3465 if (min_version
($machine_version, 2, 12)) {
3466 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3467 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3470 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3472 push @$cmd, '-daemonize';
3474 if ($conf->{smbios1
}) {
3475 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3476 if ($smbios_conf->{base64
}) {
3477 # Do not pass base64 flag to qemu
3478 delete $smbios_conf->{base64
};
3479 my $smbios_string = "";
3480 foreach my $key (keys %$smbios_conf) {
3482 if ($key eq "uuid") {
3483 $value = $smbios_conf->{uuid
}
3485 $value = decode_base64
($smbios_conf->{$key});
3487 # qemu accepts any binary data, only commas need escaping by double comma
3489 $smbios_string .= "," . $key . "=" . $value if $value;
3491 push @$cmd, '-smbios', "type=1" . $smbios_string;
3493 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3497 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3499 if (my $efidisk = $conf->{efidisk0
}) {
3500 $d = parse_drive
('efidisk0', $efidisk);
3503 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch, $d, $q35);
3504 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3506 my ($path, $format);
3507 my $read_only_str = '';
3509 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3510 $format = $d->{format
};
3512 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3513 if (!defined($format)) {
3514 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3515 $format = qemu_img_format
($scfg, $volname);
3519 die "efidisk format must be specified\n"
3520 if !defined($format);
3523 $read_only_str = ',readonly=on' if drive_is_read_only
($conf, $d);
3525 warn "no efidisk configured! Using temporary efivars disk.\n";
3526 $path = "/tmp/$vmid-ovmf.fd";
3527 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3533 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3534 $size_str = ",size=" . (-s
$ovmf_vars);
3537 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3539 if ($path =~ m/^rbd:/) {
3540 $cache = ',cache=writeback';
3541 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3544 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3545 push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3548 if ($q35) { # tell QEMU to load q35 config early
3549 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3550 if (min_version
($machine_version, 4, 0)) {
3551 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3553 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3557 if ($conf->{vmgenid
}) {
3558 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3561 # add usb controllers
3562 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3563 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3564 push @$devices, @usbcontrollers if @usbcontrollers;
3565 my $vga = parse_vga
($conf->{vga
});
3567 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3568 $vga->{type
} = 'qxl' if $qxlnum;
3570 if (!$vga->{type
}) {
3571 if ($arch eq 'aarch64') {
3572 $vga->{type
} = 'virtio';
3573 } elsif (min_version
($machine_version, 2, 9)) {
3574 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3576 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3580 # enable absolute mouse coordinates (needed by vnc)
3581 my $tablet = $conf->{tablet
};
3582 if (!defined($tablet)) {
3583 $tablet = $defaults->{tablet
};
3584 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3585 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3589 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3590 my $kbd = print_keyboarddevice_full
($conf, $arch);
3591 push @$devices, '-device', $kbd if defined($kbd);
3594 my $bootorder = device_bootorder
($conf);
3596 # host pci device passthrough
3597 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3598 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3601 my $usb_dev_features = {};
3602 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3604 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3605 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3606 push @$devices, @usbdevices if @usbdevices;
3609 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3610 my $path = $conf->{"serial$i"} or next;
3611 if ($path eq 'socket') {
3612 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3613 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3614 # On aarch64, serial0 is the UART device. Qemu only allows
3615 # connecting UART devices via the '-serial' command line, as
3616 # the device has a fixed slot on the hardware...
3617 if ($arch eq 'aarch64' && $i == 0) {
3618 push @$devices, '-serial', "chardev:serial$i";
3620 push @$devices, '-device', "isa-serial,chardev=serial$i";
3623 die "no such serial device\n" if ! -c
$path;
3624 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3625 push @$devices, '-device', "isa-serial,chardev=serial$i";
3630 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3631 if (my $path = $conf->{"parallel$i"}) {
3632 die "no such parallel device\n" if ! -c
$path;
3633 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3634 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3635 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3639 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3640 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3641 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3642 push @$devices, @$audio_devs;
3645 add_tpm_device
($vmid, $devices, $conf);
3648 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3649 $sockets = $conf->{sockets
} if $conf->{sockets
};
3651 my $cores = $conf->{cores
} || 1;
3653 my $maxcpus = $sockets * $cores;
3655 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3657 my $allowed_vcpus = $cpuinfo->{cpus
};
3659 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3661 if ($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3662 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3663 for (my $i = 2; $i <= $vcpus; $i++) {
3664 my $cpustr = print_cpu_device
($conf,$i);
3665 push @$cmd, '-device', $cpustr;
3670 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3672 push @$cmd, '-nodefaults';
3674 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3676 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3678 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3680 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3681 push @$devices, '-device', print_vga_device
(
3682 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3683 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3684 push @$cmd, '-vnc', "unix:$socket,password=on";
3686 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3687 push @$cmd, '-nographic';
3691 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3692 my $useLocaltime = $conf->{localtime};
3694 if ($winversion >= 5) { # windows
3695 $useLocaltime = 1 if !defined($conf->{localtime});
3697 # use time drift fix when acpi is enabled
3698 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3699 $tdf = 1 if !defined($conf->{tdf
});
3703 if ($winversion >= 6) {
3704 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3705 push @$cmd, '-no-hpet';
3708 push @$rtcFlags, 'driftfix=slew' if $tdf;
3710 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3711 push @$rtcFlags, "base=$conf->{startdate}";
3712 } elsif ($useLocaltime) {
3713 push @$rtcFlags, 'base=localtime';
3717 push @$cmd, '-cpu', $forcecpu;
3719 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3722 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3724 push @$cmd, '-S' if $conf->{freeze
};
3726 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3728 my $guest_agent = parse_guest_agent
($conf);
3730 if ($guest_agent->{enabled
}) {
3731 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3732 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3734 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3735 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3736 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3737 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3738 } elsif ($guest_agent->{type
} eq 'isa') {
3739 push @$devices, '-device', "isa-serial,chardev=qga0";
3743 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3744 if ($rng && $version_guard->(4, 1, 2)) {
3745 check_rng_source
($rng->{source
});
3747 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3748 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3749 my $limiter_str = "";
3751 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3754 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3755 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3756 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3764 for (my $i = 1; $i < $qxlnum; $i++){
3765 push @$devices, '-device', print_vga_device
(
3766 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3769 # assume other OS works like Linux
3770 my ($ram, $vram) = ("134217728", "67108864");
3771 if ($vga->{memory
}) {
3772 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3773 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3775 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3776 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3780 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3782 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3783 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3784 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3786 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3787 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3788 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3790 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3791 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3793 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3794 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3795 if ($spice_enhancement->{foldersharing
}) {
3796 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3797 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3800 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3801 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3802 if $spice_enhancement->{videostreaming
};
3804 push @$devices, '-spice', "$spice_opts";
3807 # enable balloon by default, unless explicitly disabled
3808 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3809 my $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3810 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3813 if ($conf->{watchdog
}) {
3814 my $wdopts = parse_watchdog
($conf->{watchdog
});
3815 my $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3816 my $watchdog = $wdopts->{model
} || 'i6300esb';
3817 push @$devices, '-device', "$watchdog$pciaddr";
3818 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3822 my $scsicontroller = {};
3823 my $ahcicontroller = {};
3824 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3826 # Add iscsi initiator name if available
3827 if (my $initiator = get_initiator_name
()) {
3828 push @$devices, '-iscsi', "initiator-name=$initiator";
3831 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3832 my ($ds, $drive) = @_;
3834 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3835 check_volume_storage_type
($storecfg, $drive->{file
});
3836 push @$vollist, $drive->{file
};
3839 # ignore efidisk here, already added in bios/fw handling code above
3840 return if $drive->{interface
} eq 'efidisk';
3842 return if $drive->{interface
} eq 'tpmstate';
3844 $use_virtio = 1 if $ds =~ m/^virtio/;
3846 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3848 if ($drive->{interface
} eq 'virtio'){
3849 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3852 if ($drive->{interface
} eq 'scsi') {
3854 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3856 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3857 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3859 my $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3860 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3863 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3864 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3865 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3866 } elsif ($drive->{iothread
}) {
3867 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3871 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3872 $queues = ",num_queues=$drive->{queues}";
3875 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3876 if !$scsicontroller->{$controller};
3877 $scsicontroller->{$controller}=1;
3880 if ($drive->{interface
} eq 'sata') {
3881 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3882 my $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3883 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3884 if !$ahcicontroller->{$controller};
3885 $ahcicontroller->{$controller}=1;
3888 my $pbs_conf = $pbs_backing->{$ds};
3889 my $pbs_name = undef;
3891 $pbs_name = "drive-$ds-pbs";
3892 push @$devices, '-blockdev', print_pbs_blockdev
($pbs_conf, $pbs_name);
3895 my $drive_cmd = print_drive_commandline_full
(
3896 $storecfg, $vmid, $drive, $pbs_name, min_version
($kvmver, 6, 0));
3898 # extra protection for templates, but SATA and IDE don't support it..
3899 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
3901 push @$devices, '-drive',$drive_cmd;
3902 push @$devices, '-device', print_drivedevice_full
(
3903 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3906 for (my $i = 0; $i < $MAX_NETS; $i++) {
3907 my $netname = "net$i";
3909 next if !$conf->{$netname};
3910 my $d = parse_net
($conf->{$netname});
3913 $use_virtio = 1 if $d->{model
} eq 'virtio';
3915 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
3917 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
3918 push @$devices, '-netdev', $netdevfull;
3920 my $netdevicefull = print_netdevice_full
(
3921 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3923 push @$devices, '-device', $netdevicefull;
3926 if ($conf->{ivshmem
}) {
3927 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3931 $bus = print_pcie_addr
("ivshmem");
3933 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3936 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3937 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3939 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3940 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3941 .",size=$ivshmem->{size}M";
3944 # pci.4 is nested in pci.1
3945 $bridges->{1} = 1 if $bridges->{4};
3947 if (!$q35) { # add pci bridges
3948 if (min_version
($machine_version, 2, 3)) {
3952 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3955 for my $k (sort {$b cmp $a} keys %$bridges) {
3956 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3959 if ($k == 2 && $legacy_igd) {
3962 my $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
3963 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3965 if ($q35) { # add after -readconfig pve-q35.cfg
3966 splice @$devices, 2, 0, '-device', $devstr;
3968 unshift @$devices, '-device', $devstr if $k > 0;
3973 push @$machineFlags, 'accel=tcg';
3976 my $machine_type_min = $machine_type;
3977 if ($add_pve_version) {
3978 $machine_type_min =~ s/\+pve\d+$//;
3979 $machine_type_min .= "+pve$required_pve_version";
3981 push @$machineFlags, "type=${machine_type_min}";
3983 push @$cmd, @$devices;
3984 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3985 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3986 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3988 if (my $vmstate = $conf->{vmstate
}) {
3989 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3990 push @$vollist, $vmstate;
3991 push @$cmd, '-loadstate', $statepath;
3992 print "activating and using '$vmstate' as vmstate\n";
3995 if (PVE
::QemuConfig-
>is_template($conf)) {
3996 # needed to workaround base volumes being read-only
3997 push @$cmd, '-snapshot';
4001 if ($conf->{args
}) {
4002 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4006 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
4009 sub check_rng_source
{
4012 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4013 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4016 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4017 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
4018 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4019 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4020 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4021 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4029 my $res = mon_cmd
($vmid, 'query-spice');
4031 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4034 sub vm_devices_list
{
4037 my $res = mon_cmd
($vmid, 'query-pci');
4038 my $devices_to_check = [];
4040 foreach my $pcibus (@$res) {
4041 push @$devices_to_check, @{$pcibus->{devices
}},
4044 while (@$devices_to_check) {
4046 for my $d (@$devices_to_check) {
4047 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4048 next if !$d->{'pci_bridge'};
4050 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4051 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4053 $devices_to_check = $to_check;
4056 my $resblock = mon_cmd
($vmid, 'query-block');
4057 foreach my $block (@$resblock) {
4058 if($block->{device
} =~ m/^drive-(\S+)/){
4063 my $resmice = mon_cmd
($vmid, 'query-mice');
4064 foreach my $mice (@$resmice) {
4065 if ($mice->{name
} eq 'QEMU HID Tablet') {
4066 $devices->{tablet
} = 1;
4071 # for usb devices there is no query-usb
4072 # but we can iterate over the entries in
4073 # qom-list path=/machine/peripheral
4074 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4075 foreach my $per (@$resperipheral) {
4076 if ($per->{name
} =~ m/^usb\d+$/) {
4077 $devices->{$per->{name
}} = 1;
4085 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4087 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4089 my $devices_list = vm_devices_list
($vmid);
4090 return 1 if defined($devices_list->{$deviceid});
4092 # add PCI bridge if we need it for the device
4093 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4095 if ($deviceid eq 'tablet') {
4096 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4097 } elsif ($deviceid eq 'keyboard') {
4098 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4099 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4100 die "usb hotplug currently not reliable\n";
4101 # since we can't reliably hot unplug all added usb devices and usb
4102 # passthrough breaks live migration we disable usb hotplugging for now
4103 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4104 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4105 qemu_iothread_add
($vmid, $deviceid, $device);
4107 qemu_driveadd
($storecfg, $vmid, $device);
4108 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4110 qemu_deviceadd
($vmid, $devicefull);
4111 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4113 eval { qemu_drivedel
($vmid, $deviceid); };
4117 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4118 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4119 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4120 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4122 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4124 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4125 qemu_iothread_add
($vmid, $deviceid, $device);
4126 $devicefull .= ",iothread=iothread-$deviceid";
4129 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4130 $devicefull .= ",num_queues=$device->{queues}";
4133 qemu_deviceadd
($vmid, $devicefull);
4134 qemu_deviceaddverify
($vmid, $deviceid);
4135 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4136 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4137 qemu_driveadd
($storecfg, $vmid, $device);
4139 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4140 eval { qemu_deviceadd
($vmid, $devicefull); };
4142 eval { qemu_drivedel
($vmid, $deviceid); };
4146 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4147 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4149 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4150 my $use_old_bios_files = undef;
4151 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4153 my $netdevicefull = print_netdevice_full
(
4154 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4155 qemu_deviceadd
($vmid, $netdevicefull);
4157 qemu_deviceaddverify
($vmid, $deviceid);
4158 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4161 eval { qemu_netdevdel
($vmid, $deviceid); };
4165 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4167 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4168 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4170 qemu_deviceadd
($vmid, $devicefull);
4171 qemu_deviceaddverify
($vmid, $deviceid);
4173 die "can't hotplug device '$deviceid'\n";
4179 # fixme: this should raise exceptions on error!
4180 sub vm_deviceunplug
{
4181 my ($vmid, $conf, $deviceid) = @_;
4183 my $devices_list = vm_devices_list
($vmid);
4184 return 1 if !defined($devices_list->{$deviceid});
4186 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4187 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4189 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4190 qemu_devicedel
($vmid, $deviceid);
4191 } elsif ($deviceid =~ m/^usb\d+$/) {
4192 die "usb hotplug currently not reliable\n";
4193 # when unplugging usb devices this way, there may be remaining usb
4194 # controllers/hubs so we disable it for now
4195 #qemu_devicedel($vmid, $deviceid);
4196 #qemu_devicedelverify($vmid, $deviceid);
4197 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4198 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4200 qemu_devicedel
($vmid, $deviceid);
4201 qemu_devicedelverify
($vmid, $deviceid);
4202 qemu_drivedel
($vmid, $deviceid);
4203 qemu_iothread_del
($vmid, $deviceid, $device);
4204 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4205 qemu_devicedel
($vmid, $deviceid);
4206 qemu_devicedelverify
($vmid, $deviceid);
4207 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4208 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4210 qemu_devicedel
($vmid, $deviceid);
4211 qemu_drivedel
($vmid, $deviceid);
4212 qemu_deletescsihw
($conf, $vmid, $deviceid);
4214 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4215 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4216 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4217 qemu_devicedel
($vmid, $deviceid);
4218 qemu_devicedelverify
($vmid, $deviceid);
4219 qemu_netdevdel
($vmid, $deviceid);
4221 die "can't unplug device '$deviceid'\n";
4227 sub qemu_deviceadd
{
4228 my ($vmid, $devicefull) = @_;
4230 $devicefull = "driver=".$devicefull;
4231 my %options = split(/[=,]/, $devicefull);
4233 mon_cmd
($vmid, "device_add" , %options);
4236 sub qemu_devicedel
{
4237 my ($vmid, $deviceid) = @_;
4239 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4242 sub qemu_iothread_add
{
4243 my ($vmid, $deviceid, $device) = @_;
4245 if ($device->{iothread
}) {
4246 my $iothreads = vm_iothreads_list
($vmid);
4247 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4251 sub qemu_iothread_del
{
4252 my ($vmid, $deviceid, $device) = @_;
4254 if ($device->{iothread
}) {
4255 my $iothreads = vm_iothreads_list
($vmid);
4256 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4260 sub qemu_objectadd
{
4261 my ($vmid, $objectid, $qomtype) = @_;
4263 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4268 sub qemu_objectdel
{
4269 my ($vmid, $objectid) = @_;
4271 mon_cmd
($vmid, "object-del", id
=> $objectid);
4277 my ($storecfg, $vmid, $device) = @_;
4279 my $kvmver = get_running_qemu_version
($vmid);
4280 my $io_uring = min_version
($kvmver, 6, 0);
4281 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4282 $drive =~ s/\\/\\\\/g;
4283 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4285 # If the command succeeds qemu prints: "OK
"
4286 return 1 if $ret =~ m/OK/s;
4288 die "adding drive failed
: $ret\n";
4292 my ($vmid, $deviceid) = @_;
4294 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4297 return 1 if $ret eq "";
4299 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4300 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4302 die "deleting drive
$deviceid failed
: $ret\n";
4305 sub qemu_deviceaddverify {
4306 my ($vmid, $deviceid) = @_;
4308 for (my $i = 0; $i <= 5; $i++) {
4309 my $devices_list = vm_devices_list($vmid);
4310 return 1 if defined($devices_list->{$deviceid});
4314 die "error on hotplug device
'$deviceid'\n";
4318 sub qemu_devicedelverify {
4319 my ($vmid, $deviceid) = @_;
4321 # need to verify that the device is correctly removed as device_del
4322 # is async and empty return is not reliable
4324 for (my $i = 0; $i <= 5; $i++) {
4325 my $devices_list = vm_devices_list($vmid);
4326 return 1 if !defined($devices_list->{$deviceid});
4330 die "error on hot-unplugging device
'$deviceid'\n";
4333 sub qemu_findorcreatescsihw {
4334 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4336 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4338 my $scsihwid="$controller_prefix$controller";
4339 my $devices_list = vm_devices_list($vmid);
4341 if (!defined($devices_list->{$scsihwid})) {
4342 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4348 sub qemu_deletescsihw {
4349 my ($conf, $vmid, $opt) = @_;
4351 my $device = parse_drive($opt, $conf->{$opt});
4353 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4354 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4358 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4360 my $devices_list = vm_devices_list($vmid);
4361 foreach my $opt (keys %{$devices_list}) {
4362 if (is_valid_drivename($opt)) {
4363 my $drive = parse_drive($opt, $conf->{$opt});
4364 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4370 my $scsihwid="scsihw
$controller";
4372 vm_deviceunplug($vmid, $conf, $scsihwid);
4377 sub qemu_add_pci_bridge {
4378 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4384 print_pci_addr($device, $bridges, $arch, $machine_type);
4386 while (my ($k, $v) = each %$bridges) {
4389 return 1 if !defined($bridgeid) || $bridgeid < 1;
4391 my $bridge = "pci
.$bridgeid";
4392 my $devices_list = vm_devices_list($vmid);
4394 if (!defined($devices_list->{$bridge})) {
4395 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4401 sub qemu_set_link_status {
4402 my ($vmid, $device, $up) = @_;
4404 mon_cmd($vmid, "set_link
", name => $device,
4405 up => $up ? JSON::true : JSON::false);
4408 sub qemu_netdevadd {
4409 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4411 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4412 my %options = split(/[=,]/, $netdev);
4414 if (defined(my $vhost = $options{vhost})) {
4415 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4418 if (defined(my $queues = $options{queues})) {
4419 $options{queues} = $queues + 0;
4422 mon_cmd($vmid, "netdev_add
", %options);
4426 sub qemu_netdevdel {
4427 my ($vmid, $deviceid) = @_;
4429 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4432 sub qemu_usb_hotplug {
4433 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4437 # remove the old one first
4438 vm_deviceunplug($vmid, $conf, $deviceid);
4440 # check if xhci controller is necessary and available
4441 if ($device->{usb3}) {
4443 my $devicelist = vm_devices_list($vmid);
4445 if (!$devicelist->{xhci}) {
4446 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4447 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4450 my $d = parse_usb_device($device->{host});
4451 $d->{usb3} = $device->{usb3};
4454 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4457 sub qemu_cpu_hotplug {
4458 my ($vmid, $conf, $vcpus) = @_;
4460 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4463 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4464 $sockets = $conf->{sockets} if $conf->{sockets};
4465 my $cores = $conf->{cores} || 1;
4466 my $maxcpus = $sockets * $cores;
4468 $vcpus = $maxcpus if !$vcpus;
4470 die "you can
't add more vcpus than maxcpus\n"
4471 if $vcpus > $maxcpus;
4473 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4475 if ($vcpus < $currentvcpus) {
4477 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4479 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4480 qemu_devicedel($vmid, "cpu$i");
4482 my $currentrunningvcpus = undef;
4484 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4485 last if scalar(@{$currentrunningvcpus}) == $i-1;
4486 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4490 #update conf after each succesfull cpu unplug
4491 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4492 PVE::QemuConfig->write_config($vmid, $conf);
4495 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4501 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4502 die "vcpus in running vm does not match its configuration\n"
4503 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4505 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4507 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4508 my $cpustr = print_cpu_device($conf, $i);
4509 qemu_deviceadd($vmid, $cpustr);
4512 my $currentrunningvcpus = undef;
4514 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4515 last if scalar(@{$currentrunningvcpus}) == $i;
4516 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4520 #update conf after each succesfull cpu hotplug
4521 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4522 PVE::QemuConfig->write_config($vmid, $conf);
4526 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4527 mon_cmd($vmid, "cpu-add", id => int($i));
4532 sub qemu_block_set_io_throttle {
4533 my ($vmid, $deviceid,
4534 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4535 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4536 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4537 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4539 return if !check_running($vmid) ;
4541 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4543 bps_rd => int($bps_rd),
4544 bps_wr => int($bps_wr),
4546 iops_rd => int($iops_rd),
4547 iops_wr => int($iops_wr),
4548 bps_max => int($bps_max),
4549 bps_rd_max => int($bps_rd_max),
4550 bps_wr_max => int($bps_wr_max),
4551 iops_max => int($iops_max),
4552 iops_rd_max => int($iops_rd_max),
4553 iops_wr_max => int($iops_wr_max),
4554 bps_max_length => int($bps_max_length),
4555 bps_rd_max_length => int($bps_rd_max_length),
4556 bps_wr_max_length => int($bps_wr_max_length),
4557 iops_max_length => int($iops_max_length),
4558 iops_rd_max_length => int($iops_rd_max_length),
4559 iops_wr_max_length => int($iops_wr_max_length),
4564 sub qemu_block_resize {
4565 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4567 my $running = check_running($vmid);
4569 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4571 return if !$running;
4573 my $padding = (1024 - $size % 1024) % 1024;
4574 $size = $size + $padding;
4579 device => $deviceid,
4585 sub qemu_volume_snapshot {
4586 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4588 my $running = check_running($vmid);
4590 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4591 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4593 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4597 sub qemu_volume_snapshot_delete {
4598 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4600 my $running = check_running($vmid);
4605 my $conf = PVE::QemuConfig->load_config($vmid);
4606 PVE::QemuConfig->foreach_volume($conf, sub {
4607 my ($ds, $drive) = @_;
4608 $running = 1 if $drive->{file} eq $volid;
4612 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4613 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4615 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4619 sub set_migration_caps {
4620 my ($vmid, $savevm) = @_;
4622 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4624 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4625 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4630 "auto-converge" => 1,
4632 "x-rdma-pin-all" => 0,
4635 "dirty-bitmaps" => $dirty_bitmaps,
4638 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4640 for my $supported_capability (@$supported_capabilities) {
4642 capability => $supported_capability->{capability},
4643 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4647 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4651 my ($conf, $func, @param) = @_;
4655 my $test_volid = sub {
4656 my ($key, $drive, $snapname) = @_;
4658 my $volid = $drive->{file};
4661 $volhash->{$volid}->{cdrom} //= 1;
4662 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4664 my $replicate = $drive->{replicate} // 1;
4665 $volhash->{$volid}->{replicate} //= 0;
4666 $volhash->{$volid}->{replicate} = 1 if $replicate;
4668 $volhash->{$volid}->{shared} //= 0;
4669 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4671 $volhash->{$volid}->{referenced_in_config} //= 0;
4672 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4674 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4675 if defined($snapname);
4677 my $size = $drive->{size};
4678 $volhash->{$volid}->{size} //= $size if $size;
4680 $volhash->{$volid}->{is_vmstate} //= 0;
4681 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4683 $volhash->{$volid}->{is_tpmstate} //= 0;
4684 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0
';
4686 $volhash->{$volid}->{is_unused} //= 0;
4687 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4689 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4692 my $include_opts = {
4693 extra_keys => ['vmstate
'],
4694 include_unused => 1,
4697 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4698 foreach my $snapname (keys %{$conf->{snapshots}}) {
4699 my $snap = $conf->{snapshots}->{$snapname};
4700 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4703 foreach my $volid (keys %$volhash) {
4704 &$func($volid, $volhash->{$volid}, @param);
4708 my $fast_plug_option = {
4716 'vmstatestorage
' => 1,
4721 # hotplug changes in [PENDING]
4722 # $selection hash can be used to only apply specified options, for
4723 # example: { cores => 1 } (only apply changed 'cores
')
4724 # $errors ref is used to return error messages
4725 sub vmconfig_hotplug_pending {
4726 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4728 my $defaults = load_defaults();
4729 my $arch = get_vm_arch($conf);
4730 my $machine_type = get_vm_machine($conf, undef, $arch);
4732 # commit values which do not have any impact on running VM first
4733 # Note: those option cannot raise errors, we we do not care about
4734 # $selection and always apply them.
4736 my $add_error = sub {
4737 my ($opt, $msg) = @_;
4738 $errors->{$opt} = "hotplug problem - $msg";
4742 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4743 if ($fast_plug_option->{$opt}) {
4744 $conf->{$opt} = $conf->{pending}->{$opt};
4745 delete $conf->{pending}->{$opt};
4751 PVE::QemuConfig->write_config($vmid, $conf);
4754 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4756 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4757 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4758 foreach my $opt (sort keys %$pending_delete_hash) {
4759 next if $selection && !$selection->{$opt};
4760 my $force = $pending_delete_hash->{$opt}->{force};
4762 if ($opt eq 'hotplug
') {
4763 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4764 } elsif ($opt eq 'tablet
') {
4765 die "skip\n" if !$hotplug_features->{usb};
4766 if ($defaults->{tablet}) {
4767 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4768 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4769 if $arch eq 'aarch64
';
4771 vm_deviceunplug($vmid, $conf, 'tablet
');
4772 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4774 } elsif ($opt =~ m/^usb\d+/) {
4776 # since we cannot reliably hot unplug usb devices we are disabling it
4777 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4778 #vm_deviceunplug($vmid, $conf, $opt);
4779 } elsif ($opt eq 'vcpus
') {
4780 die "skip\n" if !$hotplug_features->{cpu};
4781 qemu_cpu_hotplug($vmid, $conf, undef);
4782 } elsif ($opt eq 'balloon
') {
4783 # enable balloon device is not hotpluggable
4784 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4785 # here we reset the ballooning value to memory
4786 my $balloon = $conf->{memory} || $defaults->{memory};
4787 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4788 } elsif ($fast_plug_option->{$opt}) {
4790 } elsif ($opt =~ m/^net(\d+)$/) {
4791 die "skip\n" if !$hotplug_features->{network};
4792 vm_deviceunplug($vmid, $conf, $opt);
4793 } elsif (is_valid_drivename($opt)) {
4794 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4795 vm_deviceunplug($vmid, $conf, $opt);
4796 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4797 } elsif ($opt =~ m/^memory$/) {
4798 die "skip\n" if !$hotplug_features->{memory};
4799 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4800 } elsif ($opt eq 'cpuunits
') {
4801 $cgroup->change_cpu_shares(undef, 1024);
4802 } elsif ($opt eq 'cpulimit
') {
4803 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4809 &$add_error($opt, $err) if $err ne "skip\n";
4811 delete $conf->{$opt};
4812 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4816 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4817 $apply_pending_cloudinit = sub {
4818 return if $apply_pending_cloudinit_done; # once is enough
4819 $apply_pending_cloudinit_done = 1; # once is enough
4821 my ($key, $value) = @_;
4823 my @cloudinit_opts = keys %$confdesc_cloudinit;
4824 foreach my $opt (keys %{$conf->{pending}}) {
4825 next if !grep { $_ eq $opt } @cloudinit_opts;
4826 $conf->{$opt} = delete $conf->{pending}->{$opt};
4829 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4830 foreach my $opt (sort keys %$pending_delete_hash) {
4831 next if !grep { $_ eq $opt } @cloudinit_opts;
4832 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4833 delete $conf->{$opt};
4836 my $new_conf = { %$conf };
4837 $new_conf->{$key} = $value;
4838 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4841 foreach my $opt (keys %{$conf->{pending}}) {
4842 next if $selection && !$selection->{$opt};
4843 my $value = $conf->{pending}->{$opt};
4845 if ($opt eq 'hotplug
') {
4846 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4847 } elsif ($opt eq 'tablet
') {
4848 die "skip\n" if !$hotplug_features->{usb};
4850 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4851 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4852 if $arch eq 'aarch64
';
4853 } elsif ($value == 0) {
4854 vm_deviceunplug($vmid, $conf, 'tablet
');
4855 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4857 } elsif ($opt =~ m/^usb\d+$/) {
4859 # since we cannot reliably hot unplug usb devices we disable it for now
4860 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4861 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4862 #die "skip\n" if !$d;
4863 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4864 } elsif ($opt eq 'vcpus
') {
4865 die "skip\n" if !$hotplug_features->{cpu};
4866 qemu_cpu_hotplug($vmid, $conf, $value);
4867 } elsif ($opt eq 'balloon
') {
4868 # enable/disable balloning device is not hotpluggable
4869 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4870 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4871 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4873 # allow manual ballooning if shares is set to zero
4874 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4875 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4876 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4878 } elsif ($opt =~ m/^net(\d+)$/) {
4879 # some changes can be done without hotplug
4880 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4881 $vmid, $opt, $value, $arch, $machine_type);
4882 } elsif (is_valid_drivename($opt)) {
4883 die "skip\n" if $opt eq 'efidisk0
' || $opt eq 'tpmstate0
';
4884 # some changes can be done without hotplug
4885 my $drive = parse_drive($opt, $value);
4886 if (drive_is_cloudinit($drive)) {
4887 &$apply_pending_cloudinit($opt, $value);
4889 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4890 $vmid, $opt, $value, $arch, $machine_type);
4891 } elsif ($opt =~ m/^memory$/) { #dimms
4892 die "skip\n" if !$hotplug_features->{memory};
4893 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4894 } elsif ($opt eq 'cpuunits
') {
4895 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4896 } elsif ($opt eq 'cpulimit
') {
4897 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4898 $cgroup->change_cpu_quota($cpulimit, 100000);
4899 } elsif ($opt eq 'agent
') {
4900 vmconfig_update_agent($conf, $opt, $value);
4902 die "skip\n"; # skip non-hot-pluggable options
4906 &$add_error($opt, $err) if $err ne "skip\n";
4908 $conf->{$opt} = $value;
4909 delete $conf->{pending}->{$opt};
4913 PVE::QemuConfig->write_config($vmid, $conf);
4916 sub try_deallocate_drive {
4917 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4919 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4920 my $volid = $drive->{file};
4921 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4922 my $sid = PVE::Storage::parse_volume_id($volid);
4923 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4925 # check if the disk is really unused
4926 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4927 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4928 PVE::Storage::vdisk_free($storecfg, $volid);
4931 # If vm is not owner of this disk remove from config
4939 sub vmconfig_delete_or_detach_drive {
4940 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4942 my $drive = parse_drive($opt, $conf->{$opt});
4944 my $rpcenv = PVE::RPCEnvironment::get();
4945 my $authuser = $rpcenv->get_user();
4948 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4949 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4951 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4957 sub vmconfig_apply_pending {
4958 my ($vmid, $conf, $storecfg, $errors) = @_;
4960 my $add_apply_error = sub {
4961 my ($opt, $msg) = @_;
4962 my $err_msg = "unable to apply pending change $opt : $msg";
4963 $errors->{$opt} = $err_msg;
4969 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4970 foreach my $opt (sort keys %$pending_delete_hash) {
4971 my $force = $pending_delete_hash->{$opt}->{force};
4973 if ($opt =~ m/^unused/) {
4974 die "internal error";
4975 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4976 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4980 $add_apply_error->($opt, $err);
4982 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4983 delete $conf->{$opt};
4987 PVE::QemuConfig->cleanup_pending($conf);
4989 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4990 next if $opt eq 'delete'; # just to be sure
4992 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4993 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4997 $add_apply_error->($opt, $err);
4999 $conf->{$opt} = delete $conf->{pending}->{$opt};
5003 # write all changes at once to avoid unnecessary i/o
5004 PVE::QemuConfig->write_config($vmid, $conf);
5007 sub vmconfig_update_net {
5008 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5010 my $newnet = parse_net($value);
5012 if ($conf->{$opt}) {
5013 my $oldnet = parse_net($conf->{$opt});
5015 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5016 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5017 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5018 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5020 # for non online change, we try to hot-unplug
5021 die "skip\n" if !$hotplug;
5022 vm_deviceunplug($vmid, $conf, $opt);
5025 die "internal error" if $opt !~ m/net(\d+)/;
5026 my $iface = "tap${vmid}i$1";
5028 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5029 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5030 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5031 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5032 PVE::Network::tap_unplug($iface);
5035 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5037 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5039 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5040 # Rate can be applied on its own but any change above needs to
5041 # include the rate in tap_plug since OVS resets everything.
5042 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5045 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5046 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5054 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5060 sub vmconfig_update_agent {
5061 my ($conf, $opt, $value) = @_;
5063 die "skip\n" if !$conf->{$opt};
5065 my $hotplug_options = { fstrim_cloned_disks => 1 };
5067 my $old_agent = parse_guest_agent($conf);
5068 my $agent = parse_guest_agent({$opt => $value});
5070 for my $option (keys %$agent) { # added/changed options
5071 next if defined($hotplug_options->{$option});
5072 die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5075 for my $option (keys %$old_agent) { # removed options
5076 next if defined($hotplug_options->{$option});
5077 die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5080 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5083 sub vmconfig_update_disk {
5084 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5086 my $drive = parse_drive($opt, $value);
5088 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5089 my $media = $drive->{media} || 'disk
';
5090 my $oldmedia = $old_drive->{media} || 'disk
';
5091 die "unable to change media type\n" if $media ne $oldmedia;
5093 if (!drive_is_cdrom($old_drive)) {
5095 if ($drive->{file} ne $old_drive->{file}) {
5097 die "skip\n" if !$hotplug;
5099 # unplug and register as unused
5100 vm_deviceunplug($vmid, $conf, $opt);
5101 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5104 # update existing disk
5106 # skip non hotpluggable value
5107 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5108 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5109 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5110 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5111 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5116 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5117 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5118 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5119 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5120 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5121 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5122 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5123 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5124 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5125 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5126 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5127 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5128 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5129 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5130 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5131 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5132 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5133 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5135 qemu_block_set_io_throttle(
5137 ($drive->{mbps} || 0)*1024*1024,
5138 ($drive->{mbps_rd} || 0)*1024*1024,
5139 ($drive->{mbps_wr} || 0)*1024*1024,
5140 $drive->{iops} || 0,
5141 $drive->{iops_rd} || 0,
5142 $drive->{iops_wr} || 0,
5143 ($drive->{mbps_max} || 0)*1024*1024,
5144 ($drive->{mbps_rd_max} || 0)*1024*1024,
5145 ($drive->{mbps_wr_max} || 0)*1024*1024,
5146 $drive->{iops_max} || 0,
5147 $drive->{iops_rd_max} || 0,
5148 $drive->{iops_wr_max} || 0,
5149 $drive->{bps_max_length} || 1,
5150 $drive->{bps_rd_max_length} || 1,
5151 $drive->{bps_wr_max_length} || 1,
5152 $drive->{iops_max_length} || 1,
5153 $drive->{iops_rd_max_length} || 1,
5154 $drive->{iops_wr_max_length} || 1,
5164 if ($drive->{file} eq 'none
') {
5165 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5166 if (drive_is_cloudinit($old_drive)) {
5167 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5170 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5172 # force eject if locked
5173 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5176 mon_cmd($vmid, "blockdev-change-medium",
5177 id => "$opt", filename => "$path");
5185 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5187 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5188 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5191 # called in locked context by incoming migration
5192 sub vm_migrate_get_nbd_disks {
5193 my ($storecfg, $conf, $replicated_volumes) = @_;
5195 my $local_volumes = {};
5196 PVE::QemuConfig->foreach_volume($conf, sub {
5197 my ($ds, $drive) = @_;
5199 return if drive_is_cdrom($drive);
5201 my $volid = $drive->{file};
5205 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5207 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5208 return if $scfg->{shared};
5210 # replicated disks re-use existing state via bitmap
5211 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5212 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5214 return $local_volumes;
5217 # called in locked context by incoming migration
5218 sub vm_migrate_alloc_nbd_disks {
5219 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5224 foreach my $opt (sort keys %$source_volumes) {
5225 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5227 if ($use_existing) {
5228 $nbd->{$opt}->{drivestr} = print_drive($drive);
5229 $nbd->{$opt}->{volid} = $volid;
5230 $nbd->{$opt}->{replicated} = 1;
5234 # If a remote storage is specified and the format of the original
5235 # volume is not available there, fall back to the default format.
5236 # Otherwise use the same format as the original.
5237 if (!$storagemap->{identity}) {
5238 $storeid = map_storage($storagemap, $storeid);
5239 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5240 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5241 my $fileFormat = qemu_img_format($scfg, $volname);
5242 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5244 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5245 $format = qemu_img_format($scfg, $volname);
5248 my $size = $drive->{size} / 1024;
5249 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5250 my $newdrive = $drive;
5251 $newdrive->{format} = $format;
5252 $newdrive->{file} = $newvolid;
5253 my $drivestr = print_drive($newdrive);
5254 $nbd->{$opt}->{drivestr} = $drivestr;
5255 $nbd->{$opt}->{volid} = $newvolid;
5261 # see vm_start_nolock for parameters, additionally:
5263 # storagemap = parsed storage map for allocating NBD disks
5265 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5267 return PVE::QemuConfig->lock_config($vmid, sub {
5268 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5270 die "you can't start a vm
if it
's a template\n"
5271 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5273 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended
');
5274 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup
');
5276 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5278 if ($has_backup_lock && $running) {
5279 # a backup is currently running, attempt to start the guest in the
5280 # existing QEMU instance
5281 return vm_resume($vmid);
5284 PVE::QemuConfig->check_lock($conf)
5285 if !($params->{skiplock} || $has_suspended_lock);
5287 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5289 die "VM $vmid already running\n" if $running;
5291 if (my $storagemap = $migrate_opts->{storagemap}) {
5292 my $replicated = $migrate_opts->{replicated_volumes};
5293 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5294 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5296 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5297 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5301 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5307 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
5308 # skiplock => 0/1, skip checking for config lock
5309 # skiptemplate => 0/1, skip checking whether VM is template
5310 # forcemachine => to force Qemu machine (rollback/migration)
5311 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
5312 # timeout => in seconds
5313 # paused => start VM in paused state (backup)
5314 # resume => resume from hibernation
5325 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5326 # migratedfrom => source node
5327 # spice_ticket => used for spice migration, passed via tunnel/stdin
5328 # network => CIDR of migration network
5329 # type => secure/insecure - tunnel over encrypted connection or plain-text
5330 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5331 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5332 sub vm_start_nolock {
5333 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5335 my $statefile = $params->{statefile};
5336 my $resume = $params->{resume};
5338 my $migratedfrom = $migrate_opts->{migratedfrom};
5339 my $migration_type = $migrate_opts->{type};
5343 # clean up leftover reboot request files
5344 eval { clear_reboot_request($vmid); };
5347 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5348 vmconfig_apply_pending($vmid, $conf, $storecfg);
5349 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5352 # don't regenerate the ISO
if the VM
is started as part of a live migration
5353 # this way we can reuse the old ISO with the correct config
5354 PVE
::QemuServer
::Cloudinit
::generate_cloudinitconfig
($conf, $vmid) if !$migratedfrom;
5356 my $defaults = load_defaults
();
5358 # set environment variable useful inside network script
5359 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom if $migratedfrom;
5361 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5363 my $forcemachine = $params->{forcemachine
};
5364 my $forcecpu = $params->{forcecpu
};
5366 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5367 $forcemachine = $conf->{runningmachine
};
5368 $forcecpu = $conf->{runningcpu
};
5369 print "Resuming suspended VM\n";
5372 my ($cmd, $vollist, $spice_port) = config_to_command
($storecfg, $vmid,
5373 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5376 my $get_migration_ip = sub {
5377 my ($nodename) = @_;
5379 return $migration_ip if defined($migration_ip);
5381 my $cidr = $migrate_opts->{network
};
5383 if (!defined($cidr)) {
5384 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5385 $cidr = $dc_conf->{migration
}->{network
};
5388 if (defined($cidr)) {
5389 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5391 die "could not get IP: no address configured on local " .
5392 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5394 die "could not get IP: multiple addresses configured on local " .
5395 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5397 $migration_ip = @$ips[0];
5400 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5401 if !defined($migration_ip);
5403 return $migration_ip;
5408 if ($statefile eq 'tcp') {
5409 my $localip = "localhost";
5410 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5411 my $nodename = nodename
();
5413 if (!defined($migration_type)) {
5414 if (defined($datacenterconf->{migration
}->{type
})) {
5415 $migration_type = $datacenterconf->{migration
}->{type
};
5417 $migration_type = 'secure';
5421 if ($migration_type eq 'insecure') {
5422 $localip = $get_migration_ip->($nodename);
5423 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5426 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5427 my $migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5428 $migrate_uri = "tcp:${localip}:${migrate_port}";
5429 push @$cmd, '-incoming', $migrate_uri;
5432 } elsif ($statefile eq 'unix') {
5433 # should be default for secure migrations as a ssh TCP forward
5434 # tunnel is not deterministic reliable ready and fails regurarly
5435 # to set up in time, so use UNIX socket forwards
5436 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5437 unlink $socket_addr;
5439 $migrate_uri = "unix:$socket_addr";
5441 push @$cmd, '-incoming', $migrate_uri;
5444 } elsif (-e
$statefile) {
5445 push @$cmd, '-loadstate', $statefile;
5447 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5448 push @$vollist, $statefile;
5449 push @$cmd, '-loadstate', $statepath;
5451 } elsif ($params->{paused
}) {
5455 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $resume);
5457 my $pci_devices = {}; # host pci devices
5458 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
5459 my $dev = $conf->{"hostpci$i"} or next;
5460 $pci_devices->{$i} = parse_hostpci
($dev);
5463 my $pci_id_list = [ map { $_->{id
} } map { $_->{pciid
}->@* } values $pci_devices->%* ];
5464 # reserve all PCI IDs before actually doing anything with them
5465 PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_id_list, $vmid, $start_timeout);
5468 for my $id (sort keys %$pci_devices) {
5469 my $d = $pci_devices->{$id};
5470 for my $dev ($d->{pciid
}->@*) {
5471 PVE
::QemuServer
::PCI
::prepare_pci_device
($vmid, $dev->{id
}, $id, $d->{mdev
});
5476 eval { PVE
::QemuServer
::PCI
::remove_pci_reservation
($pci_id_list) };
5481 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5484 run_command
(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc
=> sub{}, errfunc
=> sub{});
5486 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5487 # timeout should be more than enough here...
5488 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 5);
5490 my $cpuunits = get_cpuunits
($conf);
5493 timeout
=> $statefile ?
undef : $start_timeout,
5498 # when migrating, prefix QEMU output so other side can pick up any
5499 # errors that might occur and show the user
5500 if ($migratedfrom) {
5501 $run_params{quiet
} = 1;
5502 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5505 my %systemd_properties = (
5506 Slice
=> 'qemu.slice',
5507 KillMode
=> 'process',
5509 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5512 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5513 $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5514 $systemd_properties{CPUWeight
} = $cpuunits;
5516 $systemd_properties{CPUShares
} = $cpuunits;
5519 if (my $cpulimit = $conf->{cpulimit
}) {
5520 $systemd_properties{CPUQuota
} = int($cpulimit * 100);
5522 $systemd_properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5524 my $run_qemu = sub {
5525 PVE
::Tools
::run_fork
sub {
5526 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5529 if (my $tpm = $conf->{tpmstate0
}) {
5530 # start the TPM emulator so QEMU can connect on start
5531 $tpmpid = start_swtpm
($storecfg, $vmid, $tpm, $migratedfrom);
5534 my $exitcode = run_command
($cmd, %run_params);
5537 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5538 kill 'TERM', $tpmpid;
5540 die "QEMU exited with code $exitcode\n";
5545 if ($conf->{hugepages
}) {
5548 my $hugepages_topology = PVE
::QemuServer
::Memory
::hugepages_topology
($conf);
5549 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5551 PVE
::QemuServer
::Memory
::hugepages_mount
();
5552 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5554 eval { $run_qemu->() };
5556 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5557 if !$conf->{keephugepages
};
5561 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5562 if !$conf->{keephugepages
};
5564 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5567 eval { $run_qemu->() };
5571 # deactivate volumes if start fails
5572 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5573 eval { PVE
::QemuServer
::PCI
::remove_pci_reservation
($pci_id_list) };
5575 die "start failed: $err";
5578 # re-reserve all PCI IDs now that we can know the actual VM PID
5579 my $pid = PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
5580 eval { PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_id_list, $vmid, undef, $pid) };
5583 print "migration listens on $migrate_uri\n" if $migrate_uri;
5584 $res->{migrate_uri
} = $migrate_uri;
5586 if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
5587 eval { mon_cmd
($vmid, "cont"); };
5591 #start nbd server for storage migration
5592 if (my $nbd = $migrate_opts->{nbd
}) {
5593 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5595 my $migrate_storage_uri;
5596 # nbd_protocol_version > 0 for unix socket support
5597 if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5598 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5599 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
5600 $migrate_storage_uri = "nbd:unix:$socket_path";
5602 my $nodename = nodename
();
5603 my $localip = $get_migration_ip->($nodename);
5604 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5605 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5607 mon_cmd
($vmid, "nbd-server-start", addr
=> {
5610 host
=> "${localip}",
5611 port
=> "${storage_migrate_port}",
5614 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5615 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5618 $res->{migrate_storage_uri
} = $migrate_storage_uri;
5620 foreach my $opt (sort keys %$nbd) {
5621 my $drivestr = $nbd->{$opt}->{drivestr
};
5622 my $volid = $nbd->{$opt}->{volid
};
5623 mon_cmd
($vmid, "nbd-server-add", device
=> "drive-$opt", writable
=> JSON
::true
);
5624 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5625 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5626 print "re-using replicated volume: $opt - $volid\n"
5627 if $nbd->{$opt}->{replicated
};
5629 $res->{drives
}->{$opt} = $nbd->{$opt};
5630 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
5634 if ($migratedfrom) {
5636 set_migration_caps
($vmid);
5641 print "spice listens on port $spice_port\n";
5642 $res->{spice_port
} = $spice_port;
5643 if ($migrate_opts->{spice_ticket
}) {
5644 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
5645 $migrate_opts->{spice_ticket
});
5646 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
5651 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
5652 if !$statefile && $conf->{balloon
};
5654 foreach my $opt (keys %$conf) {
5655 next if $opt !~ m/^net\d+$/;
5656 my $nicconf = parse_net
($conf->{$opt});
5657 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
5661 mon_cmd
($vmid, 'qom-set',
5662 path
=> "machine/peripheral/balloon0",
5663 property
=> "guest-stats-polling-interval",
5664 value
=> 2) if (!defined($conf->{balloon
}) || $conf->{balloon
});
5667 print "Resumed VM, removing state\n";
5668 if (my $vmstate = $conf->{vmstate
}) {
5669 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5670 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5672 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5673 PVE
::QemuConfig-
>write_config($vmid, $conf);
5676 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5681 sub vm_commandline
{
5682 my ($storecfg, $vmid, $snapname) = @_;
5684 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5686 my ($forcemachine, $forcecpu);
5688 my $snapshot = $conf->{snapshots
}->{$snapname};
5689 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5691 # check for machine or CPU overrides in snapshot
5692 $forcemachine = $snapshot->{runningmachine
};
5693 $forcecpu = $snapshot->{runningcpu
};
5695 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5700 my $defaults = load_defaults
();
5702 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5704 return PVE
::Tools
::cmd2string
($cmd);
5708 my ($vmid, $skiplock) = @_;
5710 PVE
::QemuConfig-
>lock_config($vmid, sub {
5712 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5714 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5716 mon_cmd
($vmid, "system_reset");
5720 sub get_vm_volumes
{
5724 foreach_volid
($conf, sub {
5725 my ($volid, $attr) = @_;
5727 return if $volid =~ m
|^/|;
5729 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5732 push @$vollist, $volid;
5738 sub vm_stop_cleanup
{
5739 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5744 my $vollist = get_vm_volumes
($conf);
5745 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5747 if (my $tpmdrive = $conf->{tpmstate0
}) {
5748 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
5749 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
5751 PVE
::Storage
::unmap_volume
($storecfg, $tpm->{file
});
5756 foreach my $ext (qw(mon qmp pid vnc qga)) {
5757 unlink "/var/run/qemu-server/${vmid}.$ext";
5760 if ($conf->{ivshmem
}) {
5761 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5762 # just delete it for now, VMs which have this already open do not
5763 # are affected, but new VMs will get a separated one. If this
5764 # becomes an issue we either add some sort of ref-counting or just
5765 # add a "don't delete on stop" flag to the ivshmem format.
5766 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5770 foreach my $key (keys %$conf) {
5771 next if $key !~ m/^hostpci(\d+)$/;
5772 my $hostpciindex = $1;
5773 my $d = parse_hostpci
($conf->{$key});
5774 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5776 foreach my $pci (@{$d->{pciid
}}) {
5777 my $pciid = $pci->{id
};
5778 push @$ids, $pci->{id
};
5779 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5782 PVE
::QemuServer
::PCI
::remove_pci_reservation
($ids);
5784 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5786 warn $@ if $@; # avoid errors - just warn
5789 # call only in locked context
5791 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5793 my $pid = check_running
($vmid, $nocheck);
5798 $conf = PVE
::QemuConfig-
>load_config($vmid);
5799 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5800 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5801 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5802 $timeout = $opts->{down
} if $opts->{down
};
5804 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5809 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
5810 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5812 mon_cmd
($vmid, "system_powerdown");
5815 mon_cmd
($vmid, "quit");
5821 $timeout = 60 if !defined($timeout);
5824 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5829 if ($count >= $timeout) {
5831 warn "VM still running - terminating now with SIGTERM\n";
5834 die "VM quit/powerdown failed - got timeout\n";
5837 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5841 if (!check_running
($vmid, $nocheck)) {
5842 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5846 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5849 die "VM quit/powerdown failed\n";
5857 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5862 if ($count >= $timeout) {
5863 warn "VM still running - terminating now with SIGKILL\n";
5868 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5871 # Note: use $nocheck to skip tests if VM configuration file exists.
5872 # We need that when migration VMs to other nodes (files already moved)
5873 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5875 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5877 $force = 1 if !defined($force) && !$shutdown;
5880 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5881 kill 15, $pid if $pid;
5882 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5883 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5887 PVE
::QemuConfig-
>lock_config($vmid, sub {
5888 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5893 my ($vmid, $timeout) = @_;
5895 PVE
::QemuConfig-
>lock_config($vmid, sub {
5898 # only reboot if running, as qmeventd starts it again on a stop event
5899 return if !check_running
($vmid);
5901 create_reboot_request
($vmid);
5903 my $storecfg = PVE
::Storage
::config
();
5904 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5908 # avoid that the next normal shutdown will be confused for a reboot
5909 clear_reboot_request
($vmid);
5915 # note: if using the statestorage parameter, the caller has to check privileges
5917 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5924 PVE
::QemuConfig-
>lock_config($vmid, sub {
5926 $conf = PVE
::QemuConfig-
>load_config($vmid);
5928 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5929 PVE
::QemuConfig-
>check_lock($conf)
5930 if !($skiplock || $is_backing_up);
5932 die "cannot suspend to disk during backup\n"
5933 if $is_backing_up && $includestate;
5935 if ($includestate) {
5936 $conf->{lock} = 'suspending';
5937 my $date = strftime
("%Y-%m-%d", localtime(time()));
5938 $storecfg = PVE
::Storage
::config
();
5939 if (!$statestorage) {
5940 $statestorage = find_vmstate_storage
($conf, $storecfg);
5941 # check permissions for the storage
5942 my $rpcenv = PVE
::RPCEnvironment
::get
();
5943 if ($rpcenv->{type
} ne 'cli') {
5944 my $authuser = $rpcenv->get_user();
5945 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5950 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
5951 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5952 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5953 PVE
::QemuConfig-
>write_config($vmid, $conf);
5955 mon_cmd
($vmid, "stop");
5959 if ($includestate) {
5961 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5964 set_migration_caps
($vmid, 1);
5965 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5967 my $state = mon_cmd
($vmid, "query-savevm");
5968 if (!$state->{status
}) {
5969 die "savevm not active\n";
5970 } elsif ($state->{status
} eq 'active') {
5973 } elsif ($state->{status
} eq 'completed') {
5974 print "State saved, quitting\n";
5976 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5977 die "query-savevm failed with error '$state->{error}'\n"
5979 die "query-savevm returned status '$state->{status}'\n";
5985 PVE
::QemuConfig-
>lock_config($vmid, sub {
5986 $conf = PVE
::QemuConfig-
>load_config($vmid);
5988 # cleanup, but leave suspending lock, to indicate something went wrong
5990 mon_cmd
($vmid, "savevm-end");
5991 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5992 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5993 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5994 PVE
::QemuConfig-
>write_config($vmid, $conf);
6000 die "lock changed unexpectedly\n"
6001 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
6003 mon_cmd
($vmid, "quit");
6004 $conf->{lock} = 'suspended';
6005 PVE
::QemuConfig-
>write_config($vmid, $conf);
6011 my ($vmid, $skiplock, $nocheck) = @_;
6013 PVE
::QemuConfig-
>lock_config($vmid, sub {
6014 my $res = mon_cmd
($vmid, 'query-status');
6015 my $resume_cmd = 'cont';
6018 if ($res->{status
}) {
6019 return if $res->{status
} eq 'running'; # job done, go home
6020 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
6021 $reset = 1 if $res->{status
} eq 'shutdown';
6026 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6028 PVE
::QemuConfig-
>check_lock($conf)
6029 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
6033 # required if a VM shuts down during a backup and we get a resume
6034 # request before the backup finishes for example
6035 mon_cmd
($vmid, "system_reset");
6037 mon_cmd
($vmid, $resume_cmd);
6042 my ($vmid, $skiplock, $key) = @_;
6044 PVE
::QemuConfig-
>lock_config($vmid, sub {
6046 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6048 # there is no qmp command, so we use the human monitor command
6049 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
6050 die $res if $res ne '';
6054 # vzdump restore implementaion
6056 sub tar_archive_read_firstfile
{
6057 my $archive = shift;
6059 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
6061 # try to detect archive type first
6062 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6063 die "unable to open file '$archive'\n";
6064 my $firstfile = <$fh>;
6068 die "ERROR: archive contaions no data\n" if !$firstfile;
6074 sub tar_restore_cleanup
{
6075 my ($storecfg, $statfile) = @_;
6077 print STDERR
"starting cleanup\n";
6079 if (my $fd = IO
::File-
>new($statfile, "r")) {
6080 while (defined(my $line = <$fd>)) {
6081 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6084 if ($volid =~ m
|^/|) {
6085 unlink $volid || die 'unlink failed\n';
6087 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6089 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6091 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6093 print STDERR
"unable to parse line in statfile - $line";
6100 sub restore_file_archive
{
6101 my ($archive, $vmid, $user, $opts) = @_;
6103 return restore_vma_archive
($archive, $vmid, $user, $opts)
6106 my $info = PVE
::Storage
::archive_info
($archive);
6107 my $format = $opts->{format
} // $info->{format
};
6108 my $comp = $info->{compression
};
6110 # try to detect archive format
6111 if ($format eq 'tar') {
6112 return restore_tar_archive
($archive, $vmid, $user, $opts);
6114 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
6118 # hepler to remove disks that will not be used after restore
6119 my $restore_cleanup_oldconf = sub {
6120 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6122 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
6123 my ($ds, $drive) = @_;
6125 return if drive_is_cdrom
($drive, 1);
6127 my $volid = $drive->{file
};
6128 return if !$volid || $volid =~ m
|^/|;
6130 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
6131 return if !$path || !$owner || ($owner != $vmid);
6133 # Note: only delete disk we want to restore
6134 # other volumes will become unused
6135 if ($virtdev_hash->{$ds}) {
6136 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
6143 # delete vmstate files, after the restore we have no snapshots anymore
6144 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
6145 my $snap = $oldconf->{snapshots
}->{$snapname};
6146 if ($snap->{vmstate
}) {
6147 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
6155 # Helper to parse vzdump backup device hints
6157 # $rpcenv: Environment, used to ckeck storage permissions
6158 # $user: User ID, to check storage permissions
6159 # $storecfg: Storage configuration
6160 # $fh: the file handle for reading the configuration
6161 # $devinfo: should contain device sizes for all backu-up'ed devices
6162 # $options: backup options (pool, default storage)
6164 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6165 my $parse_backup_hints = sub {
6166 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6168 my $virtdev_hash = {};
6170 while (defined(my $line = <$fh>)) {
6171 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6172 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6173 die "archive does not contain data for drive '$virtdev'\n"
6174 if !$devinfo->{$devname};
6176 if (defined($options->{storage
})) {
6177 $storeid = $options->{storage
} || 'local';
6178 } elsif (!$storeid) {
6181 $format = 'raw' if !$format;
6182 $devinfo->{$devname}->{devname
} = $devname;
6183 $devinfo->{$devname}->{virtdev
} = $virtdev;
6184 $devinfo->{$devname}->{format
} = $format;
6185 $devinfo->{$devname}->{storeid
} = $storeid;
6187 # check permission on storage
6188 my $pool = $options->{pool
}; # todo: do we need that?
6189 if ($user ne 'root@pam') {
6190 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6193 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6194 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6196 my $drive = parse_drive
($virtdev, $2);
6197 if (drive_is_cloudinit
($drive)) {
6198 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6199 $storeid = $options->{storage
} if defined ($options->{storage
});
6200 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6201 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6203 $virtdev_hash->{$virtdev} = {
6205 storeid
=> $storeid,
6206 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6213 return $virtdev_hash;
6216 # Helper to allocate and activate all volumes required for a restore
6218 # $storecfg: Storage configuration
6219 # $virtdev_hash: as returned by parse_backup_hints()
6221 # Returns: { $virtdev => $volid }
6222 my $restore_allocate_devices = sub {
6223 my ($storecfg, $virtdev_hash, $vmid) = @_;
6226 foreach my $virtdev (sort keys %$virtdev_hash) {
6227 my $d = $virtdev_hash->{$virtdev};
6228 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6229 my $storeid = $d->{storeid
};
6230 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6232 # test if requested format is supported
6233 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6234 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6235 $d->{format
} = $defFormat if !$supported;
6238 if ($d->{is_cloudinit
}) {
6239 $name = "vm-$vmid-cloudinit";
6240 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6241 if ($scfg->{path
}) {
6242 $name .= ".$d->{format}";
6246 my $volid = PVE
::Storage
::vdisk_alloc
(
6247 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6249 print STDERR
"new volume ID is '$volid'\n";
6250 $d->{volid
} = $volid;
6252 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6254 $map->{$virtdev} = $volid;
6260 sub restore_update_config_line
{
6261 my ($cookie, $map, $line, $unique) = @_;
6263 return '' if $line =~ m/^\#qmdump\#/;
6264 return '' if $line =~ m/^\#vzdump\#/;
6265 return '' if $line =~ m/^lock:/;
6266 return '' if $line =~ m/^unused\d+:/;
6267 return '' if $line =~ m/^parent:/;
6271 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6272 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6273 # try to convert old 1.X settings
6274 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6275 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6276 my ($model, $macaddr) = split(/\=/, $devconfig);
6277 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6280 bridge
=> "vmbr$ind",
6281 macaddr
=> $macaddr,
6283 my $netstr = print_net
($net);
6285 $res .= "net$cookie->{netcount}: $netstr\n";
6286 $cookie->{netcount
}++;
6288 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6289 my ($id, $netstr) = ($1, $2);
6290 my $net = parse_net
($netstr);
6291 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6292 $netstr = print_net
($net);
6293 $res .= "$id: $netstr\n";
6294 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6297 my $di = parse_drive
($virtdev, $value);
6298 if (defined($di->{backup
}) && !$di->{backup
}) {
6300 } elsif ($map->{$virtdev}) {
6301 delete $di->{format
}; # format can change on restore
6302 $di->{file
} = $map->{$virtdev};
6303 $value = print_drive
($di);
6304 $res .= "$virtdev: $value\n";
6308 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6310 if ($vmgenid ne '0') {
6311 # always generate a new vmgenid if there was a valid one setup
6312 $vmgenid = generate_uuid
();
6314 $res .= "vmgenid: $vmgenid\n";
6315 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6316 my ($uuid, $uuid_str);
6317 UUID
::generate
($uuid);
6318 UUID
::unparse
($uuid, $uuid_str);
6319 my $smbios1 = parse_smbios1
($2);
6320 $smbios1->{uuid
} = $uuid_str;
6321 $res .= $1.print_smbios1
($smbios1)."\n";
6329 my $restore_deactivate_volumes = sub {
6330 my ($storecfg, $devinfo) = @_;
6333 foreach my $devname (keys %$devinfo) {
6334 my $volid = $devinfo->{$devname}->{volid
};
6335 push @$vollist, $volid if $volid;
6338 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6341 my $restore_destroy_volumes = sub {
6342 my ($storecfg, $devinfo) = @_;
6344 foreach my $devname (keys %$devinfo) {
6345 my $volid = $devinfo->{$devname}->{volid
};
6348 if ($volid =~ m
|^/|) {
6349 unlink $volid || die 'unlink failed\n';
6351 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6353 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6355 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6360 my ($cfg, $vmid) = @_;
6362 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6364 my $volid_hash = {};
6365 foreach my $storeid (keys %$info) {
6366 foreach my $item (@{$info->{$storeid}}) {
6367 next if !($item->{volid
} && $item->{size
});
6368 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6369 $volid_hash->{$item->{volid
}} = $item;
6376 sub update_disk_config
{
6377 my ($vmid, $conf, $volid_hash) = @_;
6380 my $prefix = "VM $vmid";
6382 # used and unused disks
6383 my $referenced = {};
6385 # Note: it is allowed to define multiple storages with same path (alias), so
6386 # we need to check both 'volid' and real 'path' (two different volid can point
6387 # to the same path).
6389 my $referencedpath = {};
6392 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6393 my ($opt, $drive) = @_;
6395 my $volid = $drive->{file
};
6397 my $volume = $volid_hash->{$volid};
6399 # mark volid as "in-use" for next step
6400 $referenced->{$volid} = 1;
6401 if ($volume && (my $path = $volume->{path
})) {
6402 $referencedpath->{$path} = 1;
6405 return if drive_is_cdrom
($drive);
6408 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6409 if (defined($updated)) {
6411 $conf->{$opt} = print_drive
($updated);
6412 print "$prefix ($opt): $msg\n";
6416 # remove 'unusedX' entry if volume is used
6417 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6418 my ($opt, $drive) = @_;
6420 my $volid = $drive->{file
};
6424 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6425 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6426 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6428 delete $conf->{$opt};
6431 $referenced->{$volid} = 1;
6432 $referencedpath->{$path} = 1 if $path;
6435 foreach my $volid (sort keys %$volid_hash) {
6436 next if $volid =~ m/vm-$vmid-state-/;
6437 next if $referenced->{$volid};
6438 my $path = $volid_hash->{$volid}->{path
};
6439 next if !$path; # just to be sure
6440 next if $referencedpath->{$path};
6442 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6443 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6444 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6451 my ($vmid, $nolock, $dryrun) = @_;
6453 my $cfg = PVE
::Storage
::config
();
6455 print "rescan volumes...\n";
6456 my $volid_hash = scan_volids
($cfg, $vmid);
6458 my $updatefn = sub {
6461 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6463 PVE
::QemuConfig-
>check_lock($conf);
6466 foreach my $volid (keys %$volid_hash) {
6467 my $info = $volid_hash->{$volid};
6468 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6471 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6473 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6476 if (defined($vmid)) {
6480 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6483 my $vmlist = config_list
();
6484 foreach my $vmid (keys %$vmlist) {
6488 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6494 sub restore_proxmox_backup_archive
{
6495 my ($archive, $vmid, $user, $options) = @_;
6497 my $storecfg = PVE
::Storage
::config
();
6499 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6500 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6502 my $fingerprint = $scfg->{fingerprint
};
6503 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6505 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6507 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6508 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6509 local $ENV{PBS_PASSWORD
} = $password;
6510 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6512 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6513 PVE
::Storage
::parse_volname
($storecfg, $archive);
6515 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6517 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6519 my $tmpdir = "/var/tmp/vzdumptmp$$";
6523 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6524 # disable interrupts (always do cleanups)
6528 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6530 # Note: $oldconf is undef if VM does not exists
6531 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6532 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6533 my $new_conf_raw = '';
6535 my $rpcenv = PVE
::RPCEnvironment
::get
();
6544 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6546 my $cfgfn = "$tmpdir/qemu-server.conf";
6547 my $firewall_config_fn = "$tmpdir/fw.conf";
6548 my $index_fn = "$tmpdir/index.json";
6550 my $cmd = "restore";
6552 my $param = [$pbs_backup_name, "index.json", $index_fn];
6553 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6554 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6555 $index = decode_json
($index);
6557 # print Dumper($index);
6558 foreach my $info (@{$index->{files
}}) {
6559 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6561 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6562 $devinfo->{$devname}->{size
} = $1;
6564 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6569 my $is_qemu_server_backup = scalar(
6570 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6572 if (!$is_qemu_server_backup) {
6573 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6575 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6577 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6578 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6580 if ($has_firewall_config) {
6581 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6582 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6584 my $pve_firewall_dir = '/etc/pve/firewall';
6585 mkdir $pve_firewall_dir; # make sure the dir exists
6586 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6589 my $fh = IO
::File-
>new($cfgfn, "r") ||
6590 die "unable to read qemu-server.conf - $!\n";
6592 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6594 # fixme: rate limit?
6596 # create empty/temp config
6597 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6599 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6602 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6604 foreach my $virtdev (sort keys %$virtdev_hash) {
6605 my $d = $virtdev_hash->{$virtdev};
6606 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6608 # this fails if storage is unavailable
6609 my $volid = $d->{volid
};
6610 my $path = PVE
::Storage
::path
($storecfg, $volid);
6612 # for live-restore we only want to preload the efidisk and TPM state
6613 next if $options->{live
} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6615 my $pbs_restore_cmd = [
6616 '/usr/bin/pbs-restore',
6617 '--repository', $repo,
6619 "$d->{devname}.img.fidx",
6624 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6625 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6627 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6628 push @$pbs_restore_cmd, '--skip-zero';
6631 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6632 print "restore proxmox backup image: $dbg_cmdstring\n";
6633 run_command
($pbs_restore_cmd);
6636 $fh->seek(0, 0) || die "seek failed - $!\n";
6638 my $cookie = { netcount
=> 0 };
6639 while (defined(my $line = <$fh>)) {
6640 $new_conf_raw .= restore_update_config_line
(
6652 if ($err || !$options->{live
}) {
6653 $restore_deactivate_volumes->($storecfg, $devinfo);
6659 $restore_destroy_volumes->($storecfg, $devinfo);
6663 if ($options->{live
}) {
6664 # keep lock during live-restore
6665 $new_conf_raw .= "\nlock: create";
6668 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6670 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6672 eval { rescan
($vmid, 1); };
6675 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
6677 if ($options->{live
}) {
6683 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
6685 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6686 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
6688 # these special drives are already restored before start
6689 delete $devinfo->{'drive-efidisk0'};
6690 delete $devinfo->{'drive-tpmstate0-backup'};
6691 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6693 PVE
::QemuConfig-
>remove_lock($vmid, "create");
6697 sub pbs_live_restore
{
6698 my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6700 print "starting VM for live-restore\n";
6701 print "repository: '$repo', snapshot: '$snap'\n";
6703 my $pbs_backing = {};
6704 for my $ds (keys %$restored_disks) {
6705 $ds =~ m/^drive-(.*)$/;
6707 $pbs_backing->{$confname} = {
6708 repository
=> $repo,
6710 archive
=> "$ds.img.fidx",
6712 $pbs_backing->{$confname}->{keyfile
} = $keyfile if -e
$keyfile;
6714 my $drive = parse_drive
($confname, $conf->{$confname});
6715 print "restoring '$ds' to '$drive->{file}'\n";
6718 my $drives_streamed = 0;
6720 # make sure HA doesn't interrupt our restore by stopping the VM
6721 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
6722 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
6725 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6726 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6727 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'pbs-backing' => $pbs_backing}, {});
6729 my $qmeventd_fd = register_qmeventd_handle
($vmid);
6731 # begin streaming, i.e. data copy from PBS to target disk for every vol,
6732 # this will effectively collapse the backing image chain consisting of
6733 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6734 # removes itself once all backing images vanish with 'auto-remove=on')
6736 for my $ds (sort keys %$restored_disks) {
6737 my $job_id = "restore-$ds";
6738 mon_cmd
($vmid, 'block-stream',
6739 'job-id' => $job_id,
6742 $jobs->{$job_id} = {};
6745 mon_cmd
($vmid, 'cont');
6746 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
6748 print "restore-drive jobs finished successfully, removing all tracking block devices"
6749 ." to disconnect from Proxmox Backup Server\n";
6751 for my $ds (sort keys %$restored_disks) {
6752 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6755 close($qmeventd_fd);
6761 warn "An error occured during live-restore: $err\n";
6762 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
6763 die "live-restore failed\n";
6767 sub restore_vma_archive
{
6768 my ($archive, $vmid, $user, $opts, $comp) = @_;
6770 my $readfrom = $archive;
6772 my $cfg = PVE
::Storage
::config
();
6774 my $bwlimit = $opts->{bwlimit
};
6776 my $dbg_cmdstring = '';
6777 my $add_pipe = sub {
6779 push @$commands, $cmd;
6780 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6781 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6786 if ($archive eq '-') {
6789 # If we use a backup from a PVE defined storage we also consider that
6790 # storage's rate limit:
6791 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6792 if (defined($volid)) {
6793 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6794 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6796 print STDERR
"applying read rate limit: $readlimit\n";
6797 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6798 $add_pipe->($cstream);
6804 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6805 my $cmd = $info->{decompressor
};
6806 push @$cmd, $readfrom;
6810 my $tmpdir = "/var/tmp/vzdumptmp$$";
6813 # disable interrupts (always do cleanups)
6817 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6819 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6820 POSIX
::mkfifo
($mapfifo, 0600);
6822 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6824 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6831 my $rpcenv = PVE
::RPCEnvironment
::get
();
6833 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6835 # Note: $oldconf is undef if VM does not exist
6836 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6837 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6838 my $new_conf_raw = '';
6842 my $print_devmap = sub {
6843 my $cfgfn = "$tmpdir/qemu-server.conf";
6845 # we can read the config - that is already extracted
6846 my $fh = IO
::File-
>new($cfgfn, "r") ||
6847 die "unable to read qemu-server.conf - $!\n";
6849 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6851 my $pve_firewall_dir = '/etc/pve/firewall';
6852 mkdir $pve_firewall_dir; # make sure the dir exists
6853 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6856 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6858 foreach my $info (values %{$virtdev_hash}) {
6859 my $storeid = $info->{storeid
};
6860 next if defined($storage_limits{$storeid});
6862 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
6863 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6864 $storage_limits{$storeid} = $limit * 1024;
6867 foreach my $devname (keys %$devinfo) {
6868 die "found no device mapping information for device '$devname'\n"
6869 if !$devinfo->{$devname}->{virtdev
};
6872 # create empty/temp config
6874 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6875 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6879 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6881 # print restore information to $fifofh
6882 foreach my $virtdev (sort keys %$virtdev_hash) {
6883 my $d = $virtdev_hash->{$virtdev};
6884 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6886 my $storeid = $d->{storeid
};
6887 my $volid = $d->{volid
};
6890 if (my $limit = $storage_limits{$storeid}) {
6891 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6894 my $write_zeros = 1;
6895 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6899 my $path = PVE
::Storage
::path
($cfg, $volid);
6901 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6903 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6906 $fh->seek(0, 0) || die "seek failed - $!\n";
6908 my $cookie = { netcount
=> 0 };
6909 while (defined(my $line = <$fh>)) {
6910 $new_conf_raw .= restore_update_config_line
(
6927 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6928 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6930 $oldtimeout = alarm($timeout);
6937 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6938 my ($dev_id, $size, $devname) = ($1, $2, $3);
6939 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6940 } elsif ($line =~ m/^CTIME: /) {
6941 # we correctly received the vma config, so we can disable
6942 # the timeout now for disk allocation (set to 10 minutes, so
6943 # that we always timeout if something goes wrong)
6946 print $fifofh "done\n";
6947 my $tmp = $oldtimeout || 0;
6948 $oldtimeout = undef;
6955 print "restore vma archive: $dbg_cmdstring\n";
6956 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6960 alarm($oldtimeout) if $oldtimeout;
6962 $restore_deactivate_volumes->($cfg, $devinfo);
6964 close($fifofh) if $fifofh;
6969 $restore_destroy_volumes->($cfg, $devinfo);
6973 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6975 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6977 eval { rescan
($vmid, 1); };
6980 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
6983 sub restore_tar_archive
{
6984 my ($archive, $vmid, $user, $opts) = @_;
6986 if ($archive ne '-') {
6987 my $firstfile = tar_archive_read_firstfile
($archive);
6988 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6989 if $firstfile ne 'qemu-server.conf';
6992 my $storecfg = PVE
::Storage
::config
();
6994 # avoid zombie disks when restoring over an existing VM -> cleanup first
6995 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6996 # skiplock=1 because qmrestore has set the 'create' lock itself already
6997 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6998 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
7000 my $tocmd = "/usr/lib/qemu-server/qmextract";
7002 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
7003 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
7004 $tocmd .= ' --prealloc' if $opts->{prealloc
};
7005 $tocmd .= ' --info' if $opts->{info
};
7007 # tar option "xf" does not autodetect compression when read from STDIN,
7008 # so we pipe to zcat
7009 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
7010 PVE
::Tools
::shellquote
("--to-command=$tocmd");
7012 my $tmpdir = "/var/tmp/vzdumptmp$$";
7015 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
7016 local $ENV{VZDUMP_VMID
} = $vmid;
7017 local $ENV{VZDUMP_USER
} = $user;
7019 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7020 my $new_conf_raw = '';
7022 # disable interrupts (always do cleanups)
7026 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7034 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7036 if ($archive eq '-') {
7037 print "extracting archive from STDIN\n";
7038 run_command
($cmd, input
=> "<&STDIN");
7040 print "extracting archive '$archive'\n";
7044 return if $opts->{info
};
7048 my $statfile = "$tmpdir/qmrestore.stat";
7049 if (my $fd = IO
::File-
>new($statfile, "r")) {
7050 while (defined (my $line = <$fd>)) {
7051 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7052 $map->{$1} = $2 if $1;
7054 print STDERR
"unable to parse line in statfile - $line\n";
7060 my $confsrc = "$tmpdir/qemu-server.conf";
7062 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7064 my $cookie = { netcount
=> 0 };
7065 while (defined (my $line = <$srcfd>)) {
7066 $new_conf_raw .= restore_update_config_line
(
7077 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
7083 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7085 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7087 eval { rescan
($vmid, 1); };
7091 sub foreach_storage_used_by_vm
{
7092 my ($conf, $func) = @_;
7096 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7097 my ($ds, $drive) = @_;
7098 return if drive_is_cdrom
($drive);
7100 my $volid = $drive->{file
};
7102 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
7103 $sidhash->{$sid} = $sid if $sid;
7106 foreach my $sid (sort keys %$sidhash) {
7111 my $qemu_snap_storage = {
7114 sub do_snapshots_with_qemu
{
7115 my ($storecfg, $volid, $deviceid) = @_;
7117 return if $deviceid =~ m/tpmstate0/;
7119 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
7120 my $scfg = $storecfg->{ids
}->{$storage_name};
7121 die "could not find storage '$storage_name'\n" if !defined($scfg);
7123 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
7127 if ($volid =~ m/\.(qcow2|qed)$/){
7134 sub qga_check_running
{
7135 my ($vmid, $nowarn) = @_;
7137 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
7139 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7145 sub template_create
{
7146 my ($vmid, $conf, $disk) = @_;
7148 my $storecfg = PVE
::Storage
::config
();
7150 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7151 my ($ds, $drive) = @_;
7153 return if drive_is_cdrom
($drive);
7154 return if $disk && $ds ne $disk;
7156 my $volid = $drive->{file
};
7157 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
7159 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
7160 $drive->{file
} = $voliddst;
7161 $conf->{$ds} = print_drive
($drive);
7162 PVE
::QemuConfig-
>write_config($vmid, $conf);
7166 sub convert_iscsi_path
{
7169 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
7174 my $initiator_name = get_initiator_name
();
7176 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7177 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7180 die "cannot convert iscsi path '$path', unkown format\n";
7183 sub qemu_img_convert
{
7184 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7186 my $storecfg = PVE
::Storage
::config
();
7187 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
7188 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
7190 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7194 my $src_is_iscsi = 0;
7198 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
7199 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7200 $src_format = qemu_img_format
($src_scfg, $src_volname);
7201 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
7202 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
7203 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
7204 } elsif (-f
$src_volid) {
7205 $src_path = $src_volid;
7206 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7211 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7213 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7214 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7215 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7216 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7219 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7220 push @$cmd, '-l', "snapshot.name=$snapname"
7221 if $snapname && $src_format && $src_format eq "qcow2";
7222 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7223 push @$cmd, '-T', $cachemode if defined($cachemode);
7225 if ($src_is_iscsi) {
7226 push @$cmd, '--image-opts';
7227 $src_path = convert_iscsi_path
($src_path);
7228 } elsif ($src_format) {
7229 push @$cmd, '-f', $src_format;
7232 if ($dst_is_iscsi) {
7233 push @$cmd, '--target-image-opts';
7234 $dst_path = convert_iscsi_path
($dst_path);
7236 push @$cmd, '-O', $dst_format;
7239 push @$cmd, $src_path;
7241 if (!$dst_is_iscsi && $is_zero_initialized) {
7242 push @$cmd, "zeroinit:$dst_path";
7244 push @$cmd, $dst_path;
7249 if($line =~ m/\((\S+)\/100\
%\)/){
7251 my $transferred = int($size * $percent / 100);
7252 my $total_h = render_bytes
($size, 1);
7253 my $transferred_h = render_bytes
($transferred, 1);
7255 print "transferred $transferred_h of $total_h ($percent%)\n";
7260 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7262 die "copy failed: $err" if $err;
7265 sub qemu_img_format
{
7266 my ($scfg, $volname) = @_;
7268 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7275 sub qemu_drive_mirror
{
7276 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7278 $jobs = {} if !$jobs;
7282 $jobs->{"drive-$drive"} = {};
7284 if ($dst_volid =~ /^nbd:/) {
7285 $qemu_target = $dst_volid;
7288 my $storecfg = PVE
::Storage
::config
();
7289 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7291 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7293 $format = qemu_img_format
($dst_scfg, $dst_volname);
7295 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7297 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7300 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7301 $opts->{format
} = $format if $format;
7303 if (defined($src_bitmap)) {
7304 $opts->{sync
} = 'incremental';
7305 $opts->{bitmap
} = $src_bitmap;
7306 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7309 if (defined($bwlimit)) {
7310 $opts->{speed
} = $bwlimit * 1024;
7311 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7313 print "drive mirror is starting for drive-$drive\n";
7316 # if a job already runs for this device we get an error, catch it for cleanup
7317 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7319 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7321 die "mirroring error: $err\n";
7324 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7327 # $completion can be either
7328 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7329 # 'cancel': wait until all jobs are ready, block-job-cancel them
7330 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7331 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7332 sub qemu_drive_mirror_monitor
{
7333 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7335 $completion //= 'complete';
7339 my $err_complete = 0;
7341 my $starttime = time ();
7343 die "block job ('$op') timed out\n" if $err_complete > 300;
7345 my $stats = mon_cmd
($vmid, "query-block-jobs");
7348 my $running_jobs = {};
7349 for my $stat (@$stats) {
7350 next if $stat->{type
} ne $op;
7351 $running_jobs->{$stat->{device
}} = $stat;
7354 my $readycounter = 0;
7356 for my $job_id (sort keys %$jobs) {
7357 my $job = $running_jobs->{$job_id};
7359 my $vanished = !defined($job);
7360 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7361 if($complete || ($vanished && $completion eq 'auto')) {
7362 print "$job_id: $op-job finished\n";
7363 delete $jobs->{$job_id};
7367 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7369 my $busy = $job->{busy
};
7370 my $ready = $job->{ready
};
7371 if (my $total = $job->{len
}) {
7372 my $transferred = $job->{offset
} || 0;
7373 my $remaining = $total - $transferred;
7374 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7376 my $duration = $ctime - $starttime;
7377 my $total_h = render_bytes
($total, 1);
7378 my $transferred_h = render_bytes
($transferred, 1);
7380 my $status = sprintf(
7381 "transferred $transferred_h of $total_h ($percent%%) in %s",
7382 render_duration
($duration),
7387 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7389 $status .= ", ready";
7392 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
7393 $jobs->{$job_id}->{ready
} = $ready;
7396 $readycounter++ if $job->{ready
};
7399 last if scalar(keys %$jobs) == 0;
7401 if ($readycounter == scalar(keys %$jobs)) {
7402 print "all '$op' jobs are ready\n";
7404 # do the complete later (or has already been done)
7405 last if $completion eq 'skip' || $completion eq 'auto';
7407 if ($vmiddst && $vmiddst != $vmid) {
7408 my $agent_running = $qga && qga_check_running
($vmid);
7409 if ($agent_running) {
7410 print "freeze filesystem\n";
7411 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
7413 print "suspend vm\n";
7414 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
7417 # if we clone a disk for a new target vm, we don't switch the disk
7418 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
7420 if ($agent_running) {
7421 print "unfreeze filesystem\n";
7422 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
7424 print "resume vm\n";
7425 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
7431 for my $job_id (sort keys %$jobs) {
7432 # try to switch the disk if source and destination are on the same guest
7433 print "$job_id: Completing block job_id...\n";
7436 if ($completion eq 'complete') {
7437 $op = 'block-job-complete';
7438 } elsif ($completion eq 'cancel') {
7439 $op = 'block-job-cancel';
7441 die "invalid completion value: $completion\n";
7443 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
7444 if ($@ =~ m/cannot be completed/) {
7445 print "$job_id: block job cannot be completed, trying again.\n";
7448 print "$job_id: Completed successfully.\n";
7449 $jobs->{$job_id}->{complete
} = 1;
7460 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7461 die "block job ($op) error: $err";
7465 sub qemu_blockjobs_cancel
{
7466 my ($vmid, $jobs) = @_;
7468 foreach my $job (keys %$jobs) {
7469 print "$job: Cancelling block job\n";
7470 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
7471 $jobs->{$job}->{cancel
} = 1;
7475 my $stats = mon_cmd
($vmid, "query-block-jobs");
7477 my $running_jobs = {};
7478 foreach my $stat (@$stats) {
7479 $running_jobs->{$stat->{device
}} = $stat;
7482 foreach my $job (keys %$jobs) {
7484 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
7485 print "$job: Done.\n";
7486 delete $jobs->{$job};
7490 last if scalar(keys %$jobs) == 0;
7497 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7498 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7503 print "create linked clone of drive $drivename ($drive->{file})\n";
7504 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
7505 push @$newvollist, $newvolid;
7508 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
7509 $storeid = $storage if $storage;
7511 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
7513 print "create full clone of drive $drivename ($drive->{file})\n";
7516 if (drive_is_cloudinit
($drive)) {
7517 $name = "vm-$newvmid-cloudinit";
7518 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7519 if ($scfg->{path
}) {
7520 $name .= ".$dst_format";
7523 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
7524 } elsif ($drivename eq 'efidisk0') {
7525 $size = get_efivars_size
($conf);
7526 } elsif ($drivename eq 'tpmstate0') {
7527 $size = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
7529 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
7531 $newvolid = PVE
::Storage
::vdisk_alloc
(
7532 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7534 push @$newvollist, $newvolid;
7536 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
7538 if (drive_is_cloudinit
($drive)) {
7539 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7540 # if this is the case, we have to complete any block-jobs still there from
7541 # previous drive-mirrors
7542 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7543 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
7548 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
7549 if (!$running || $snapname) {
7550 # TODO: handle bwlimits
7551 if ($drivename eq 'efidisk0') {
7552 # the relevant data on the efidisk may be smaller than the source
7553 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7554 # that is given by the OVMF_VARS.fd
7555 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
7556 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
7558 # better for Ceph if block size is not too small, see bug #3324
7561 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7562 "if=$src_path", "of=$dst_path"]);
7564 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
7568 die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7570 my $kvmver = get_running_qemu_version
($vmid);
7571 if (!min_version
($kvmver, 2, 7)) {
7572 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7573 if $drive->{iothread
};
7576 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7577 $completion, $qga, $bwlimit);
7582 my ($size) = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
7585 $disk->{format
} = undef;
7586 $disk->{file
} = $newvolid;
7587 $disk->{size
} = $size if defined($size);
7592 sub get_running_qemu_version
{
7594 my $res = mon_cmd
($vmid, "query-version");
7595 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7598 sub qemu_use_old_bios_files
{
7599 my ($machine_type) = @_;
7601 return if !$machine_type;
7603 my $use_old_bios_files = undef;
7605 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7607 $use_old_bios_files = 1;
7609 my $version = extract_version
($machine_type, kvm_user_version
());
7610 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7611 # load new efi bios files on migration. So this hack is required to allow
7612 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7613 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7614 $use_old_bios_files = !min_version
($version, 2, 4);
7617 return ($use_old_bios_files, $machine_type);
7620 sub get_efivars_size
{
7622 my $arch = get_vm_arch
($conf);
7623 my $efidisk = $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
7624 my $smm = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
7625 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
7626 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
7627 return -s
$ovmf_vars;
7630 sub update_efidisk_size
{
7633 return if !defined($conf->{efidisk0
});
7635 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
7636 $disk->{size
} = get_efivars_size
($conf);
7637 $conf->{efidisk0
} = print_drive
($disk);
7642 sub update_tpmstate_size
{
7645 my $disk = PVE
::QemuServer
::parse_drive
('tpmstate0', $conf->{tpmstate0
});
7646 $disk->{size
} = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
7647 $conf->{tpmstate0
} = print_drive
($disk);
7650 sub create_efidisk
($$$$$$$) {
7651 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7653 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
7654 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7656 my $vars_size_b = -s
$ovmf_vars;
7657 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7658 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7659 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7661 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7662 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7664 return ($volid, $size/1024);
7667 sub vm_iothreads_list
{
7670 my $res = mon_cmd
($vmid, 'query-iothreads');
7673 foreach my $iothread (@$res) {
7674 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7681 my ($conf, $drive) = @_;
7685 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7687 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7693 my $controller = int($drive->{index} / $maxdev);
7694 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
7698 return ($maxdev, $controller, $controller_prefix);
7701 sub windows_version
{
7704 return 0 if !$ostype;
7708 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7710 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7712 } elsif ($ostype =~ m/^win(\d+)$/) {
7719 sub resolve_dst_disk_format
{
7720 my ($storecfg, $storeid, $src_volname, $format) = @_;
7721 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7724 # if no target format is specified, use the source disk format as hint
7726 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7727 $format = qemu_img_format
($scfg, $src_volname);
7733 # test if requested format is supported - else use default
7734 my $supported = grep { $_ eq $format } @$validFormats;
7735 $format = $defFormat if !$supported;
7739 # NOTE: if this logic changes, please update docs & possibly gui logic
7740 sub find_vmstate_storage
{
7741 my ($conf, $storecfg) = @_;
7743 # first, return storage from conf if set
7744 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7746 my ($target, $shared, $local);
7748 foreach_storage_used_by_vm
($conf, sub {
7750 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7751 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7752 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7755 # second, use shared storage where VM has at least one disk
7756 # third, use local storage where VM has at least one disk
7757 # fall back to local storage
7758 $target = $shared // $local // 'local';
7764 my ($uuid, $uuid_str);
7765 UUID
::generate
($uuid);
7766 UUID
::unparse
($uuid, $uuid_str);
7770 sub generate_smbios1_uuid
{
7771 return "uuid=".generate_uuid
();
7777 mon_cmd
($vmid, 'nbd-server-stop');
7780 sub create_reboot_request
{
7782 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7783 or die "failed to create reboot trigger file: $!\n";
7787 sub clear_reboot_request
{
7789 my $path = "/run/qemu-server/$vmid.reboot";
7792 $res = unlink($path);
7793 die "could not remove reboot request for $vmid: $!"
7794 if !$res && $! != POSIX
::ENOENT
;
7799 sub bootorder_from_legacy
{
7800 my ($conf, $bootcfg) = @_;
7802 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
7803 my $bootindex_hash = {};
7805 foreach my $o (split(//, $boot)) {
7806 $bootindex_hash->{$o} = $i*100;
7812 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7813 my ($ds, $drive) = @_;
7815 if (drive_is_cdrom
($drive, 1)) {
7816 if ($bootindex_hash->{d
}) {
7817 $bootorder->{$ds} = $bootindex_hash->{d
};
7818 $bootindex_hash->{d
} += 1;
7820 } elsif ($bootindex_hash->{c
}) {
7821 $bootorder->{$ds} = $bootindex_hash->{c
}
7822 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
7823 $bootindex_hash->{c
} += 1;
7827 if ($bootindex_hash->{n
}) {
7828 for (my $i = 0; $i < $MAX_NETS; $i++) {
7829 my $netname = "net$i";
7830 next if !$conf->{$netname};
7831 $bootorder->{$netname} = $bootindex_hash->{n
};
7832 $bootindex_hash->{n
} += 1;
7839 # Generate default device list for 'boot: order=' property. Matches legacy
7840 # default boot order, but with explicit device names. This is important, since
7841 # the fallback for when neither 'order' nor the old format is specified relies
7842 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7843 sub get_default_bootdevices
{
7849 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
7850 push @ret, $first if $first;
7853 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
7854 push @ret, $first if $first;
7857 for (my $i = 0; $i < $MAX_NETS; $i++) {
7858 my $netname = "net$i";
7859 next if !$conf->{$netname};
7860 push @ret, $netname;
7867 sub device_bootorder
{
7870 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
7872 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
7875 if (!defined($boot) || $boot->{legacy
}) {
7876 $bootorder = bootorder_from_legacy
($conf, $boot);
7877 } elsif ($boot->{order
}) {
7878 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7879 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
7880 $bootorder->{$dev} = $i++;
7887 sub register_qmeventd_handle
{
7891 my $peer = "/var/run/qmeventd.sock";
7896 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
7898 if ($! != EINTR
&& $! != EAGAIN
) {
7899 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7902 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7903 . "after $count retries\n";
7908 # send handshake to mark VM as backing up
7909 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
7911 # return handle to be closed later when inhibit is no longer required
7915 # bash completion helper
7917 sub complete_backup_archives
{
7918 my ($cmdname, $pname, $cvalue) = @_;
7920 my $cfg = PVE
::Storage
::config
();
7924 if ($cvalue =~ m/^([^:]+):/) {
7928 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7931 foreach my $id (keys %$data) {
7932 foreach my $item (@{$data->{$id}}) {
7933 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7934 push @$res, $item->{volid
} if defined($item->{volid
});
7941 my $complete_vmid_full = sub {
7944 my $idlist = vmstatus
();
7948 foreach my $id (keys %$idlist) {
7949 my $d = $idlist->{$id};
7950 if (defined($running)) {
7951 next if $d->{template
};
7952 next if $running && $d->{status
} ne 'running';
7953 next if !$running && $d->{status
} eq 'running';
7962 return &$complete_vmid_full();
7965 sub complete_vmid_stopped
{
7966 return &$complete_vmid_full(0);
7969 sub complete_vmid_running
{
7970 return &$complete_vmid_full(1);
7973 sub complete_storage
{
7975 my $cfg = PVE
::Storage
::config
();
7976 my $ids = $cfg->{ids
};
7979 foreach my $sid (keys %$ids) {
7980 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7981 next if !$ids->{$sid}->{content
}->{images
};
7988 sub complete_migration_storage
{
7989 my ($cmd, $param, $current_value, $all_args) = @_;
7991 my $targetnode = @$all_args[1];
7993 my $cfg = PVE
::Storage
::config
();
7994 my $ids = $cfg->{ids
};
7997 foreach my $sid (keys %$ids) {
7998 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7999 next if !$ids->{$sid}->{content
}->{images
};
8008 my $qmpstatus = eval {
8009 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
8010 mon_cmd
($vmid, "query-status");
8013 return $qmpstatus && $qmpstatus->{status
} eq "paused";
8016 sub check_volume_storage_type
{
8017 my ($storecfg, $vol) = @_;
8019 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
8020 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8021 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
8023 die "storage '$storeid' does not support content-type '$vtype'\n"
8024 if !$scfg->{content
}->{$vtype};