1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
31 use PVE
::DataCenterConfig
;
32 use PVE
::Exception
qw(raise raise_param_exc);
33 use PVE
::Format
qw(render_duration render_bytes);
34 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
36 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
39 use PVE
::RPCEnvironment
;
43 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
47 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
48 use PVE
::QemuServer
::Cloudinit
;
49 use PVE
::QemuServer
::CGroup
;
50 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
51 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
52 use PVE
::QemuServer
::Machine
;
53 use PVE
::QemuServer
::Memory
;
54 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
55 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
56 use PVE
::QemuServer
::USB
qw(parse_usb_device);
60 require PVE
::Network
::SDN
::Zones
;
64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
68 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
69 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
72 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
73 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
76 "$EDK2_FW_BASE/OVMF_CODE.fd",
77 "$EDK2_FW_BASE/OVMF_VARS.fd",
82 "$EDK2_FW_BASE/AAVMF_CODE.fd",
83 "$EDK2_FW_BASE/AAVMF_VARS.fd",
88 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
90 # Note about locking: we use flock on the config file protect
91 # against concurent actions.
92 # Aditionaly, we have a 'lock' setting in the config file. This
93 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
94 # allowed when such lock is set. But you can ignore this kind of
95 # lock with the --skiplock flag.
97 cfs_register_file
('/qemu-server/',
101 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
102 description
=> "Some command save/restore state from this location.",
108 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
109 description
=> "Specifies the Qemu machine type.",
111 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
118 my ($map, $source) = @_;
120 return $source if !defined($map);
122 return $map->{entries
}->{$source}
123 if $map->{entries
} && defined($map->{entries
}->{$source});
125 return $map->{default} if $map->{default};
127 # identity (fallback)
131 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
132 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
134 format
=> 'storagepair-list',
138 #no warnings 'redefine';
142 $nodename_cache //= PVE
::INotify
::nodename
();
143 return $nodename_cache;
150 enum
=> [qw(i6300esb ib700)],
151 description
=> "Watchdog type to emulate.",
152 default => 'i6300esb',
157 enum
=> [qw(reset shutdown poweroff pause debug none)],
158 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
162 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
166 description
=> "Enable/disable Qemu GuestAgent.",
171 fstrim_cloned_disks
=> {
172 description
=> "Run fstrim after moving a disk or migrating the VM.",
178 description
=> "Select the agent type",
182 enum
=> [qw(virtio isa)],
188 description
=> "Select the VGA type.",
193 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
196 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
208 description
=> "The size of the file in MB.",
212 pattern
=> '[a-zA-Z0-9\-]+',
214 format_description
=> 'string',
215 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
222 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
223 description
=> "Configure an audio device."
227 enum
=> ['spice', 'none'],
230 description
=> "Driver backend for the audio device."
234 my $spice_enhancements_fmt = {
239 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
243 enum
=> ['off', 'all', 'filter'],
246 description
=> "Enable video streaming. Uses compression for detected video streams."
253 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
255 description
=> "The file on the host to gather entropy from. In most"
256 . " cases /dev/urandom should be preferred over /dev/random"
257 . " to avoid entropy-starvation issues on the host. Using"
258 . " urandom does *not* decrease security in any meaningful"
259 . " way, as it's still seeded from real entropy, and the"
260 . " bytes provided will most likely be mixed with real"
261 . " entropy on the guest as well. /dev/hwrng can be used"
262 . " to pass through a hardware RNG from the host.",
266 description
=> "Maximum bytes of entropy injected into the guest every"
267 . " 'period' milliseconds. Prefer a lower value when using"
268 . " /dev/random as source. Use 0 to disable limiting"
269 . " (potentially dangerous!).",
272 # default is 1 KiB/s, provides enough entropy to the guest to avoid
273 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
274 # of overwhelming the host, provided we're reading from /dev/urandom
279 description
=> "Every 'period' milliseconds the entropy-injection quota"
280 . " is reset, allowing the guest to retrieve another"
281 . " 'max_bytes' of entropy.",
291 description
=> "Specifies whether a VM will be started during system bootup.",
297 description
=> "Automatic restart after crash (currently ignored).",
302 type
=> 'string', format
=> 'pve-hotplug-features',
303 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
304 default => 'network,disk,usb',
309 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
315 description
=> "Lock/unlock the VM.",
316 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
321 description
=> "Limit of CPU usage.",
322 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
330 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
331 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
332 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
333 ." weights of all the other running VMs.",
336 default => 'cgroup v1: 1024, cgroup v2: 100',
341 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
348 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
354 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
362 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
363 "It should not be necessary to set it.",
364 enum
=> PVE
::Tools
::kvmkeymaplist
(),
369 type
=> 'string', format
=> 'dns-name',
370 description
=> "Set a name for the VM. Only used on the configuration web interface.",
375 description
=> "SCSI controller model",
376 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
382 description
=> "Description for the VM. Shown in the web-interface VM's summary."
383 ." This is saved as comment inside the configuration file.",
384 maxLength
=> 1024 * 8,
389 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
390 description
=> "Specify guest operating system.",
391 verbose_description
=> <<EODESC,
392 Specify guest operating system. This is used to enable special
393 optimization/features for specific operating systems:
396 other;; unspecified OS
397 wxp;; Microsoft Windows XP
398 w2k;; Microsoft Windows 2000
399 w2k3;; Microsoft Windows 2003
400 w2k8;; Microsoft Windows 2008
401 wvista;; Microsoft Windows Vista
402 win7;; Microsoft Windows 7
403 win8;; Microsoft Windows 8/2012/2012r2
404 win10;; Microsoft Windows 10/2016/2019
405 l24;; Linux 2.4 Kernel
406 l26;; Linux 2.6 - 5.X Kernel
407 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
412 type
=> 'string', format
=> 'pve-qm-boot',
413 description
=> "Specify guest boot order. Use with 'order=', usage with"
414 . " no key or 'legacy=' is deprecated.",
418 type
=> 'string', format
=> 'pve-qm-bootdisk',
419 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
420 pattern
=> '(ide|sata|scsi|virtio)\d+',
425 description
=> "The number of CPUs. Please use option -sockets instead.",
432 description
=> "The number of CPU sockets.",
439 description
=> "The number of cores per socket.",
446 description
=> "Enable/disable NUMA.",
452 description
=> "Enable/disable hugepages memory.",
453 enum
=> [qw(any 2 1024)],
459 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
460 ." after VM shutdown and can be used for subsequent starts.",
465 description
=> "Number of hotplugged vcpus.",
472 description
=> "Enable/disable ACPI.",
477 description
=> "Enable/disable Qemu GuestAgent and its properties.",
479 format
=> $agent_fmt,
484 description
=> "Enable/disable KVM hardware virtualization.",
490 description
=> "Enable/disable time drift fix.",
496 description
=> "Set the real time clock to local time. This is enabled by default if ostype"
497 ." indicates a Microsoft OS.",
502 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
506 type
=> 'string', format
=> $vga_fmt,
507 description
=> "Configure the VGA hardware.",
508 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
509 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
510 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
511 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
512 ." display server. For win* OS you can select how many independent displays you want,"
513 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
514 ." using a serial device as terminal.",
518 type
=> 'string', format
=> 'pve-qm-watchdog',
519 description
=> "Create a virtual hardware watchdog device.",
520 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
521 ." action), the watchdog must be periodically polled by an agent inside the guest or"
522 ." else the watchdog will reset the guest (or execute the respective action specified)",
527 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
528 description
=> "Set the initial date of the real time clock. Valid format for date are:"
529 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
530 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
533 startup
=> get_standard_option
('pve-startup-order'),
537 description
=> "Enable/disable Template.",
543 description
=> "Arbitrary arguments passed to kvm.",
544 verbose_description
=> <<EODESCR,
545 Arbitrary arguments passed to kvm, for example:
547 args: -no-reboot -no-hpet
549 NOTE: this option is for experts only.
556 description
=> "Enable/disable the USB tablet device.",
557 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
558 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
559 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
560 ." may consider disabling this to save some context switches. This is turned off by"
561 ." default if you use spice (`qm set <vmid> --vga qxl`).",
566 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
570 migrate_downtime
=> {
573 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
579 type
=> 'string', format
=> 'pve-qm-ide',
580 typetext
=> '<volume>',
581 description
=> "This is an alias for option -ide2",
585 description
=> "Emulated CPU type.",
587 format
=> 'pve-vm-cpu-conf',
589 parent
=> get_standard_option
('pve-snapshot-name', {
591 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
595 description
=> "Timestamp for snapshots.",
601 type
=> 'string', format
=> 'pve-volume-id',
602 description
=> "Reference to a volume which stores the VM state. This is used internally"
605 vmstatestorage
=> get_standard_option
('pve-storage-id', {
606 description
=> "Default storage for VM state volumes/files.",
609 runningmachine
=> get_standard_option
('pve-qemu-machine', {
610 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
614 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
615 ." internally for snapshots.",
618 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
619 format_description
=> 'QEMU -cpu parameter'
621 machine
=> get_standard_option
('pve-qemu-machine'),
623 description
=> "Virtual processor architecture. Defaults to the host.",
626 enum
=> [qw(x86_64 aarch64)],
629 description
=> "Specify SMBIOS type 1 fields.",
630 type
=> 'string', format
=> 'pve-qm-smbios1',
637 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
638 ." remove disk operations.",
644 enum
=> [ qw(seabios ovmf) ],
645 description
=> "Select BIOS implementation.",
646 default => 'seabios',
650 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
651 format_description
=> 'UUID',
652 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
653 ." to disable explicitly.",
654 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
655 ." value identifier to the guest OS. This allows to notify the guest operating system"
656 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
657 ." execution or creation from a template). The guest operating system notices the"
658 ." change, and is then able to react as appropriate by marking its copies of"
659 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
660 ."Note that auto-creation only works when done through API/CLI create or update methods"
661 .", but not when manually editing the config file.",
662 default => "1 (autogenerated)",
667 format
=> 'pve-volume-id',
669 description
=> "Script that will be executed during various steps in the vms lifetime.",
673 format
=> $ivshmem_fmt,
674 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
680 format
=> $audio_fmt,
681 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
684 spice_enhancements
=> {
686 format
=> $spice_enhancements_fmt,
687 description
=> "Configure additional enhancements for SPICE.",
691 type
=> 'string', format
=> 'pve-tag-list',
692 description
=> 'Tags of the VM. This is only meta information.',
698 description
=> "Configure a VirtIO-based Random Number Generator.",
707 description
=> 'Specify a custom file containing all meta data passed to the VM via"
708 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
709 format
=> 'pve-volume-id',
710 format_description
=> 'volume',
715 description
=> 'Specify a custom file containing all network data passed to the VM via'
717 format
=> 'pve-volume-id',
718 format_description
=> 'volume',
723 description
=> 'Specify a custom file containing all user data passed to the VM via'
725 format
=> 'pve-volume-id',
726 format_description
=> 'volume',
729 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
731 my $confdesc_cloudinit = {
735 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
736 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
737 .' and `configdrive2` for windows.',
738 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
743 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
744 ." image's configured default user.",
749 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
750 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
751 .' support hashed passwords.',
756 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
758 format
=> 'pve-qm-cicustom',
763 description
=> "cloud-init: Sets DNS search domains for a container. Create will'
764 .' automatically use the setting from the host if neither searchdomain nor nameserver'
769 type
=> 'string', format
=> 'address-list',
770 description
=> "cloud-init: Sets DNS server IP address for a container. Create will'
771 .' automatically use the setting from the host if neither searchdomain nor nameserver'
777 format
=> 'urlencoded',
778 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
782 # what about other qemu settings ?
784 #machine => 'string',
797 ##soundhw => 'string',
799 while (my ($k, $v) = each %$confdesc) {
800 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
803 my $MAX_USB_DEVICES = 5;
805 my $MAX_SERIAL_PORTS = 4;
806 my $MAX_PARALLEL_PORTS = 3;
812 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
813 description
=> "CPUs accessing this NUMA node.",
814 format_description
=> "id[-id];...",
818 description
=> "Amount of memory this NUMA node provides.",
823 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
824 description
=> "Host NUMA nodes to use.",
825 format_description
=> "id[-id];...",
830 enum
=> [qw(preferred bind interleave)],
831 description
=> "NUMA allocation policy.",
835 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
838 type
=> 'string', format
=> $numa_fmt,
839 description
=> "NUMA topology.",
841 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
843 for (my $i = 0; $i < $MAX_NUMA; $i++) {
844 $confdesc->{"numa$i"} = $numadesc;
847 my $nic_model_list = [
863 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
865 my $net_fmt_bridge_descr = <<__EOD__;
866 Bridge to attach the network device to. The Proxmox VE standard bridge
869 If you do not specify a bridge, we create a kvm user (NATed) network
870 device, which provides DHCP and DNS services. The following addresses
877 The DHCP server assign addresses to the guest starting from 10.0.2.15.
881 macaddr
=> get_standard_option
('mac-addr', {
882 description
=> "MAC address. That address must be unique withing your network. This is"
883 ." automatically generated if not specified.",
887 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
888 ." very low CPU overhead. If your guest does not support this driver, it is usually"
889 ." best to use 'e1000'.",
890 enum
=> $nic_model_list,
893 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
896 description
=> $net_fmt_bridge_descr,
897 format_description
=> 'bridge',
898 pattern
=> '[-_.\w\d]+',
903 minimum
=> 0, maximum
=> 16,
904 description
=> 'Number of packet queues to be used on the device.',
910 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
915 minimum
=> 1, maximum
=> 4094,
916 description
=> 'VLAN tag to apply to packets on this interface.',
921 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
922 description
=> 'VLAN trunks to pass through this interface.',
923 format_description
=> 'vlanid[;vlanid...]',
928 description
=> 'Whether this interface should be protected by the firewall.',
933 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
938 minimum
=> 1, maximum
=> 65520,
939 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
946 type
=> 'string', format
=> $net_fmt,
947 description
=> "Specify network devices.",
950 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
955 format
=> 'pve-ipv4-config',
956 format_description
=> 'IPv4Format/CIDR',
957 description
=> 'IPv4 address in CIDR format.',
964 format_description
=> 'GatewayIPv4',
965 description
=> 'Default gateway for IPv4 traffic.',
971 format
=> 'pve-ipv6-config',
972 format_description
=> 'IPv6Format/CIDR',
973 description
=> 'IPv6 address in CIDR format.',
980 format_description
=> 'GatewayIPv6',
981 description
=> 'Default gateway for IPv6 traffic.',
986 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
989 type
=> 'string', format
=> 'pve-qm-ipconfig',
990 description
=> <<'EODESCR',
991 cloud-init: Specify IP addresses and gateways for the corresponding interface.
993 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
995 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
996 gateway should be provided.
997 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
998 cloud-init 19.4 or newer.
1000 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1004 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
1006 for (my $i = 0; $i < $MAX_NETS; $i++) {
1007 $confdesc->{"net$i"} = $netdesc;
1008 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1011 foreach my $key (keys %$confdesc_cloudinit) {
1012 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1015 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1016 sub verify_volume_id_or_qm_path
{
1017 my ($volid, $noerr) = @_;
1019 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
1023 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1024 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1035 type
=> 'string', format
=> 'pve-qm-usb-device',
1036 format_description
=> 'HOSTUSBDEVICE|spice',
1037 description
=> <<EODESCR,
1038 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1040 'bus-port(.port)*' (decimal numbers) or
1041 'vendor_id:product_id' (hexadeciaml numbers) or
1044 You can use the 'lsusb -t' command to list existing usb devices.
1046 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1047 machines - use with special care.
1049 The value 'spice' can be used to add a usb redirection devices for spice.
1055 description
=> "Specifies whether if given host option is a USB3 device or port.",
1062 type
=> 'string', format
=> $usb_fmt,
1063 description
=> "Configure an USB device (n is 0 to 4).",
1065 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1070 pattern
=> '(/dev/.+|socket)',
1071 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1072 verbose_description
=> <<EODESCR,
1073 Create a serial device inside the VM (n is 0 to 3), and pass through a
1074 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1075 host side (use 'qm terminal' to open a terminal connection).
1077 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1078 use with special care.
1080 CAUTION: Experimental! User reported problems with this option.
1087 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1088 description
=> "Map host parallel devices (n is 0 to 2).",
1089 verbose_description
=> <<EODESCR,
1090 Map host parallel devices (n is 0 to 2).
1092 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1093 machines - use with special care.
1095 CAUTION: Experimental! User reported problems with this option.
1099 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1100 $confdesc->{"parallel$i"} = $paralleldesc;
1103 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1104 $confdesc->{"serial$i"} = $serialdesc;
1107 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1108 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1111 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1112 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1115 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1116 $confdesc->{"usb$i"} = $usbdesc;
1124 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1125 . " Deprecated, use 'order=' instead.",
1126 pattern
=> '[acdn]{1,4}',
1127 format_description
=> "[acdn]{1,4}",
1129 # note: this is also the fallback if boot: is not given at all
1135 format
=> 'pve-qm-bootdev-list',
1136 format_description
=> "device[;device...]",
1137 description
=> <<EODESC,
1138 The guest will attempt to boot from devices in the order they appear here.
1140 Disks, optical drives and passed-through storage USB devices will be directly
1141 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1142 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1144 Note that only devices in this list will be marked as bootable and thus loaded
1145 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1146 (e.g. software-raid), you need to specify all of them here.
1148 Overrides the deprecated 'legacy=[acdn]*' value when given.
1152 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1154 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1155 sub verify_bootdev
{
1156 my ($dev, $noerr) = @_;
1158 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1159 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && !$special;
1163 return 0 if $dev !~ m/^$base\d+$/;
1164 return 0 if !$confdesc->{$dev};
1168 return $dev if $check->("net");
1169 return $dev if $check->("usb");
1170 return $dev if $check->("hostpci");
1173 die "invalid boot device '$dev'\n";
1176 sub print_bootorder
{
1178 return "" if !@$devs;
1179 my $data = { order
=> join(';', @$devs) };
1180 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1183 my $kvm_api_version = 0;
1186 return $kvm_api_version if $kvm_api_version;
1188 open my $fh, '<', '/dev/kvm' or return;
1190 # 0xae00 => KVM_GET_API_VERSION
1191 $kvm_api_version = ioctl($fh, 0xae00, 0);
1194 return $kvm_api_version;
1197 my $kvm_user_version = {};
1200 sub kvm_user_version
{
1203 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1204 my $st = stat($binary);
1206 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1207 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1208 $cachedmtime == $st->mtime;
1210 $kvm_user_version->{$binary} = 'unknown';
1211 $kvm_mtime->{$binary} = $st->mtime;
1215 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1216 $kvm_user_version->{$binary} = $2;
1220 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1223 return $kvm_user_version->{$binary};
1226 my sub extract_version
{
1227 my ($machine_type, $version) = @_;
1228 $version = kvm_user_version
() if !defined($version);
1229 PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1232 sub kernel_has_vhost_net
{
1233 return -c
'/dev/vhost-net';
1238 return defined($confdesc->{$key});
1242 sub get_cdrom_path
{
1244 return $cdrom_path if $cdrom_path;
1246 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1247 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1248 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1252 my ($storecfg, $vmid, $cdrom) = @_;
1254 if ($cdrom eq 'cdrom') {
1255 return get_cdrom_path
();
1256 } elsif ($cdrom eq 'none') {
1258 } elsif ($cdrom =~ m
|^/|) {
1261 return PVE
::Storage
::path
($storecfg, $cdrom);
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id
{
1267 my ($vmid, $file, $media) = @_;
1269 if (!($file eq 'none' || $file eq 'cdrom' ||
1270 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1272 return if $file =~ m
|/|;
1274 if ($media && $media eq 'cdrom') {
1275 $file = "local:iso/$file";
1277 $file = "local:$vmid/$file";
1284 sub verify_media_type
{
1285 my ($opt, $vtype, $media) = @_;
1290 if ($media eq 'disk') {
1292 } elsif ($media eq 'cdrom') {
1295 die "internal error";
1298 return if ($vtype eq $etype);
1300 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1303 sub cleanup_drive_path
{
1304 my ($opt, $storecfg, $drive) = @_;
1306 # try to convert filesystem paths to volume IDs
1308 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1309 ($drive->{file
} !~ m
|^/dev/.+|) &&
1310 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1311 ($drive->{file
} !~ m/^\d+$/)) {
1312 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1313 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1315 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1316 verify_media_type
($opt, $vtype, $drive->{media
});
1317 $drive->{file
} = $volid;
1320 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1323 sub parse_hotplug_features
{
1328 return $res if $data eq '0';
1330 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1332 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1333 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1336 die "invalid hotplug feature '$feature'\n";
1342 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1343 sub pve_verify_hotplug_features
{
1344 my ($value, $noerr) = @_;
1346 return $value if parse_hotplug_features
($value);
1350 die "unable to parse hotplug option\n";
1354 my($fh, $noerr) = @_;
1357 my $SG_GET_VERSION_NUM = 0x2282;
1359 my $versionbuf = "\x00" x
8;
1360 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1362 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1365 my $version = unpack("I", $versionbuf);
1366 if ($version < 30000) {
1367 die "scsi generic interface too old\n" if !$noerr;
1371 my $buf = "\x00" x
36;
1372 my $sensebuf = "\x00" x
8;
1373 my $cmd = pack("C x3 C x1", 0x12, 36);
1375 # see /usr/include/scsi/sg.h
1376 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1378 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1379 length($sensebuf), 0, length($buf), $buf,
1380 $cmd, $sensebuf, 6000);
1382 $ret = ioctl($fh, $SG_IO, $packet);
1384 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1388 my @res = unpack($sg_io_hdr_t, $packet);
1389 if ($res[17] || $res[18]) {
1390 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1395 (my $byte0, my $byte1, $res->{vendor
},
1396 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1398 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1399 $res->{type
} = $byte0 & 31;
1407 my $fh = IO
::File-
>new("+<$path") || return;
1408 my $res = scsi_inquiry
($fh, 1);
1414 sub print_tabletdevice_full
{
1415 my ($conf, $arch) = @_;
1417 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1419 # we use uhci for old VMs because tablet driver was buggy in older qemu
1421 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1427 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1430 sub print_keyboarddevice_full
{
1431 my ($conf, $arch, $machine) = @_;
1433 return if $arch ne 'aarch64';
1435 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1438 my sub get_drive_id
{
1440 return "$drive->{interface}$drive->{index}";
1443 sub print_drivedevice_full
{
1444 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1449 my $drive_id = get_drive_id
($drive);
1450 if ($drive->{interface
} eq 'virtio') {
1451 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1452 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1453 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1454 } elsif ($drive->{interface
} eq 'scsi') {
1456 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1457 my $unit = $drive->{index} % $maxdev;
1458 my $devicetype = 'hd';
1460 if (drive_is_cdrom
($drive)) {
1463 if ($drive->{file
} =~ m
|^/|) {
1464 $path = $drive->{file
};
1465 if (my $info = path_is_scsi
($path)) {
1466 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1467 $devicetype = 'block';
1468 } elsif ($info->{type
} == 1) { # tape
1469 $devicetype = 'generic';
1473 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1476 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1477 my $version = extract_version
($machine_type, kvm_user_version
());
1478 if ($path =~ m/^iscsi\:\/\
// &&
1479 !min_version
($version, 4, 1)) {
1480 $devicetype = 'generic';
1484 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1485 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1487 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1488 .",lun=$drive->{index}";
1490 $device .= ",drive=drive-$drive_id,id=$drive_id";
1492 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1493 $device .= ",rotation_rate=1";
1495 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1497 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1498 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1499 my $controller = int($drive->{index} / $maxdev);
1500 my $unit = $drive->{index} % $maxdev;
1501 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1503 $device = "ide-$devicetype";
1504 if ($drive->{interface
} eq 'ide') {
1505 $device .= ",bus=ide.$controller,unit=$unit";
1507 $device .= ",bus=ahci$controller.$unit";
1509 $device .= ",drive=drive-$drive_id,id=$drive_id";
1511 if ($devicetype eq 'hd') {
1512 if (my $model = $drive->{model
}) {
1513 $model = URI
::Escape
::uri_unescape
($model);
1514 $device .= ",model=$model";
1516 if ($drive->{ssd
}) {
1517 $device .= ",rotation_rate=1";
1520 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1521 } elsif ($drive->{interface
} eq 'usb') {
1523 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1525 die "unsupported interface type";
1528 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1530 if (my $serial = $drive->{serial
}) {
1531 $serial = URI
::Escape
::uri_unescape
($serial);
1532 $device .= ",serial=$serial";
1539 sub get_initiator_name
{
1542 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1543 while (defined(my $line = <$fh>)) {
1544 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1553 sub print_drive_commandline_full
{
1554 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1557 my $volid = $drive->{file
};
1558 my $format = $drive->{format
};
1559 my $drive_id = get_drive_id
($drive);
1561 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1562 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1564 if (drive_is_cdrom
($drive)) {
1565 $path = get_iso_path
($storecfg, $vmid, $volid);
1566 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1569 $path = PVE
::Storage
::path
($storecfg, $volid);
1570 $format //= qemu_img_format
($scfg, $volname);
1577 my $is_rbd = $path =~ m/^rbd:/;
1580 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1581 foreach my $o (@qemu_drive_options) {
1582 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1585 # snapshot only accepts on|off
1586 if (defined($drive->{snapshot
})) {
1587 my $v = $drive->{snapshot
} ?
'on' : 'off';
1588 $opts .= ",snapshot=$v";
1591 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1592 my ($dir, $qmpname) = @$type;
1593 if (my $v = $drive->{"mbps$dir"}) {
1594 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1596 if (my $v = $drive->{"mbps${dir}_max"}) {
1597 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1599 if (my $v = $drive->{"bps${dir}_max_length"}) {
1600 $opts .= ",throttling.bps$qmpname-max-length=$v";
1602 if (my $v = $drive->{"iops${dir}"}) {
1603 $opts .= ",throttling.iops$qmpname=$v";
1605 if (my $v = $drive->{"iops${dir}_max"}) {
1606 $opts .= ",throttling.iops$qmpname-max=$v";
1608 if (my $v = $drive->{"iops${dir}_max_length"}) {
1609 $opts .= ",throttling.iops$qmpname-max-length=$v";
1614 $format = "rbd" if $is_rbd;
1615 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1617 $opts .= ",format=alloc-track,file.driver=$format";
1619 $opts .= ",format=$format";
1622 my $cache_direct = 0;
1624 if (my $cache = $drive->{cache
}) {
1625 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1626 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1627 $opts .= ",cache=none";
1631 # io_uring with cache mode writeback or writethrough on krbd will hang...
1632 my $rbd_no_io_uring = $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1634 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1635 # sometimes, just plain disable...
1636 my $lvm_no_io_uring = $scfg && $scfg->{type
} eq 'lvm';
1638 if (!$drive->{aio
}) {
1639 if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1640 # io_uring supports all cache modes
1641 $opts .= ",aio=io_uring";
1643 # aio native works only with O_DIRECT
1645 $opts .= ",aio=native";
1647 $opts .= ",aio=threads";
1652 if (!drive_is_cdrom
($drive)) {
1654 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1655 $detectzeroes = 'off';
1656 } elsif ($drive->{discard
}) {
1657 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1659 # This used to be our default with discard not being specified:
1660 $detectzeroes = 'on';
1663 # note: 'detect-zeroes' works per blockdev and we want it to persist
1664 # after the alloc-track is removed, so put it on 'file' directly
1665 my $dz_param = $pbs_name ?
"file.detect-zeroes" : "detect-zeroes";
1666 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1670 $opts .= ",backing=$pbs_name";
1671 $opts .= ",auto-remove=on";
1674 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1675 my $file_param = "file";
1677 # non-rbd drivers require the underlying file to be a seperate block
1678 # node, so add a second .file indirection
1679 $file_param .= ".file" if !$is_rbd;
1680 $file_param .= ".filename";
1682 my $pathinfo = $path ?
"$file_param=$path," : '';
1684 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1687 sub print_pbs_blockdev
{
1688 my ($pbs_conf, $pbs_name) = @_;
1689 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1690 $blockdev .= ",repository=$pbs_conf->{repository}";
1691 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1692 $blockdev .= ",archive=$pbs_conf->{archive}";
1693 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1697 sub print_netdevice_full
{
1698 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1700 my $device = $net->{model
};
1701 if ($net->{model
} eq 'virtio') {
1702 $device = 'virtio-net-pci';
1705 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1706 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1707 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1708 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1709 # and out of each queue plus one config interrupt and control vector queue
1710 my $vectors = $net->{queues
} * 2 + 2;
1711 $tmpstr .= ",vectors=$vectors,mq=on";
1713 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1715 if (my $mtu = $net->{mtu
}) {
1716 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1717 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1720 } elsif ($mtu < 576) {
1721 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1722 } elsif ($mtu > $bridge_mtu) {
1723 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1725 $tmpstr .= ",host_mtu=$mtu";
1727 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1731 if ($use_old_bios_files) {
1733 if ($device eq 'virtio-net-pci') {
1734 $romfile = 'pxe-virtio.rom';
1735 } elsif ($device eq 'e1000') {
1736 $romfile = 'pxe-e1000.rom';
1737 } elsif ($device eq 'e1000e') {
1738 $romfile = 'pxe-e1000e.rom';
1739 } elsif ($device eq 'ne2k') {
1740 $romfile = 'pxe-ne2k_pci.rom';
1741 } elsif ($device eq 'pcnet') {
1742 $romfile = 'pxe-pcnet.rom';
1743 } elsif ($device eq 'rtl8139') {
1744 $romfile = 'pxe-rtl8139.rom';
1746 $tmpstr .= ",romfile=$romfile" if $romfile;
1752 sub print_netdev_full
{
1753 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1756 if ($netid =~ m/^net(\d+)$/) {
1760 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1762 my $ifname = "tap${vmid}i$i";
1764 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1765 die "interface name '$ifname' is too long (max 15 character)\n"
1766 if length($ifname) >= 16;
1768 my $vhostparam = '';
1769 if (is_native
($arch)) {
1770 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1773 my $vmname = $conf->{name
} || "vm$vmid";
1776 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1778 if ($net->{bridge
}) {
1779 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1780 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1782 $netdev = "type=user,id=$netid,hostname=$vmname";
1785 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1791 'cirrus' => 'cirrus-vga',
1793 'vmware' => 'vmware-svga',
1794 'virtio' => 'virtio-vga',
1797 sub print_vga_device
{
1798 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1800 my $type = $vga_map->{$vga->{type
}};
1801 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1802 $type = 'virtio-gpu';
1804 my $vgamem_mb = $vga->{memory
};
1806 my $max_outputs = '';
1808 $type = $id ?
'qxl' : 'qxl-vga';
1810 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1811 # set max outputs so linux can have up to 4 qxl displays with one device
1812 if (min_version
($machine_version, 4, 1)) {
1813 $max_outputs = ",max_outputs=4";
1818 die "no devicetype for $vga->{type}\n" if !$type;
1822 if ($vga->{type
} eq 'virtio') {
1823 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1824 $memory = ",max_hostmem=$bytes";
1826 # from https://www.spice-space.org/multiple-monitors.html
1827 $memory = ",vgamem_mb=$vga->{memory}";
1828 my $ram = $vgamem_mb * 4;
1829 my $vram = $vgamem_mb * 2;
1830 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1832 $memory = ",vgamem_mb=$vga->{memory}";
1834 } elsif ($qxlnum && $id) {
1835 $memory = ",ram_size=67108864,vram_size=33554432";
1839 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1840 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1843 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1844 my $vgaid = "vga" . ($id // '');
1847 if ($q35 && $vgaid eq 'vga') {
1848 # the first display uses pcie.0 bus on q35 machines
1849 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1851 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1854 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1857 sub parse_number_sets
{
1860 foreach my $part (split(/;/, $set)) {
1861 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1862 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1863 push @$res, [ $1, $2 ];
1865 die "invalid range: $part\n";
1874 my $res = parse_property_string
($numa_fmt, $data);
1875 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1876 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1880 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1884 my $res = eval { parse_property_string
($net_fmt, $data) };
1889 if (!defined($res->{macaddr
})) {
1890 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1891 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1896 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1897 sub parse_ipconfig
{
1900 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1906 if ($res->{gw
} && !$res->{ip
}) {
1907 warn 'gateway specified without specifying an IP address';
1910 if ($res->{gw6
} && !$res->{ip6
}) {
1911 warn 'IPv6 gateway specified without specifying an IPv6 address';
1914 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1915 warn 'gateway specified together with DHCP';
1918 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1920 warn "IPv6 gateway specified together with $res->{ip6} address";
1924 if (!$res->{ip
} && !$res->{ip6
}) {
1925 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1934 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1937 sub add_random_macs
{
1938 my ($settings) = @_;
1940 foreach my $opt (keys %$settings) {
1941 next if $opt !~ m/^net(\d+)$/;
1942 my $net = parse_net
($settings->{$opt});
1944 $settings->{$opt} = print_net
($net);
1948 sub vm_is_volid_owner
{
1949 my ($storecfg, $vmid, $volid) = @_;
1951 if ($volid !~ m
|^/|) {
1953 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1954 if ($owner && ($owner == $vmid)) {
1962 sub vmconfig_register_unused_drive
{
1963 my ($storecfg, $vmid, $conf, $drive) = @_;
1965 if (drive_is_cloudinit
($drive)) {
1966 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1968 } elsif (!drive_is_cdrom
($drive)) {
1969 my $volid = $drive->{file
};
1970 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1971 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1976 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1980 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1981 format_description
=> 'UUID',
1982 description
=> "Set SMBIOS1 UUID.",
1987 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1988 format_description
=> 'Base64 encoded string',
1989 description
=> "Set SMBIOS1 version.",
1994 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1995 format_description
=> 'Base64 encoded string',
1996 description
=> "Set SMBIOS1 serial number.",
2001 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2002 format_description
=> 'Base64 encoded string',
2003 description
=> "Set SMBIOS1 manufacturer.",
2008 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2009 format_description
=> 'Base64 encoded string',
2010 description
=> "Set SMBIOS1 product ID.",
2015 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2016 format_description
=> 'Base64 encoded string',
2017 description
=> "Set SMBIOS1 SKU string.",
2022 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2023 format_description
=> 'Base64 encoded string',
2024 description
=> "Set SMBIOS1 family string.",
2029 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2037 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2044 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2047 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2049 sub parse_watchdog
{
2054 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2059 sub parse_guest_agent
{
2062 return {} if !defined($conf->{agent
});
2064 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2067 # if the agent is disabled ignore the other potentially set properties
2068 return {} if !$res->{enabled
};
2073 my ($conf, $key) = @_;
2074 return undef if !defined($conf->{agent
});
2076 my $agent = parse_guest_agent
($conf);
2077 return $agent->{$key};
2083 return {} if !$value;
2084 my $res = eval { parse_property_string
($vga_fmt, $value) };
2094 my $res = eval { parse_property_string
($rng_fmt, $value) };
2099 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2100 sub verify_usb_device
{
2101 my ($value, $noerr) = @_;
2103 return $value if parse_usb_device
($value);
2107 die "unable to parse usb device\n";
2110 # add JSON properties for create and set function
2111 sub json_config_properties
{
2114 foreach my $opt (keys %$confdesc) {
2115 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2116 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2117 $prop->{$opt} = $confdesc->{$opt};
2123 # return copy of $confdesc_cloudinit to generate documentation
2124 sub cloudinit_config_properties
{
2126 return dclone
($confdesc_cloudinit);
2130 my ($key, $value) = @_;
2132 die "unknown setting '$key'\n" if !$confdesc->{$key};
2134 my $type = $confdesc->{$key}->{type
};
2136 if (!defined($value)) {
2137 die "got undefined value\n";
2140 if ($value =~ m/[\n\r]/) {
2141 die "property contains a line feed\n";
2144 if ($type eq 'boolean') {
2145 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2146 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2147 die "type check ('boolean') failed - got '$value'\n";
2148 } elsif ($type eq 'integer') {
2149 return int($1) if $value =~ m/^(\d+)$/;
2150 die "type check ('integer') failed - got '$value'\n";
2151 } elsif ($type eq 'number') {
2152 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2153 die "type check ('number') failed - got '$value'\n";
2154 } elsif ($type eq 'string') {
2155 if (my $fmt = $confdesc->{$key}->{format
}) {
2156 PVE
::JSONSchema
::check_format
($fmt, $value);
2159 $value =~ s/^\"(.*)\"$/$1/;
2162 die "internal error"
2167 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2169 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2171 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2173 if ($conf->{template
}) {
2174 # check if any base image is still used by a linked clone
2175 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2176 my ($ds, $drive) = @_;
2177 return if drive_is_cdrom
($drive);
2179 my $volid = $drive->{file
};
2180 return if !$volid || $volid =~ m
|^/|;
2182 die "base volume '$volid' is still in use by linked cloned\n"
2183 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2189 my $remove_owned_drive = sub {
2190 my ($ds, $drive) = @_;
2191 return if drive_is_cdrom
($drive, 1);
2193 my $volid = $drive->{file
};
2194 return if !$volid || $volid =~ m
|^/|;
2195 return if $volids->{$volid};
2197 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2198 return if !$path || !$owner || ($owner != $vmid);
2200 $volids->{$volid} = 1;
2201 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2202 warn "Could not remove disk '$volid', check manually: $@" if $@;
2205 # only remove disks owned by this VM (referenced in the config)
2206 my $include_opts = {
2207 include_unused
=> 1,
2208 extra_keys
=> ['vmstate'],
2210 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2212 for my $snap (values %{$conf->{snapshots
}}) {
2213 next if !defined($snap->{vmstate
});
2214 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2215 next if !defined($drive);
2216 $remove_owned_drive->('vmstate', $drive);
2219 PVE
::QemuConfig-
>foreach_volume_full($conf->{pending
}, $include_opts, $remove_owned_drive);
2221 if ($purge_unreferenced) { # also remove unreferenced disk
2222 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2223 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2224 my ($volid, $sid, $volname, $d) = @_;
2225 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2230 if (defined $replacement_conf) {
2231 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2233 PVE
::QemuConfig-
>destroy_config($vmid);
2237 sub parse_vm_config
{
2238 my ($filename, $raw) = @_;
2240 return if !defined($raw);
2243 digest
=> Digest
::SHA
::sha1_hex
($raw),
2248 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2249 || die "got strange filename '$filename'";
2257 my @lines = split(/\n/, $raw);
2258 foreach my $line (@lines) {
2259 next if $line =~ m/^\s*$/;
2261 if ($line =~ m/^\[PENDING\]\s*$/i) {
2262 $section = 'pending';
2263 if (defined($descr)) {
2265 $conf->{description
} = $descr;
2268 $conf = $res->{$section} = {};
2271 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2273 if (defined($descr)) {
2275 $conf->{description
} = $descr;
2278 $conf = $res->{snapshots
}->{$section} = {};
2282 if ($line =~ m/^\#(.*)\s*$/) {
2283 $descr = '' if !defined($descr);
2284 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2288 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2289 $descr = '' if !defined($descr);
2290 $descr .= PVE
::Tools
::decode_text
($2);
2291 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2292 $conf->{snapstate
} = $1;
2293 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2296 $conf->{$key} = $value;
2297 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2299 if ($section eq 'pending') {
2300 $conf->{delete} = $value; # we parse this later
2302 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2304 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2307 eval { $value = check_type
($key, $value); };
2309 warn "vm $vmid - unable to parse value of '$key' - $@";
2311 $key = 'ide2' if $key eq 'cdrom';
2312 my $fmt = $confdesc->{$key}->{format
};
2313 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2314 my $v = parse_drive
($key, $value);
2315 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2316 $v->{file
} = $volid;
2317 $value = print_drive
($v);
2319 warn "vm $vmid - unable to parse value of '$key'\n";
2324 $conf->{$key} = $value;
2327 warn "vm $vmid - unable to parse config: $line\n";
2331 if (defined($descr)) {
2333 $conf->{description
} = $descr;
2335 delete $res->{snapstate
}; # just to be sure
2340 sub write_vm_config
{
2341 my ($filename, $conf) = @_;
2343 delete $conf->{snapstate
}; # just to be sure
2345 if ($conf->{cdrom
}) {
2346 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2347 $conf->{ide2
} = $conf->{cdrom
};
2348 delete $conf->{cdrom
};
2351 # we do not use 'smp' any longer
2352 if ($conf->{sockets
}) {
2353 delete $conf->{smp
};
2354 } elsif ($conf->{smp
}) {
2355 $conf->{sockets
} = $conf->{smp
};
2356 delete $conf->{cores
};
2357 delete $conf->{smp
};
2360 my $used_volids = {};
2362 my $cleanup_config = sub {
2363 my ($cref, $pending, $snapname) = @_;
2365 foreach my $key (keys %$cref) {
2366 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2367 $key eq 'snapstate' || $key eq 'pending';
2368 my $value = $cref->{$key};
2369 if ($key eq 'delete') {
2370 die "propertry 'delete' is only allowed in [PENDING]\n"
2372 # fixme: check syntax?
2375 eval { $value = check_type
($key, $value); };
2376 die "unable to parse value of '$key' - $@" if $@;
2378 $cref->{$key} = $value;
2380 if (!$snapname && is_valid_drivename
($key)) {
2381 my $drive = parse_drive
($key, $value);
2382 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2387 &$cleanup_config($conf);
2389 &$cleanup_config($conf->{pending
}, 1);
2391 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2392 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2393 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2396 # remove 'unusedX' settings if we re-add a volume
2397 foreach my $key (keys %$conf) {
2398 my $value = $conf->{$key};
2399 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2400 delete $conf->{$key};
2404 my $generate_raw_config = sub {
2405 my ($conf, $pending) = @_;
2409 # add description as comment to top of file
2410 if (defined(my $descr = $conf->{description
})) {
2412 foreach my $cl (split(/\n/, $descr)) {
2413 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2416 $raw .= "#\n" if $pending;
2420 foreach my $key (sort keys %$conf) {
2421 next if $key =~ /^(digest|description|pending|snapshots)$/;
2422 $raw .= "$key: $conf->{$key}\n";
2427 my $raw = &$generate_raw_config($conf);
2429 if (scalar(keys %{$conf->{pending
}})){
2430 $raw .= "\n[PENDING]\n";
2431 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2434 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2435 $raw .= "\n[$snapname]\n";
2436 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2446 # we use static defaults from our JSON schema configuration
2447 foreach my $key (keys %$confdesc) {
2448 if (defined(my $default = $confdesc->{$key}->{default})) {
2449 $res->{$key} = $default;
2457 my $vmlist = PVE
::Cluster
::get_vmlist
();
2459 return $res if !$vmlist || !$vmlist->{ids
};
2460 my $ids = $vmlist->{ids
};
2461 my $nodename = nodename
();
2463 foreach my $vmid (keys %$ids) {
2464 my $d = $ids->{$vmid};
2465 next if !$d->{node
} || $d->{node
} ne $nodename;
2466 next if !$d->{type
} || $d->{type
} ne 'qemu';
2467 $res->{$vmid}->{exists} = 1;
2472 # test if VM uses local resources (to prevent migration)
2473 sub check_local_resources
{
2474 my ($conf, $noerr) = @_;
2478 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2479 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2481 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2483 foreach my $k (keys %$conf) {
2484 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2485 # sockets are safe: they will recreated be on the target side post-migrate
2486 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2487 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2490 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2495 # check if used storages are available on all nodes (use by migrate)
2496 sub check_storage_availability
{
2497 my ($storecfg, $conf, $node) = @_;
2499 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2500 my ($ds, $drive) = @_;
2502 my $volid = $drive->{file
};
2505 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2508 # check if storage is available on both nodes
2509 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2510 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2512 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2514 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2515 if !$scfg->{content
}->{$vtype};
2519 # list nodes where all VM images are available (used by has_feature API)
2521 my ($conf, $storecfg) = @_;
2523 my $nodelist = PVE
::Cluster
::get_nodelist
();
2524 my $nodehash = { map { $_ => 1 } @$nodelist };
2525 my $nodename = nodename
();
2527 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2528 my ($ds, $drive) = @_;
2530 my $volid = $drive->{file
};
2533 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2535 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2536 if ($scfg->{disable
}) {
2538 } elsif (my $avail = $scfg->{nodes
}) {
2539 foreach my $node (keys %$nodehash) {
2540 delete $nodehash->{$node} if !$avail->{$node};
2542 } elsif (!$scfg->{shared
}) {
2543 foreach my $node (keys %$nodehash) {
2544 delete $nodehash->{$node} if $node ne $nodename
2553 sub check_local_storage_availability
{
2554 my ($conf, $storecfg) = @_;
2556 my $nodelist = PVE
::Cluster
::get_nodelist
();
2557 my $nodehash = { map { $_ => {} } @$nodelist };
2559 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2560 my ($ds, $drive) = @_;
2562 my $volid = $drive->{file
};
2565 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2567 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2569 if ($scfg->{disable
}) {
2570 foreach my $node (keys %$nodehash) {
2571 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2573 } elsif (my $avail = $scfg->{nodes
}) {
2574 foreach my $node (keys %$nodehash) {
2575 if (!$avail->{$node}) {
2576 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2583 foreach my $node (values %$nodehash) {
2584 if (my $unavail = $node->{unavailable_storages
}) {
2585 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2592 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2594 my ($vmid, $nocheck, $node) = @_;
2596 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2597 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2602 my $vzlist = config_list
();
2604 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2606 while (defined(my $de = $fd->read)) {
2607 next if $de !~ m/^(\d+)\.pid$/;
2609 next if !defined($vzlist->{$vmid});
2610 if (my $pid = check_running
($vmid)) {
2611 $vzlist->{$vmid}->{pid
} = $pid;
2618 our $vmstatus_return_properties = {
2619 vmid
=> get_standard_option
('pve-vmid'),
2621 description
=> "Qemu process status.",
2623 enum
=> ['stopped', 'running'],
2626 description
=> "Maximum memory in bytes.",
2629 renderer
=> 'bytes',
2632 description
=> "Root disk size in bytes.",
2635 renderer
=> 'bytes',
2638 description
=> "VM name.",
2643 description
=> "Qemu QMP agent status.",
2648 description
=> "PID of running qemu process.",
2653 description
=> "Uptime.",
2656 renderer
=> 'duration',
2659 description
=> "Maximum usable CPUs.",
2664 description
=> "The current config lock, if any.",
2669 description
=> "The current configured tags, if any",
2673 'running-machine' => {
2674 description
=> "The currently running machine type (if running).",
2679 description
=> "The currently running QEMU version (if running).",
2685 my $last_proc_pid_stat;
2687 # get VM status information
2688 # This must be fast and should not block ($full == false)
2689 # We only query KVM using QMP if $full == true (this can be slow)
2691 my ($opt_vmid, $full) = @_;
2695 my $storecfg = PVE
::Storage
::config
();
2697 my $list = vzlist
();
2698 my $defaults = load_defaults
();
2700 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2702 my $cpucount = $cpuinfo->{cpus
} || 1;
2704 foreach my $vmid (keys %$list) {
2705 next if $opt_vmid && ($vmid ne $opt_vmid);
2707 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2709 my $d = { vmid
=> int($vmid) };
2710 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2712 # fixme: better status?
2713 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2715 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2716 if (defined($size)) {
2717 $d->{disk
} = 0; # no info available
2718 $d->{maxdisk
} = $size;
2724 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2725 * ($conf->{cores
} || $defaults->{cores
});
2726 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2727 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2729 $d->{name
} = $conf->{name
} || "VM $vmid";
2730 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2731 : $defaults->{memory
}*(1024*1024);
2733 if ($conf->{balloon
}) {
2734 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2735 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2736 : $defaults->{shares
};
2747 $d->{diskwrite
} = 0;
2749 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2751 $d->{serial
} = 1 if conf_has_serial
($conf);
2752 $d->{lock} = $conf->{lock} if $conf->{lock};
2753 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2758 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2759 foreach my $dev (keys %$netdev) {
2760 next if $dev !~ m/^tap([1-9]\d*)i/;
2762 my $d = $res->{$vmid};
2765 $d->{netout
} += $netdev->{$dev}->{receive
};
2766 $d->{netin
} += $netdev->{$dev}->{transmit
};
2769 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2770 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2775 my $ctime = gettimeofday
;
2777 foreach my $vmid (keys %$list) {
2779 my $d = $res->{$vmid};
2780 my $pid = $d->{pid
};
2783 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2784 next if !$pstat; # not running
2786 my $used = $pstat->{utime} + $pstat->{stime
};
2788 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2790 if ($pstat->{vsize
}) {
2791 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2794 my $old = $last_proc_pid_stat->{$pid};
2796 $last_proc_pid_stat->{$pid} = {
2804 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2806 if ($dtime > 1000) {
2807 my $dutime = $used - $old->{used
};
2809 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2810 $last_proc_pid_stat->{$pid} = {
2816 $d->{cpu
} = $old->{cpu
};
2820 return $res if !$full;
2822 my $qmpclient = PVE
::QMPClient-
>new();
2824 my $ballooncb = sub {
2825 my ($vmid, $resp) = @_;
2827 my $info = $resp->{'return'};
2828 return if !$info->{max_mem
};
2830 my $d = $res->{$vmid};
2832 # use memory assigned to VM
2833 $d->{maxmem
} = $info->{max_mem
};
2834 $d->{balloon
} = $info->{actual
};
2836 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2837 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2838 $d->{freemem
} = $info->{free_mem
};
2841 $d->{ballooninfo
} = $info;
2844 my $blockstatscb = sub {
2845 my ($vmid, $resp) = @_;
2846 my $data = $resp->{'return'} || [];
2847 my $totalrdbytes = 0;
2848 my $totalwrbytes = 0;
2850 for my $blockstat (@$data) {
2851 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2852 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2854 $blockstat->{device
} =~ s/drive-//;
2855 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2857 $res->{$vmid}->{diskread
} = $totalrdbytes;
2858 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2861 my $machinecb = sub {
2862 my ($vmid, $resp) = @_;
2863 my $data = $resp->{'return'} || [];
2865 $res->{$vmid}->{'running-machine'} =
2866 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
2869 my $versioncb = sub {
2870 my ($vmid, $resp) = @_;
2871 my $data = $resp->{'return'} // {};
2872 my $version = 'unknown';
2874 if (my $v = $data->{qemu
}) {
2875 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
2878 $res->{$vmid}->{'running-qemu'} = $version;
2881 my $statuscb = sub {
2882 my ($vmid, $resp) = @_;
2884 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2885 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2886 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2887 # this fails if ballon driver is not loaded, so this must be
2888 # the last commnand (following command are aborted if this fails).
2889 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2891 my $status = 'unknown';
2892 if (!defined($status = $resp->{'return'}->{status
})) {
2893 warn "unable to get VM status\n";
2897 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2900 foreach my $vmid (keys %$list) {
2901 next if $opt_vmid && ($vmid ne $opt_vmid);
2902 next if !$res->{$vmid}->{pid
}; # not running
2903 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2906 $qmpclient->queue_execute(undef, 2);
2908 foreach my $vmid (keys %$list) {
2909 next if $opt_vmid && ($vmid ne $opt_vmid);
2910 next if !$res->{$vmid}->{pid
}; #not running
2912 # we can't use the $qmpclient since it might have already aborted on
2913 # 'query-balloon', but this might also fail for older versions...
2914 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
2915 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2918 foreach my $vmid (keys %$list) {
2919 next if $opt_vmid && ($vmid ne $opt_vmid);
2920 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2926 sub conf_has_serial
{
2929 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2930 if ($conf->{"serial$i"}) {
2938 sub conf_has_audio
{
2939 my ($conf, $id) = @_;
2942 my $audio = $conf->{"audio$id"};
2943 return if !defined($audio);
2945 my $audioproperties = parse_property_string
($audio_fmt, $audio);
2946 my $audiodriver = $audioproperties->{driver
} // 'spice';
2949 dev
=> $audioproperties->{device
},
2950 dev_id
=> "audiodev$id",
2951 backend
=> $audiodriver,
2952 backend_id
=> "$audiodriver-backend${id}",
2957 my ($audio, $audiopciaddr, $machine_version) = @_;
2961 my $id = $audio->{dev_id
};
2963 if (min_version
($machine_version, 4, 2)) {
2964 $audiodev = ",audiodev=$audio->{backend_id}";
2967 if ($audio->{dev
} eq 'AC97') {
2968 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2969 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
2970 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2971 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2972 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2974 die "unkown audio device '$audio->{dev}', implement me!";
2977 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2985 socket => "/var/run/qemu-server/$vmid.swtpm",
2986 pid
=> "/var/run/qemu-server/$vmid.swtpm.pid",
2990 sub add_tpm_device
{
2991 my ($vmid, $devices, $conf) = @_;
2993 return if !$conf->{tpmstate0
};
2995 my $paths = get_tpm_paths
($vmid);
2997 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
2998 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
2999 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3003 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3005 return if !$tpmdrive;
3008 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
3009 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
3011 $state = PVE
::Storage
::map_volume
($storecfg, $tpm->{file
});
3013 $state = $tpm->{file
};
3016 my $paths = get_tpm_paths
($vmid);
3018 # during migration, we will get state from remote
3021 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3028 "--create-platform-cert",
3031 "/etc/swtpm_setup.conf", # do not use XDG configs
3033 "0", # force creation as root, error if not possible
3034 "--not-overwrite", # ignore existing state, do not modify
3037 push @$setup_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3038 # TPM 2.0 supports ECC crypto, use if possible
3039 push @$setup_cmd, "--ecc" if $tpm->{version
} eq 'v2.0';
3041 run_command
($setup_cmd, outfunc
=> sub {
3042 print "swtpm_setup: $1\n";
3046 my $emulator_cmd = [
3050 "backend-uri=file://$state,mode=0600",
3052 "type=unixio,path=$paths->{socket},mode=0600",
3054 "file=$paths->{pid}",
3055 "--terminate", # terminate on QEMU disconnect
3058 push @$emulator_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3059 run_command
($emulator_cmd, outfunc
=> sub { print $1; });
3061 # return untainted PID of swtpm daemon so it can be killed on error
3062 file_read_firstline
($paths->{pid
}) =~ m/(\d+)/;
3066 sub vga_conf_has_spice
{
3069 my $vgaconf = parse_vga
($vga);
3070 my $vgatype = $vgaconf->{type
};
3071 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3078 return get_host_arch
() eq $arch;
3083 return $conf->{arch
} // get_host_arch
();
3086 my $default_machines = {
3091 sub get_installed_machine_version
{
3092 my ($kvmversion) = @_;
3093 $kvmversion = kvm_user_version
() if !defined($kvmversion);
3094 $kvmversion =~ m/^(\d+\.\d+)/;
3098 sub windows_get_pinned_machine_version
{
3099 my ($machine, $base_version, $kvmversion) = @_;
3101 my $pin_version = $base_version;
3102 if (!defined($base_version) ||
3103 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3105 $pin_version = get_installed_machine_version
($kvmversion);
3107 if (!$machine || $machine eq 'pc') {
3108 $machine = "pc-i440fx-$pin_version";
3109 } elsif ($machine eq 'q35') {
3110 $machine = "pc-q35-$pin_version";
3111 } elsif ($machine eq 'virt') {
3112 $machine = "virt-$pin_version";
3114 warn "unknown machine type '$machine', not touching that!\n";
3120 sub get_vm_machine
{
3121 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3123 my $machine = $forcemachine || $conf->{machine
};
3125 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3126 $kvmversion //= kvm_user_version
();
3127 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3128 # layout which confuses windows quite a bit and may result in various regressions..
3129 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3130 if (windows_version
($conf->{ostype
})) {
3131 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3134 $machine ||= $default_machines->{$arch};
3135 if ($add_pve_version) {
3136 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3137 $machine .= "+pve$pvever";
3141 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3142 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3143 $machine = $1 if $is_pxe;
3145 # for version-pinned machines that do not include a pve-version (e.g.
3146 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3147 $machine .= '+pve0';
3149 $machine .= '.pxe' if $is_pxe;
3155 sub get_ovmf_files
($$) {
3156 my ($arch, $efidisk) = @_;
3158 my $types = $OVMF->{$arch}
3159 or die "no OVMF images known for architecture '$arch'\n";
3161 my $type = 'default';
3162 if (defined($efidisk->{efitype
}) && $efidisk->{efitype
} eq '4m') {
3163 $type = $efidisk->{'pre-enrolled-keys'} ?
"4m-ms" : "4m";
3166 return $types->{$type}->@*;
3170 aarch64
=> '/usr/bin/qemu-system-aarch64',
3171 x86_64
=> '/usr/bin/qemu-system-x86_64',
3173 sub get_command_for_arch
($) {
3175 return '/usr/bin/kvm' if is_native
($arch);
3177 my $cmd = $Arch2Qemu->{$arch}
3178 or die "don't know how to emulate architecture '$arch'\n";
3182 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3183 # to use in a QEMU command line (-cpu element), first array_intersect the result
3184 # of query_supported_ with query_understood_. This is necessary because:
3186 # a) query_understood_ returns flags the host cannot use and
3187 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3188 # flags, but CPU settings - with most of them being flags. Those settings
3189 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3191 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3192 # expensive. If you need the value returned from this, you can get it much
3193 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3194 # $accel being 'kvm' or 'tcg'.
3196 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3197 # changes, automatically populating pmxcfs.
3199 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3200 # since kvm and tcg machines support different flags
3202 sub query_supported_cpu_flags
{
3205 $arch //= get_host_arch
();
3206 my $default_machine = $default_machines->{$arch};
3210 # FIXME: Once this is merged, the code below should work for ARM as well:
3211 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3212 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3215 my $kvm_supported = defined(kvm_version
());
3216 my $qemu_cmd = get_command_for_arch
($arch);
3218 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3220 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3221 my $query_supported_run_qemu = sub {
3227 '-machine', $default_machine,
3229 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3230 '-mon', 'chardev=qmp,mode=control',
3231 '-pidfile', $pidfile,
3236 push @$cmd, '-accel', 'tcg';
3239 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3240 die "QEMU flag querying VM exited with code " . $rc if $rc;
3243 my $cmd_result = mon_cmd
(
3245 'query-cpu-model-expansion',
3247 model
=> { name
=> 'host' }
3250 my $props = $cmd_result->{model
}->{props
};
3251 foreach my $prop (keys %$props) {
3252 next if $props->{$prop} ne '1';
3253 # QEMU returns some flags multiple times, with '_', '.' or '-'
3254 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3255 # We only keep those with underscores, to match /proc/cpuinfo
3256 $prop =~ s/\.|-/_/g;
3257 $flags->{$prop} = 1;
3262 # force stop with 10 sec timeout and 'nocheck'
3263 # always stop, even if QMP failed
3264 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3268 return [ sort keys %$flags ];
3271 # We need to query QEMU twice, since KVM and TCG have different supported flags
3272 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3273 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3274 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3276 if ($kvm_supported) {
3277 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3278 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3285 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3286 my $understood_cpu_flag_dir = "/usr/share/kvm";
3287 sub query_understood_cpu_flags
{
3288 my $arch = get_host_arch
();
3289 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3291 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3294 my $raw = file_get_contents
($filepath);
3295 $raw =~ s/^\s+|\s+$//g;
3296 my @flags = split(/\s+/, $raw);
3301 my sub get_cpuunits
{
3303 return $conf->{cpuunits
} // (PVE
::CGroup
::cgroup_mode
() == 2 ?
100 : 1024);
3305 sub config_to_command
{
3306 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3310 my $globalFlags = [];
3311 my $machineFlags = [];
3316 my $ostype = $conf->{ostype
};
3317 my $winversion = windows_version
($ostype);
3318 my $kvm = $conf->{kvm
};
3319 my $nodename = nodename
();
3321 my $arch = get_vm_arch
($conf);
3322 my $kvm_binary = get_command_for_arch
($arch);
3323 my $kvmver = kvm_user_version
($kvm_binary);
3325 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3326 $kvmver //= "undefined";
3327 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3330 my $add_pve_version = min_version
($kvmver, 4, 1);
3332 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3333 my $machine_version = extract_version
($machine_type, $kvmver);
3334 $kvm //= 1 if is_native
($arch);
3336 $machine_version =~ m/(\d+)\.(\d+)/;
3337 my ($machine_major, $machine_minor) = ($1, $2);
3339 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3340 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3341 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3342 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3343 ." please upgrade node '$nodename'\n"
3344 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3345 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3346 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3347 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3348 ." node '$nodename'\n";
3351 # if a specific +pve version is required for a feature, use $version_guard
3352 # instead of min_version to allow machines to be run with the minimum
3354 my $required_pve_version = 0;
3355 my $version_guard = sub {
3356 my ($major, $minor, $pve) = @_;
3357 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3358 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3359 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3360 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3364 if ($kvm && !defined kvm_version
()) {
3365 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3366 ." or enable in BIOS.\n";
3369 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3370 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3371 my $use_old_bios_files = undef;
3372 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3374 my $cpuunits = get_cpuunits
($conf);
3376 push @$cmd, $kvm_binary;
3378 push @$cmd, '-id', $vmid;
3380 my $vmname = $conf->{name
} || "vm$vmid";
3382 push @$cmd, '-name', $vmname;
3384 push @$cmd, '-no-shutdown';
3388 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3389 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3390 push @$cmd, '-mon', "chardev=qmp,mode=control";
3392 if (min_version
($machine_version, 2, 12)) {
3393 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3394 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3397 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3399 push @$cmd, '-daemonize';
3401 if ($conf->{smbios1
}) {
3402 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3403 if ($smbios_conf->{base64
}) {
3404 # Do not pass base64 flag to qemu
3405 delete $smbios_conf->{base64
};
3406 my $smbios_string = "";
3407 foreach my $key (keys %$smbios_conf) {
3409 if ($key eq "uuid") {
3410 $value = $smbios_conf->{uuid
}
3412 $value = decode_base64
($smbios_conf->{$key});
3414 # qemu accepts any binary data, only commas need escaping by double comma
3416 $smbios_string .= "," . $key . "=" . $value if $value;
3418 push @$cmd, '-smbios', "type=1" . $smbios_string;
3420 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3424 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3426 if (my $efidisk = $conf->{efidisk0
}) {
3427 $d = parse_drive
('efidisk0', $efidisk);
3430 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch, $d);
3431 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3433 my ($path, $format);
3434 my $read_only_str = '';
3436 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3437 $format = $d->{format
};
3439 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3440 if (!defined($format)) {
3441 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3442 $format = qemu_img_format
($scfg, $volname);
3446 die "efidisk format must be specified\n"
3447 if !defined($format);
3450 $read_only_str = ',readonly=on' if drive_is_read_only
($conf, $d);
3452 warn "no efidisk configured! Using temporary efivars disk.\n";
3453 $path = "/tmp/$vmid-ovmf.fd";
3454 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3460 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3461 $size_str = ",size=" . (-s
$ovmf_vars);
3464 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3466 if ($path =~ m/^rbd:/) {
3467 $cache = ',cache=writeback';
3468 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3471 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3472 push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3477 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3478 if (min_version
($machine_version, 4, 0)) {
3479 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3481 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3485 if ($conf->{vmgenid
}) {
3486 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3489 # add usb controllers
3490 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3491 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3492 push @$devices, @usbcontrollers if @usbcontrollers;
3493 my $vga = parse_vga
($conf->{vga
});
3495 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3496 $vga->{type
} = 'qxl' if $qxlnum;
3498 if (!$vga->{type
}) {
3499 if ($arch eq 'aarch64') {
3500 $vga->{type
} = 'virtio';
3501 } elsif (min_version
($machine_version, 2, 9)) {
3502 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3504 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3508 # enable absolute mouse coordinates (needed by vnc)
3510 if (defined($conf->{tablet
})) {
3511 $tablet = $conf->{tablet
};
3513 $tablet = $defaults->{tablet
};
3514 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3515 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3519 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3520 my $kbd = print_keyboarddevice_full
($conf, $arch);
3521 push @$devices, '-device', $kbd if defined($kbd);
3524 my $bootorder = device_bootorder
($conf);
3526 # host pci device passthrough
3527 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3528 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3531 my $usb_dev_features = {};
3532 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3534 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3535 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3536 push @$devices, @usbdevices if @usbdevices;
3539 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3540 if (my $path = $conf->{"serial$i"}) {
3541 if ($path eq 'socket') {
3542 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3543 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3544 # On aarch64, serial0 is the UART device. Qemu only allows
3545 # connecting UART devices via the '-serial' command line, as
3546 # the device has a fixed slot on the hardware...
3547 if ($arch eq 'aarch64' && $i == 0) {
3548 push @$devices, '-serial', "chardev:serial$i";
3550 push @$devices, '-device', "isa-serial,chardev=serial$i";
3553 die "no such serial device\n" if ! -c
$path;
3554 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3555 push @$devices, '-device', "isa-serial,chardev=serial$i";
3561 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3562 if (my $path = $conf->{"parallel$i"}) {
3563 die "no such parallel device\n" if ! -c
$path;
3564 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3565 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3566 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3570 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3571 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3572 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3573 push @$devices, @$audio_devs;
3576 add_tpm_device
($vmid, $devices, $conf);
3579 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3580 $sockets = $conf->{sockets
} if $conf->{sockets
};
3582 my $cores = $conf->{cores
} || 1;
3584 my $maxcpus = $sockets * $cores;
3586 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3588 my $allowed_vcpus = $cpuinfo->{cpus
};
3590 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3591 if ($allowed_vcpus < $maxcpus);
3593 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3595 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3596 for (my $i = 2; $i <= $vcpus; $i++) {
3597 my $cpustr = print_cpu_device
($conf,$i);
3598 push @$cmd, '-device', $cpustr;
3603 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3605 push @$cmd, '-nodefaults';
3607 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3609 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3611 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3613 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3614 push @$devices, '-device', print_vga_device
(
3615 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3616 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3617 push @$cmd, '-vnc', "unix:$socket,password=on";
3619 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3620 push @$cmd, '-nographic';
3624 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3625 my $useLocaltime = $conf->{localtime};
3627 if ($winversion >= 5) { # windows
3628 $useLocaltime = 1 if !defined($conf->{localtime});
3630 # use time drift fix when acpi is enabled
3631 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3632 $tdf = 1 if !defined($conf->{tdf
});
3636 if ($winversion >= 6) {
3637 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3638 push @$cmd, '-no-hpet';
3641 push @$rtcFlags, 'driftfix=slew' if $tdf;
3643 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3644 push @$rtcFlags, "base=$conf->{startdate}";
3645 } elsif ($useLocaltime) {
3646 push @$rtcFlags, 'base=localtime';
3650 push @$cmd, '-cpu', $forcecpu;
3652 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3655 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3657 push @$cmd, '-S' if $conf->{freeze
};
3659 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3661 my $guest_agent = parse_guest_agent
($conf);
3663 if ($guest_agent->{enabled
}) {
3664 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3665 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3667 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3668 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3669 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3670 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3671 } elsif ($guest_agent->{type
} eq 'isa') {
3672 push @$devices, '-device', "isa-serial,chardev=qga0";
3676 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3677 if ($rng && $version_guard->(4, 1, 2)) {
3678 check_rng_source
($rng->{source
});
3680 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3681 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3682 my $limiter_str = "";
3684 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3687 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3688 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3689 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3697 for (my $i = 1; $i < $qxlnum; $i++){
3698 push @$devices, '-device', print_vga_device
(
3699 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3702 # assume other OS works like Linux
3703 my ($ram, $vram) = ("134217728", "67108864");
3704 if ($vga->{memory
}) {
3705 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3706 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3708 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3709 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3713 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3715 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3716 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3717 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3719 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3720 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3721 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3723 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3724 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3726 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3727 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3728 if ($spice_enhancement->{foldersharing
}) {
3729 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3730 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3733 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3734 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3735 if $spice_enhancement->{videostreaming
};
3737 push @$devices, '-spice', "$spice_opts";
3740 # enable balloon by default, unless explicitly disabled
3741 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3742 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3743 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3746 if ($conf->{watchdog
}) {
3747 my $wdopts = parse_watchdog
($conf->{watchdog
});
3748 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3749 my $watchdog = $wdopts->{model
} || 'i6300esb';
3750 push @$devices, '-device', "$watchdog$pciaddr";
3751 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3755 my $scsicontroller = {};
3756 my $ahcicontroller = {};
3757 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3759 # Add iscsi initiator name if available
3760 if (my $initiator = get_initiator_name
()) {
3761 push @$devices, '-iscsi', "initiator-name=$initiator";
3764 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3765 my ($ds, $drive) = @_;
3767 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3768 check_volume_storage_type
($storecfg, $drive->{file
});
3769 push @$vollist, $drive->{file
};
3772 # ignore efidisk here, already added in bios/fw handling code above
3773 return if $drive->{interface
} eq 'efidisk';
3775 return if $drive->{interface
} eq 'tpmstate';
3777 $use_virtio = 1 if $ds =~ m/^virtio/;
3779 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3781 if ($drive->{interface
} eq 'virtio'){
3782 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3785 if ($drive->{interface
} eq 'scsi') {
3787 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3789 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3790 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3792 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3793 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3796 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3797 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3798 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3799 } elsif ($drive->{iothread
}) {
3800 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3804 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3805 $queues = ",num_queues=$drive->{queues}";
3808 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3809 if !$scsicontroller->{$controller};
3810 $scsicontroller->{$controller}=1;
3813 if ($drive->{interface
} eq 'sata') {
3814 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3815 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3816 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3817 if !$ahcicontroller->{$controller};
3818 $ahcicontroller->{$controller}=1;
3821 my $pbs_conf = $pbs_backing->{$ds};
3822 my $pbs_name = undef;
3824 $pbs_name = "drive-$ds-pbs";
3825 push @$devices, '-blockdev', print_pbs_blockdev
($pbs_conf, $pbs_name);
3828 my $drive_cmd = print_drive_commandline_full
(
3829 $storecfg, $vmid, $drive, $pbs_name, min_version
($kvmver, 6, 0));
3831 # extra protection for templates, but SATA and IDE don't support it..
3832 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
3834 push @$devices, '-drive',$drive_cmd;
3835 push @$devices, '-device', print_drivedevice_full
(
3836 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3839 for (my $i = 0; $i < $MAX_NETS; $i++) {
3840 my $netname = "net$i";
3842 next if !$conf->{$netname};
3843 my $d = parse_net
($conf->{$netname});
3846 $use_virtio = 1 if $d->{model
} eq 'virtio';
3848 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
3850 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
3851 push @$devices, '-netdev', $netdevfull;
3853 my $netdevicefull = print_netdevice_full
(
3854 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3856 push @$devices, '-device', $netdevicefull;
3859 if ($conf->{ivshmem
}) {
3860 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3864 $bus = print_pcie_addr
("ivshmem");
3866 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3869 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3870 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3872 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3873 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3874 .",size=$ivshmem->{size}M";
3877 # pci.4 is nested in pci.1
3878 $bridges->{1} = 1 if $bridges->{4};
3882 if (min_version
($machine_version, 2, 3)) {
3887 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3891 for my $k (sort {$b cmp $a} keys %$bridges) {
3892 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3895 if ($k == 2 && $legacy_igd) {
3898 $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
3900 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3902 # add after -readconfig pve-q35.cfg
3903 splice @$devices, 2, 0, '-device', $devstr;
3905 unshift @$devices, '-device', $devstr if $k > 0;
3910 push @$machineFlags, 'accel=tcg';
3913 my $machine_type_min = $machine_type;
3914 if ($add_pve_version) {
3915 $machine_type_min =~ s/\+pve\d+$//;
3916 $machine_type_min .= "+pve$required_pve_version";
3918 push @$machineFlags, "type=${machine_type_min}";
3920 push @$cmd, @$devices;
3921 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3922 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3923 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3925 if (my $vmstate = $conf->{vmstate
}) {
3926 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3927 push @$vollist, $vmstate;
3928 push @$cmd, '-loadstate', $statepath;
3929 print "activating and using '$vmstate' as vmstate\n";
3932 if (PVE
::QemuConfig-
>is_template($conf)) {
3933 # needed to workaround base volumes being read-only
3934 push @$cmd, '-snapshot';
3938 if ($conf->{args
}) {
3939 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3943 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3946 sub check_rng_source
{
3949 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3950 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3953 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3954 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
3955 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3956 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3957 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3958 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3966 my $res = mon_cmd
($vmid, 'query-spice');
3968 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3971 sub vm_devices_list
{
3974 my $res = mon_cmd
($vmid, 'query-pci');
3975 my $devices_to_check = [];
3977 foreach my $pcibus (@$res) {
3978 push @$devices_to_check, @{$pcibus->{devices
}},
3981 while (@$devices_to_check) {
3983 for my $d (@$devices_to_check) {
3984 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3985 next if !$d->{'pci_bridge'};
3987 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3988 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3990 $devices_to_check = $to_check;
3993 my $resblock = mon_cmd
($vmid, 'query-block');
3994 foreach my $block (@$resblock) {
3995 if($block->{device
} =~ m/^drive-(\S+)/){
4000 my $resmice = mon_cmd
($vmid, 'query-mice');
4001 foreach my $mice (@$resmice) {
4002 if ($mice->{name
} eq 'QEMU HID Tablet') {
4003 $devices->{tablet
} = 1;
4008 # for usb devices there is no query-usb
4009 # but we can iterate over the entries in
4010 # qom-list path=/machine/peripheral
4011 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4012 foreach my $per (@$resperipheral) {
4013 if ($per->{name
} =~ m/^usb\d+$/) {
4014 $devices->{$per->{name
}} = 1;
4022 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4024 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4026 my $devices_list = vm_devices_list
($vmid);
4027 return 1 if defined($devices_list->{$deviceid});
4029 # add PCI bridge if we need it for the device
4030 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4032 if ($deviceid eq 'tablet') {
4034 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4036 } elsif ($deviceid eq 'keyboard') {
4038 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4040 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4042 die "usb hotplug currently not reliable\n";
4043 # since we can't reliably hot unplug all added usb devices and usb
4044 # passthrough breaks live migration we disable usb hotplugging for now
4045 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4047 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4049 qemu_iothread_add
($vmid, $deviceid, $device);
4051 qemu_driveadd
($storecfg, $vmid, $device);
4052 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4054 qemu_deviceadd
($vmid, $devicefull);
4055 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4057 eval { qemu_drivedel
($vmid, $deviceid); };
4062 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4065 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4066 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4067 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4069 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4071 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4072 qemu_iothread_add
($vmid, $deviceid, $device);
4073 $devicefull .= ",iothread=iothread-$deviceid";
4076 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4077 $devicefull .= ",num_queues=$device->{queues}";
4080 qemu_deviceadd
($vmid, $devicefull);
4081 qemu_deviceaddverify
($vmid, $deviceid);
4083 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4085 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4086 qemu_driveadd
($storecfg, $vmid, $device);
4088 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4089 eval { qemu_deviceadd
($vmid, $devicefull); };
4091 eval { qemu_drivedel
($vmid, $deviceid); };
4096 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4098 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4100 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4101 my $use_old_bios_files = undef;
4102 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4104 my $netdevicefull = print_netdevice_full
(
4105 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4106 qemu_deviceadd
($vmid, $netdevicefull);
4108 qemu_deviceaddverify
($vmid, $deviceid);
4109 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4112 eval { qemu_netdevdel
($vmid, $deviceid); };
4117 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4120 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4121 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4123 qemu_deviceadd
($vmid, $devicefull);
4124 qemu_deviceaddverify
($vmid, $deviceid);
4127 die "can't hotplug device '$deviceid'\n";
4133 # fixme: this should raise exceptions on error!
4134 sub vm_deviceunplug
{
4135 my ($vmid, $conf, $deviceid) = @_;
4137 my $devices_list = vm_devices_list
($vmid);
4138 return 1 if !defined($devices_list->{$deviceid});
4140 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4141 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4143 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4144 qemu_devicedel
($vmid, $deviceid);
4145 } elsif ($deviceid =~ m/^usb\d+$/) {
4146 die "usb hotplug currently not reliable\n";
4147 # when unplugging usb devices this way, there may be remaining usb
4148 # controllers/hubs so we disable it for now
4149 #qemu_devicedel($vmid, $deviceid);
4150 #qemu_devicedelverify($vmid, $deviceid);
4151 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4152 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4154 qemu_devicedel
($vmid, $deviceid);
4155 qemu_devicedelverify
($vmid, $deviceid);
4156 qemu_drivedel
($vmid, $deviceid);
4157 qemu_iothread_del
($vmid, $deviceid, $device);
4158 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4159 qemu_devicedel
($vmid, $deviceid);
4160 qemu_devicedelverify
($vmid, $deviceid);
4161 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4162 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4164 qemu_devicedel
($vmid, $deviceid);
4165 qemu_drivedel
($vmid, $deviceid);
4166 qemu_deletescsihw
($conf, $vmid, $deviceid);
4168 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4169 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4170 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4171 qemu_devicedel
($vmid, $deviceid);
4172 qemu_devicedelverify
($vmid, $deviceid);
4173 qemu_netdevdel
($vmid, $deviceid);
4175 die "can't unplug device '$deviceid'\n";
4181 sub qemu_deviceadd
{
4182 my ($vmid, $devicefull) = @_;
4184 $devicefull = "driver=".$devicefull;
4185 my %options = split(/[=,]/, $devicefull);
4187 mon_cmd
($vmid, "device_add" , %options);
4190 sub qemu_devicedel
{
4191 my ($vmid, $deviceid) = @_;
4193 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4196 sub qemu_iothread_add
{
4197 my ($vmid, $deviceid, $device) = @_;
4199 if ($device->{iothread
}) {
4200 my $iothreads = vm_iothreads_list
($vmid);
4201 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4205 sub qemu_iothread_del
{
4206 my ($vmid, $deviceid, $device) = @_;
4208 if ($device->{iothread
}) {
4209 my $iothreads = vm_iothreads_list
($vmid);
4210 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4214 sub qemu_objectadd
{
4215 my ($vmid, $objectid, $qomtype) = @_;
4217 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4222 sub qemu_objectdel
{
4223 my ($vmid, $objectid) = @_;
4225 mon_cmd
($vmid, "object-del", id
=> $objectid);
4231 my ($storecfg, $vmid, $device) = @_;
4233 my $kvmver = get_running_qemu_version
($vmid);
4234 my $io_uring = min_version
($kvmver, 6, 0);
4235 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4236 $drive =~ s/\\/\\\\/g;
4237 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4239 # If the command succeeds qemu prints: "OK
"
4240 return 1 if $ret =~ m/OK/s;
4242 die "adding drive failed
: $ret\n";
4246 my ($vmid, $deviceid) = @_;
4248 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4251 return 1 if $ret eq "";
4253 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4254 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4256 die "deleting drive
$deviceid failed
: $ret\n";
4259 sub qemu_deviceaddverify {
4260 my ($vmid, $deviceid) = @_;
4262 for (my $i = 0; $i <= 5; $i++) {
4263 my $devices_list = vm_devices_list($vmid);
4264 return 1 if defined($devices_list->{$deviceid});
4268 die "error on hotplug device
'$deviceid'\n";
4272 sub qemu_devicedelverify {
4273 my ($vmid, $deviceid) = @_;
4275 # need to verify that the device is correctly removed as device_del
4276 # is async and empty return is not reliable
4278 for (my $i = 0; $i <= 5; $i++) {
4279 my $devices_list = vm_devices_list($vmid);
4280 return 1 if !defined($devices_list->{$deviceid});
4284 die "error on hot-unplugging device
'$deviceid'\n";
4287 sub qemu_findorcreatescsihw {
4288 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4290 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4292 my $scsihwid="$controller_prefix$controller";
4293 my $devices_list = vm_devices_list($vmid);
4295 if (!defined($devices_list->{$scsihwid})) {
4296 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4302 sub qemu_deletescsihw {
4303 my ($conf, $vmid, $opt) = @_;
4305 my $device = parse_drive($opt, $conf->{$opt});
4307 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4308 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4312 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4314 my $devices_list = vm_devices_list($vmid);
4315 foreach my $opt (keys %{$devices_list}) {
4316 if (is_valid_drivename($opt)) {
4317 my $drive = parse_drive($opt, $conf->{$opt});
4318 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4324 my $scsihwid="scsihw
$controller";
4326 vm_deviceunplug($vmid, $conf, $scsihwid);
4331 sub qemu_add_pci_bridge {
4332 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4338 print_pci_addr($device, $bridges, $arch, $machine_type);
4340 while (my ($k, $v) = each %$bridges) {
4343 return 1 if !defined($bridgeid) || $bridgeid < 1;
4345 my $bridge = "pci
.$bridgeid";
4346 my $devices_list = vm_devices_list($vmid);
4348 if (!defined($devices_list->{$bridge})) {
4349 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4355 sub qemu_set_link_status {
4356 my ($vmid, $device, $up) = @_;
4358 mon_cmd($vmid, "set_link
", name => $device,
4359 up => $up ? JSON::true : JSON::false);
4362 sub qemu_netdevadd {
4363 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4365 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4366 my %options = split(/[=,]/, $netdev);
4368 if (defined(my $vhost = $options{vhost})) {
4369 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4372 if (defined(my $queues = $options{queues})) {
4373 $options{queues} = $queues + 0;
4376 mon_cmd($vmid, "netdev_add
", %options);
4380 sub qemu_netdevdel {
4381 my ($vmid, $deviceid) = @_;
4383 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4386 sub qemu_usb_hotplug {
4387 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4391 # remove the old one first
4392 vm_deviceunplug($vmid, $conf, $deviceid);
4394 # check if xhci controller is necessary and available
4395 if ($device->{usb3}) {
4397 my $devicelist = vm_devices_list($vmid);
4399 if (!$devicelist->{xhci}) {
4400 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4401 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4404 my $d = parse_usb_device($device->{host});
4405 $d->{usb3} = $device->{usb3};
4408 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4411 sub qemu_cpu_hotplug {
4412 my ($vmid, $conf, $vcpus) = @_;
4414 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4417 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4418 $sockets = $conf->{sockets} if $conf->{sockets};
4419 my $cores = $conf->{cores} || 1;
4420 my $maxcpus = $sockets * $cores;
4422 $vcpus = $maxcpus if !$vcpus;
4424 die "you can
't add more vcpus than maxcpus\n"
4425 if $vcpus > $maxcpus;
4427 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4429 if ($vcpus < $currentvcpus) {
4431 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4433 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4434 qemu_devicedel($vmid, "cpu$i");
4436 my $currentrunningvcpus = undef;
4438 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4439 last if scalar(@{$currentrunningvcpus}) == $i-1;
4440 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4444 #update conf after each succesfull cpu unplug
4445 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4446 PVE::QemuConfig->write_config($vmid, $conf);
4449 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4455 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4456 die "vcpus in running vm does not match its configuration\n"
4457 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4459 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4461 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4462 my $cpustr = print_cpu_device($conf, $i);
4463 qemu_deviceadd($vmid, $cpustr);
4466 my $currentrunningvcpus = undef;
4468 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4469 last if scalar(@{$currentrunningvcpus}) == $i;
4470 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4474 #update conf after each succesfull cpu hotplug
4475 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4476 PVE::QemuConfig->write_config($vmid, $conf);
4480 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4481 mon_cmd($vmid, "cpu-add", id => int($i));
4486 sub qemu_block_set_io_throttle {
4487 my ($vmid, $deviceid,
4488 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4489 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4490 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4491 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4493 return if !check_running($vmid) ;
4495 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4497 bps_rd => int($bps_rd),
4498 bps_wr => int($bps_wr),
4500 iops_rd => int($iops_rd),
4501 iops_wr => int($iops_wr),
4502 bps_max => int($bps_max),
4503 bps_rd_max => int($bps_rd_max),
4504 bps_wr_max => int($bps_wr_max),
4505 iops_max => int($iops_max),
4506 iops_rd_max => int($iops_rd_max),
4507 iops_wr_max => int($iops_wr_max),
4508 bps_max_length => int($bps_max_length),
4509 bps_rd_max_length => int($bps_rd_max_length),
4510 bps_wr_max_length => int($bps_wr_max_length),
4511 iops_max_length => int($iops_max_length),
4512 iops_rd_max_length => int($iops_rd_max_length),
4513 iops_wr_max_length => int($iops_wr_max_length),
4518 sub qemu_block_resize {
4519 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4521 my $running = check_running($vmid);
4523 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4525 return if !$running;
4527 my $padding = (1024 - $size % 1024) % 1024;
4528 $size = $size + $padding;
4533 device => $deviceid,
4539 sub qemu_volume_snapshot {
4540 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4542 my $running = check_running($vmid);
4544 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4545 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4547 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4551 sub qemu_volume_snapshot_delete {
4552 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4554 my $running = check_running($vmid);
4559 my $conf = PVE::QemuConfig->load_config($vmid);
4560 PVE::QemuConfig->foreach_volume($conf, sub {
4561 my ($ds, $drive) = @_;
4562 $running = 1 if $drive->{file} eq $volid;
4566 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4567 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4569 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4573 sub set_migration_caps {
4574 my ($vmid, $savevm) = @_;
4576 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4578 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4579 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4584 "auto-converge" => 1,
4586 "x-rdma-pin-all" => 0,
4589 "dirty-bitmaps" => $dirty_bitmaps,
4592 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4594 for my $supported_capability (@$supported_capabilities) {
4596 capability => $supported_capability->{capability},
4597 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4601 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4605 my ($conf, $func, @param) = @_;
4609 my $test_volid = sub {
4610 my ($key, $drive, $snapname) = @_;
4612 my $volid = $drive->{file};
4615 $volhash->{$volid}->{cdrom} //= 1;
4616 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4618 my $replicate = $drive->{replicate} // 1;
4619 $volhash->{$volid}->{replicate} //= 0;
4620 $volhash->{$volid}->{replicate} = 1 if $replicate;
4622 $volhash->{$volid}->{shared} //= 0;
4623 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4625 $volhash->{$volid}->{referenced_in_config} //= 0;
4626 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4628 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4629 if defined($snapname);
4631 my $size = $drive->{size};
4632 $volhash->{$volid}->{size} //= $size if $size;
4634 $volhash->{$volid}->{is_vmstate} //= 0;
4635 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4637 $volhash->{$volid}->{is_tpmstate} //= 0;
4638 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0
';
4640 $volhash->{$volid}->{is_unused} //= 0;
4641 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4643 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4646 my $include_opts = {
4647 extra_keys => ['vmstate
'],
4648 include_unused => 1,
4651 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4652 foreach my $snapname (keys %{$conf->{snapshots}}) {
4653 my $snap = $conf->{snapshots}->{$snapname};
4654 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4657 foreach my $volid (keys %$volhash) {
4658 &$func($volid, $volhash->{$volid}, @param);
4662 my $fast_plug_option = {
4670 'vmstatestorage
' => 1,
4675 # hotplug changes in [PENDING]
4676 # $selection hash can be used to only apply specified options, for
4677 # example: { cores => 1 } (only apply changed 'cores
')
4678 # $errors ref is used to return error messages
4679 sub vmconfig_hotplug_pending {
4680 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4682 my $defaults = load_defaults();
4683 my $arch = get_vm_arch($conf);
4684 my $machine_type = get_vm_machine($conf, undef, $arch);
4686 # commit values which do not have any impact on running VM first
4687 # Note: those option cannot raise errors, we we do not care about
4688 # $selection and always apply them.
4690 my $add_error = sub {
4691 my ($opt, $msg) = @_;
4692 $errors->{$opt} = "hotplug problem - $msg";
4696 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4697 if ($fast_plug_option->{$opt}) {
4698 $conf->{$opt} = $conf->{pending}->{$opt};
4699 delete $conf->{pending}->{$opt};
4705 PVE::QemuConfig->write_config($vmid, $conf);
4708 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4710 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4711 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4712 foreach my $opt (sort keys %$pending_delete_hash) {
4713 next if $selection && !$selection->{$opt};
4714 my $force = $pending_delete_hash->{$opt}->{force};
4716 if ($opt eq 'hotplug
') {
4717 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4718 } elsif ($opt eq 'tablet
') {
4719 die "skip\n" if !$hotplug_features->{usb};
4720 if ($defaults->{tablet}) {
4721 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4722 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4723 if $arch eq 'aarch64
';
4725 vm_deviceunplug($vmid, $conf, 'tablet
');
4726 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4728 } elsif ($opt =~ m/^usb\d+/) {
4730 # since we cannot reliably hot unplug usb devices we are disabling it
4731 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4732 #vm_deviceunplug($vmid, $conf, $opt);
4733 } elsif ($opt eq 'vcpus
') {
4734 die "skip\n" if !$hotplug_features->{cpu};
4735 qemu_cpu_hotplug($vmid, $conf, undef);
4736 } elsif ($opt eq 'balloon
') {
4737 # enable balloon device is not hotpluggable
4738 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4739 # here we reset the ballooning value to memory
4740 my $balloon = $conf->{memory} || $defaults->{memory};
4741 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4742 } elsif ($fast_plug_option->{$opt}) {
4744 } elsif ($opt =~ m/^net(\d+)$/) {
4745 die "skip\n" if !$hotplug_features->{network};
4746 vm_deviceunplug($vmid, $conf, $opt);
4747 } elsif (is_valid_drivename($opt)) {
4748 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4749 vm_deviceunplug($vmid, $conf, $opt);
4750 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4751 } elsif ($opt =~ m/^memory$/) {
4752 die "skip\n" if !$hotplug_features->{memory};
4753 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4754 } elsif ($opt eq 'cpuunits
') {
4755 $cgroup->change_cpu_shares(undef, 1024);
4756 } elsif ($opt eq 'cpulimit
') {
4757 $cgroup->change_cpu_quota(-1, 100000);
4763 &$add_error($opt, $err) if $err ne "skip\n";
4765 delete $conf->{$opt};
4766 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4770 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4771 $apply_pending_cloudinit = sub {
4772 return if $apply_pending_cloudinit_done; # once is enough
4773 $apply_pending_cloudinit_done = 1; # once is enough
4775 my ($key, $value) = @_;
4777 my @cloudinit_opts = keys %$confdesc_cloudinit;
4778 foreach my $opt (keys %{$conf->{pending}}) {
4779 next if !grep { $_ eq $opt } @cloudinit_opts;
4780 $conf->{$opt} = delete $conf->{pending}->{$opt};
4783 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4784 foreach my $opt (sort keys %$pending_delete_hash) {
4785 next if !grep { $_ eq $opt } @cloudinit_opts;
4786 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4787 delete $conf->{$opt};
4790 my $new_conf = { %$conf };
4791 $new_conf->{$key} = $value;
4792 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4795 foreach my $opt (keys %{$conf->{pending}}) {
4796 next if $selection && !$selection->{$opt};
4797 my $value = $conf->{pending}->{$opt};
4799 if ($opt eq 'hotplug
') {
4800 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4801 } elsif ($opt eq 'tablet
') {
4802 die "skip\n" if !$hotplug_features->{usb};
4804 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4805 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4806 if $arch eq 'aarch64
';
4807 } elsif ($value == 0) {
4808 vm_deviceunplug($vmid, $conf, 'tablet
');
4809 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4811 } elsif ($opt =~ m/^usb\d+$/) {
4813 # since we cannot reliably hot unplug usb devices we disable it for now
4814 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4815 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4816 #die "skip\n" if !$d;
4817 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4818 } elsif ($opt eq 'vcpus
') {
4819 die "skip\n" if !$hotplug_features->{cpu};
4820 qemu_cpu_hotplug($vmid, $conf, $value);
4821 } elsif ($opt eq 'balloon
') {
4822 # enable/disable balloning device is not hotpluggable
4823 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4824 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4825 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4827 # allow manual ballooning if shares is set to zero
4828 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4829 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4830 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4832 } elsif ($opt =~ m/^net(\d+)$/) {
4833 # some changes can be done without hotplug
4834 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4835 $vmid, $opt, $value, $arch, $machine_type);
4836 } elsif (is_valid_drivename($opt)) {
4837 die "skip\n" if $opt eq 'efidisk0
' || $opt eq 'tpmstate0
';
4838 # some changes can be done without hotplug
4839 my $drive = parse_drive($opt, $value);
4840 if (drive_is_cloudinit($drive)) {
4841 &$apply_pending_cloudinit($opt, $value);
4843 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4844 $vmid, $opt, $value, $arch, $machine_type);
4845 } elsif ($opt =~ m/^memory$/) { #dimms
4846 die "skip\n" if !$hotplug_features->{memory};
4847 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4848 } elsif ($opt eq 'cpuunits
') {
4849 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4850 } elsif ($opt eq 'cpulimit
') {
4851 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4852 $cgroup->change_cpu_quota($cpulimit, 100000);
4854 die "skip\n"; # skip non-hot-pluggable options
4858 &$add_error($opt, $err) if $err ne "skip\n";
4860 $conf->{$opt} = $value;
4861 delete $conf->{pending}->{$opt};
4865 PVE::QemuConfig->write_config($vmid, $conf);
4868 sub try_deallocate_drive {
4869 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4871 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4872 my $volid = $drive->{file};
4873 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4874 my $sid = PVE::Storage::parse_volume_id($volid);
4875 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4877 # check if the disk is really unused
4878 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4879 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4880 PVE::Storage::vdisk_free($storecfg, $volid);
4883 # If vm is not owner of this disk remove from config
4891 sub vmconfig_delete_or_detach_drive {
4892 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4894 my $drive = parse_drive($opt, $conf->{$opt});
4896 my $rpcenv = PVE::RPCEnvironment::get();
4897 my $authuser = $rpcenv->get_user();
4900 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4901 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4903 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4909 sub vmconfig_apply_pending {
4910 my ($vmid, $conf, $storecfg, $errors) = @_;
4912 my $add_apply_error = sub {
4913 my ($opt, $msg) = @_;
4914 my $err_msg = "unable to apply pending change $opt : $msg";
4915 $errors->{$opt} = $err_msg;
4921 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4922 foreach my $opt (sort keys %$pending_delete_hash) {
4923 my $force = $pending_delete_hash->{$opt}->{force};
4925 if ($opt =~ m/^unused/) {
4926 die "internal error";
4927 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4928 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4932 $add_apply_error->($opt, $err);
4934 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4935 delete $conf->{$opt};
4939 PVE::QemuConfig->cleanup_pending($conf);
4941 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4942 next if $opt eq 'delete'; # just to be sure
4944 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4945 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4949 $add_apply_error->($opt, $err);
4951 $conf->{$opt} = delete $conf->{pending}->{$opt};
4955 # write all changes at once to avoid unnecessary i/o
4956 PVE::QemuConfig->write_config($vmid, $conf);
4959 sub vmconfig_update_net {
4960 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4962 my $newnet = parse_net($value);
4964 if ($conf->{$opt}) {
4965 my $oldnet = parse_net($conf->{$opt});
4967 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4968 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4969 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4970 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4972 # for non online change, we try to hot-unplug
4973 die "skip\n" if !$hotplug;
4974 vm_deviceunplug($vmid, $conf, $opt);
4977 die "internal error" if $opt !~ m/net(\d+)/;
4978 my $iface = "tap${vmid}i$1";
4980 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4981 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4982 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4983 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4984 PVE::Network::tap_unplug($iface);
4987 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4989 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4991 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4992 # Rate can be applied on its own but any change above needs to
4993 # include the rate in tap_plug since OVS resets everything.
4994 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4997 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4998 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5006 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5012 sub vmconfig_update_disk {
5013 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5015 my $drive = parse_drive($opt, $value);
5017 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5018 my $media = $drive->{media} || 'disk
';
5019 my $oldmedia = $old_drive->{media} || 'disk
';
5020 die "unable to change media type\n" if $media ne $oldmedia;
5022 if (!drive_is_cdrom($old_drive)) {
5024 if ($drive->{file} ne $old_drive->{file}) {
5026 die "skip\n" if !$hotplug;
5028 # unplug and register as unused
5029 vm_deviceunplug($vmid, $conf, $opt);
5030 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5033 # update existing disk
5035 # skip non hotpluggable value
5036 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5037 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5038 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5039 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5040 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5045 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5046 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5047 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5048 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5049 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5050 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5051 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5052 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5053 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5054 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5055 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5056 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5057 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5058 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5059 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5060 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5061 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5062 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5064 qemu_block_set_io_throttle(
5066 ($drive->{mbps} || 0)*1024*1024,
5067 ($drive->{mbps_rd} || 0)*1024*1024,
5068 ($drive->{mbps_wr} || 0)*1024*1024,
5069 $drive->{iops} || 0,
5070 $drive->{iops_rd} || 0,
5071 $drive->{iops_wr} || 0,
5072 ($drive->{mbps_max} || 0)*1024*1024,
5073 ($drive->{mbps_rd_max} || 0)*1024*1024,
5074 ($drive->{mbps_wr_max} || 0)*1024*1024,
5075 $drive->{iops_max} || 0,
5076 $drive->{iops_rd_max} || 0,
5077 $drive->{iops_wr_max} || 0,
5078 $drive->{bps_max_length} || 1,
5079 $drive->{bps_rd_max_length} || 1,
5080 $drive->{bps_wr_max_length} || 1,
5081 $drive->{iops_max_length} || 1,
5082 $drive->{iops_rd_max_length} || 1,
5083 $drive->{iops_wr_max_length} || 1,
5093 if ($drive->{file} eq 'none
') {
5094 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5095 if (drive_is_cloudinit($old_drive)) {
5096 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5099 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5101 # force eject if locked
5102 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5105 mon_cmd($vmid, "blockdev-change-medium",
5106 id => "$opt", filename => "$path");
5114 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5116 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5117 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5120 # called in locked context by incoming migration
5121 sub vm_migrate_get_nbd_disks {
5122 my ($storecfg, $conf, $replicated_volumes) = @_;
5124 my $local_volumes = {};
5125 PVE::QemuConfig->foreach_volume($conf, sub {
5126 my ($ds, $drive) = @_;
5128 return if drive_is_cdrom($drive);
5130 my $volid = $drive->{file};
5134 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5136 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5137 return if $scfg->{shared};
5139 # replicated disks re-use existing state via bitmap
5140 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5141 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5143 return $local_volumes;
5146 # called in locked context by incoming migration
5147 sub vm_migrate_alloc_nbd_disks {
5148 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5153 foreach my $opt (sort keys %$source_volumes) {
5154 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5156 if ($use_existing) {
5157 $nbd->{$opt}->{drivestr} = print_drive($drive);
5158 $nbd->{$opt}->{volid} = $volid;
5159 $nbd->{$opt}->{replicated} = 1;
5163 # If a remote storage is specified and the format of the original
5164 # volume is not available there, fall back to the default format.
5165 # Otherwise use the same format as the original.
5166 if (!$storagemap->{identity}) {
5167 $storeid = map_storage($storagemap, $storeid);
5168 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5169 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5170 my $fileFormat = qemu_img_format($scfg, $volname);
5171 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5173 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5174 $format = qemu_img_format($scfg, $volname);
5177 my $size = $drive->{size} / 1024;
5178 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5179 my $newdrive = $drive;
5180 $newdrive->{format} = $format;
5181 $newdrive->{file} = $newvolid;
5182 my $drivestr = print_drive($newdrive);
5183 $nbd->{$opt}->{drivestr} = $drivestr;
5184 $nbd->{$opt}->{volid} = $newvolid;
5190 # see vm_start_nolock for parameters, additionally:
5192 # storagemap = parsed storage map for allocating NBD disks
5194 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5196 return PVE::QemuConfig->lock_config($vmid, sub {
5197 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5199 die "you can't start a vm
if it
's a template\n"
5200 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5202 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended
');
5203 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup
');
5205 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5207 if ($has_backup_lock && $running) {
5208 # a backup is currently running, attempt to start the guest in the
5209 # existing QEMU instance
5210 return vm_resume($vmid);
5213 PVE::QemuConfig->check_lock($conf)
5214 if !($params->{skiplock} || $has_suspended_lock);
5216 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5218 die "VM $vmid already running\n" if $running;
5220 if (my $storagemap = $migrate_opts->{storagemap}) {
5221 my $replicated = $migrate_opts->{replicated_volumes};
5222 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5223 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5225 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5226 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5230 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5236 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
5237 # skiplock => 0/1, skip checking for config lock
5238 # skiptemplate => 0/1, skip checking whether VM is template
5239 # forcemachine => to force Qemu machine (rollback/migration)
5240 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
5241 # timeout => in seconds
5242 # paused => start VM in paused state (backup)
5243 # resume => resume from hibernation
5254 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5255 # migratedfrom => source node
5256 # spice_ticket => used for spice migration, passed via tunnel/stdin
5257 # network => CIDR of migration network
5258 # type => secure/insecure - tunnel over encrypted connection or plain-text
5259 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5260 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5261 sub vm_start_nolock {
5262 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5264 my $statefile = $params->{statefile};
5265 my $resume = $params->{resume};
5267 my $migratedfrom = $migrate_opts->{migratedfrom};
5268 my $migration_type = $migrate_opts->{type};
5272 # clean up leftover reboot request files
5273 eval { clear_reboot_request($vmid); };
5276 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5277 vmconfig_apply_pending($vmid, $conf, $storecfg);
5278 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5281 # don't regenerate the ISO
if the VM
is started as part of a live migration
5282 # this way we can reuse the old ISO with the correct config
5283 PVE
::QemuServer
::Cloudinit
::generate_cloudinitconfig
($conf, $vmid) if !$migratedfrom;
5285 my $defaults = load_defaults
();
5287 # set environment variable useful inside network script
5288 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom if $migratedfrom;
5290 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5292 my $forcemachine = $params->{forcemachine
};
5293 my $forcecpu = $params->{forcecpu
};
5295 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5296 $forcemachine = $conf->{runningmachine
};
5297 $forcecpu = $conf->{runningcpu
};
5298 print "Resuming suspended VM\n";
5301 my ($cmd, $vollist, $spice_port) = config_to_command
($storecfg, $vmid,
5302 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5305 my $get_migration_ip = sub {
5306 my ($nodename) = @_;
5308 return $migration_ip if defined($migration_ip);
5310 my $cidr = $migrate_opts->{network
};
5312 if (!defined($cidr)) {
5313 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5314 $cidr = $dc_conf->{migration
}->{network
};
5317 if (defined($cidr)) {
5318 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5320 die "could not get IP: no address configured on local " .
5321 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5323 die "could not get IP: multiple addresses configured on local " .
5324 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5326 $migration_ip = @$ips[0];
5329 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5330 if !defined($migration_ip);
5332 return $migration_ip;
5337 if ($statefile eq 'tcp') {
5338 my $localip = "localhost";
5339 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5340 my $nodename = nodename
();
5342 if (!defined($migration_type)) {
5343 if (defined($datacenterconf->{migration
}->{type
})) {
5344 $migration_type = $datacenterconf->{migration
}->{type
};
5346 $migration_type = 'secure';
5350 if ($migration_type eq 'insecure') {
5351 $localip = $get_migration_ip->($nodename);
5352 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5355 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5356 my $migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5357 $migrate_uri = "tcp:${localip}:${migrate_port}";
5358 push @$cmd, '-incoming', $migrate_uri;
5361 } elsif ($statefile eq 'unix') {
5362 # should be default for secure migrations as a ssh TCP forward
5363 # tunnel is not deterministic reliable ready and fails regurarly
5364 # to set up in time, so use UNIX socket forwards
5365 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5366 unlink $socket_addr;
5368 $migrate_uri = "unix:$socket_addr";
5370 push @$cmd, '-incoming', $migrate_uri;
5373 } elsif (-e
$statefile) {
5374 push @$cmd, '-loadstate', $statefile;
5376 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5377 push @$vollist, $statefile;
5378 push @$cmd, '-loadstate', $statepath;
5380 } elsif ($params->{paused
}) {
5384 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $resume);
5386 my $pci_devices = {}; # host pci devices
5387 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
5388 my $dev = $conf->{"hostpci$i"} or next;
5389 $pci_devices->{$i} = parse_hostpci
($dev);
5392 my $pci_id_list = [ map { $_->{id
} } map { $_->{pciid
}->@* } values $pci_devices->%* ];
5393 # reserve all PCI IDs before actually doing anything with them
5394 PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_id_list, $vmid, $start_timeout);
5397 for my $id (sort keys %$pci_devices) {
5398 my $d = $pci_devices->{$id};
5399 for my $dev ($d->{pciid
}->@*) {
5400 PVE
::QemuServer
::PCI
::prepare_pci_device
($vmid, $dev->{id
}, $id, $d->{mdev
});
5405 eval { PVE
::QemuServer
::PCI
::remove_pci_reservation
($pci_id_list) };
5410 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5413 run_command
(['/bin/systemctl', 'stop', "$vmid.scope"],
5414 outfunc
=> sub {}, errfunc
=> sub {});
5416 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5417 # timeout should be more than enough here...
5418 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 5);
5420 my $cpuunits = get_cpuunits
($conf);
5423 timeout
=> $statefile ?
undef : $start_timeout,
5428 # when migrating, prefix QEMU output so other side can pick up any
5429 # errors that might occur and show the user
5430 if ($migratedfrom) {
5431 $run_params{quiet
} = 1;
5432 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5436 Slice
=> 'qemu.slice',
5437 KillMode
=> 'process',
5439 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5442 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5443 $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5444 $properties{CPUWeight
} = $cpuunits;
5446 $properties{CPUShares
} = $cpuunits;
5449 if (my $cpulimit = $conf->{cpulimit
}) {
5450 $properties{CPUQuota
} = int($cpulimit * 100);
5452 $properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5454 my $run_qemu = sub {
5455 PVE
::Tools
::run_fork
sub {
5456 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %properties);
5459 if (my $tpm = $conf->{tpmstate0
}) {
5460 # start the TPM emulator so QEMU can connect on start
5461 $tpmpid = start_swtpm
($storecfg, $vmid, $tpm, $migratedfrom);
5464 my $exitcode = run_command
($cmd, %run_params);
5466 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5467 kill 'TERM', $tpmpid if $tpmpid;
5468 die "QEMU exited with code $exitcode\n";
5473 if ($conf->{hugepages
}) {
5476 my $hugepages_topology = PVE
::QemuServer
::Memory
::hugepages_topology
($conf);
5477 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5479 PVE
::QemuServer
::Memory
::hugepages_mount
();
5480 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5482 eval { $run_qemu->() };
5484 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5485 if !$conf->{keephugepages
};
5489 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5490 if !$conf->{keephugepages
};
5492 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5495 eval { $run_qemu->() };
5499 # deactivate volumes if start fails
5500 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5501 eval { PVE
::QemuServer
::PCI
::remove_pci_reservation
($pci_id_list) };
5503 die "start failed: $err";
5506 # re-reserve all PCI IDs now that we can know the actual VM PID
5507 my $pid = PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
5508 eval { PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_id_list, $vmid, undef, $pid) };
5511 print "migration listens on $migrate_uri\n" if $migrate_uri;
5512 $res->{migrate_uri
} = $migrate_uri;
5514 if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
5515 eval { mon_cmd
($vmid, "cont"); };
5519 #start nbd server for storage migration
5520 if (my $nbd = $migrate_opts->{nbd
}) {
5521 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5523 my $migrate_storage_uri;
5524 # nbd_protocol_version > 0 for unix socket support
5525 if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5526 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5527 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
5528 $migrate_storage_uri = "nbd:unix:$socket_path";
5530 my $nodename = nodename
();
5531 my $localip = $get_migration_ip->($nodename);
5532 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5533 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5535 mon_cmd
($vmid, "nbd-server-start", addr
=> {
5538 host
=> "${localip}",
5539 port
=> "${storage_migrate_port}",
5542 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5543 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5546 $res->{migrate_storage_uri
} = $migrate_storage_uri;
5548 foreach my $opt (sort keys %$nbd) {
5549 my $drivestr = $nbd->{$opt}->{drivestr
};
5550 my $volid = $nbd->{$opt}->{volid
};
5551 mon_cmd
($vmid, "nbd-server-add", device
=> "drive-$opt", writable
=> JSON
::true
);
5552 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5553 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5554 print "re-using replicated volume: $opt - $volid\n"
5555 if $nbd->{$opt}->{replicated
};
5557 $res->{drives
}->{$opt} = $nbd->{$opt};
5558 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
5562 if ($migratedfrom) {
5564 set_migration_caps
($vmid);
5569 print "spice listens on port $spice_port\n";
5570 $res->{spice_port
} = $spice_port;
5571 if ($migrate_opts->{spice_ticket
}) {
5572 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
5573 $migrate_opts->{spice_ticket
});
5574 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
5579 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
5580 if !$statefile && $conf->{balloon
};
5582 foreach my $opt (keys %$conf) {
5583 next if $opt !~ m/^net\d+$/;
5584 my $nicconf = parse_net
($conf->{$opt});
5585 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
5589 mon_cmd
($vmid, 'qom-set',
5590 path
=> "machine/peripheral/balloon0",
5591 property
=> "guest-stats-polling-interval",
5592 value
=> 2) if (!defined($conf->{balloon
}) || $conf->{balloon
});
5595 print "Resumed VM, removing state\n";
5596 if (my $vmstate = $conf->{vmstate
}) {
5597 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5598 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5600 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5601 PVE
::QemuConfig-
>write_config($vmid, $conf);
5604 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5609 sub vm_commandline
{
5610 my ($storecfg, $vmid, $snapname) = @_;
5612 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5617 my $snapshot = $conf->{snapshots
}->{$snapname};
5618 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5620 # check for machine or CPU overrides in snapshot
5621 $forcemachine = $snapshot->{runningmachine
};
5622 $forcecpu = $snapshot->{runningcpu
};
5624 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5629 my $defaults = load_defaults
();
5631 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults,
5632 $forcemachine, $forcecpu);
5634 return PVE
::Tools
::cmd2string
($cmd);
5638 my ($vmid, $skiplock) = @_;
5640 PVE
::QemuConfig-
>lock_config($vmid, sub {
5642 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5644 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5646 mon_cmd
($vmid, "system_reset");
5650 sub get_vm_volumes
{
5654 foreach_volid
($conf, sub {
5655 my ($volid, $attr) = @_;
5657 return if $volid =~ m
|^/|;
5659 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5662 push @$vollist, $volid;
5668 sub vm_stop_cleanup
{
5669 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5674 my $vollist = get_vm_volumes
($conf);
5675 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5677 if (my $tpmdrive = $conf->{tpmstate0
}) {
5678 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
5679 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
5681 PVE
::Storage
::unmap_volume
($storecfg, $tpm->{file
});
5686 foreach my $ext (qw(mon qmp pid vnc qga)) {
5687 unlink "/var/run/qemu-server/${vmid}.$ext";
5690 if ($conf->{ivshmem
}) {
5691 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5692 # just delete it for now, VMs which have this already open do not
5693 # are affected, but new VMs will get a separated one. If this
5694 # becomes an issue we either add some sort of ref-counting or just
5695 # add a "don't delete on stop" flag to the ivshmem format.
5696 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5700 foreach my $key (keys %$conf) {
5701 next if $key !~ m/^hostpci(\d+)$/;
5702 my $hostpciindex = $1;
5703 my $d = parse_hostpci
($conf->{$key});
5704 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5706 foreach my $pci (@{$d->{pciid
}}) {
5707 my $pciid = $pci->{id
};
5708 push @$ids, $pci->{id
};
5709 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5712 PVE
::QemuServer
::PCI
::remove_pci_reservation
($ids);
5714 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5716 warn $@ if $@; # avoid errors - just warn
5719 # call only in locked context
5721 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5723 my $pid = check_running
($vmid, $nocheck);
5728 $conf = PVE
::QemuConfig-
>load_config($vmid);
5729 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5730 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5731 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5732 $timeout = $opts->{down
} if $opts->{down
};
5734 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5739 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
5740 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5742 mon_cmd
($vmid, "system_powerdown");
5745 mon_cmd
($vmid, "quit");
5751 $timeout = 60 if !defined($timeout);
5754 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5759 if ($count >= $timeout) {
5761 warn "VM still running - terminating now with SIGTERM\n";
5764 die "VM quit/powerdown failed - got timeout\n";
5767 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5771 if (!check_running
($vmid, $nocheck)) {
5772 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5776 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5779 die "VM quit/powerdown failed\n";
5787 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5792 if ($count >= $timeout) {
5793 warn "VM still running - terminating now with SIGKILL\n";
5798 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5801 # Note: use $nocheck to skip tests if VM configuration file exists.
5802 # We need that when migration VMs to other nodes (files already moved)
5803 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5805 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5807 $force = 1 if !defined($force) && !$shutdown;
5810 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5811 kill 15, $pid if $pid;
5812 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5813 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5817 PVE
::QemuConfig-
>lock_config($vmid, sub {
5818 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5823 my ($vmid, $timeout) = @_;
5825 PVE
::QemuConfig-
>lock_config($vmid, sub {
5828 # only reboot if running, as qmeventd starts it again on a stop event
5829 return if !check_running
($vmid);
5831 create_reboot_request
($vmid);
5833 my $storecfg = PVE
::Storage
::config
();
5834 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5838 # avoid that the next normal shutdown will be confused for a reboot
5839 clear_reboot_request
($vmid);
5845 # note: if using the statestorage parameter, the caller has to check privileges
5847 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5854 PVE
::QemuConfig-
>lock_config($vmid, sub {
5856 $conf = PVE
::QemuConfig-
>load_config($vmid);
5858 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5859 PVE
::QemuConfig-
>check_lock($conf)
5860 if !($skiplock || $is_backing_up);
5862 die "cannot suspend to disk during backup\n"
5863 if $is_backing_up && $includestate;
5865 if ($includestate) {
5866 $conf->{lock} = 'suspending';
5867 my $date = strftime
("%Y-%m-%d", localtime(time()));
5868 $storecfg = PVE
::Storage
::config
();
5869 if (!$statestorage) {
5870 $statestorage = find_vmstate_storage
($conf, $storecfg);
5871 # check permissions for the storage
5872 my $rpcenv = PVE
::RPCEnvironment
::get
();
5873 if ($rpcenv->{type
} ne 'cli') {
5874 my $authuser = $rpcenv->get_user();
5875 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5880 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
5881 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5882 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5883 PVE
::QemuConfig-
>write_config($vmid, $conf);
5885 mon_cmd
($vmid, "stop");
5889 if ($includestate) {
5891 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5894 set_migration_caps
($vmid, 1);
5895 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5897 my $state = mon_cmd
($vmid, "query-savevm");
5898 if (!$state->{status
}) {
5899 die "savevm not active\n";
5900 } elsif ($state->{status
} eq 'active') {
5903 } elsif ($state->{status
} eq 'completed') {
5904 print "State saved, quitting\n";
5906 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5907 die "query-savevm failed with error '$state->{error}'\n"
5909 die "query-savevm returned status '$state->{status}'\n";
5915 PVE
::QemuConfig-
>lock_config($vmid, sub {
5916 $conf = PVE
::QemuConfig-
>load_config($vmid);
5918 # cleanup, but leave suspending lock, to indicate something went wrong
5920 mon_cmd
($vmid, "savevm-end");
5921 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5922 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5923 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5924 PVE
::QemuConfig-
>write_config($vmid, $conf);
5930 die "lock changed unexpectedly\n"
5931 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5933 mon_cmd
($vmid, "quit");
5934 $conf->{lock} = 'suspended';
5935 PVE
::QemuConfig-
>write_config($vmid, $conf);
5941 my ($vmid, $skiplock, $nocheck) = @_;
5943 PVE
::QemuConfig-
>lock_config($vmid, sub {
5944 my $res = mon_cmd
($vmid, 'query-status');
5945 my $resume_cmd = 'cont';
5948 if ($res->{status
}) {
5949 return if $res->{status
} eq 'running'; # job done, go home
5950 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
5951 $reset = 1 if $res->{status
} eq 'shutdown';
5956 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5958 PVE
::QemuConfig-
>check_lock($conf)
5959 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5963 # required if a VM shuts down during a backup and we get a resume
5964 # request before the backup finishes for example
5965 mon_cmd
($vmid, "system_reset");
5967 mon_cmd
($vmid, $resume_cmd);
5972 my ($vmid, $skiplock, $key) = @_;
5974 PVE
::QemuConfig-
>lock_config($vmid, sub {
5976 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5978 # there is no qmp command, so we use the human monitor command
5979 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5980 die $res if $res ne '';
5984 # vzdump restore implementaion
5986 sub tar_archive_read_firstfile
{
5987 my $archive = shift;
5989 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5991 # try to detect archive type first
5992 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5993 die "unable to open file '$archive'\n";
5994 my $firstfile = <$fh>;
5998 die "ERROR: archive contaions no data\n" if !$firstfile;
6004 sub tar_restore_cleanup
{
6005 my ($storecfg, $statfile) = @_;
6007 print STDERR
"starting cleanup\n";
6009 if (my $fd = IO
::File-
>new($statfile, "r")) {
6010 while (defined(my $line = <$fd>)) {
6011 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6014 if ($volid =~ m
|^/|) {
6015 unlink $volid || die 'unlink failed\n';
6017 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6019 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6021 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6023 print STDERR
"unable to parse line in statfile - $line";
6030 sub restore_file_archive
{
6031 my ($archive, $vmid, $user, $opts) = @_;
6033 return restore_vma_archive
($archive, $vmid, $user, $opts)
6036 my $info = PVE
::Storage
::archive_info
($archive);
6037 my $format = $opts->{format
} // $info->{format
};
6038 my $comp = $info->{compression
};
6040 # try to detect archive format
6041 if ($format eq 'tar') {
6042 return restore_tar_archive
($archive, $vmid, $user, $opts);
6044 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
6048 # hepler to remove disks that will not be used after restore
6049 my $restore_cleanup_oldconf = sub {
6050 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6052 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
6053 my ($ds, $drive) = @_;
6055 return if drive_is_cdrom
($drive, 1);
6057 my $volid = $drive->{file
};
6058 return if !$volid || $volid =~ m
|^/|;
6060 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
6061 return if !$path || !$owner || ($owner != $vmid);
6063 # Note: only delete disk we want to restore
6064 # other volumes will become unused
6065 if ($virtdev_hash->{$ds}) {
6066 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
6073 # delete vmstate files, after the restore we have no snapshots anymore
6074 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
6075 my $snap = $oldconf->{snapshots
}->{$snapname};
6076 if ($snap->{vmstate
}) {
6077 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
6085 # Helper to parse vzdump backup device hints
6087 # $rpcenv: Environment, used to ckeck storage permissions
6088 # $user: User ID, to check storage permissions
6089 # $storecfg: Storage configuration
6090 # $fh: the file handle for reading the configuration
6091 # $devinfo: should contain device sizes for all backu-up'ed devices
6092 # $options: backup options (pool, default storage)
6094 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6095 my $parse_backup_hints = sub {
6096 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6098 my $virtdev_hash = {};
6100 while (defined(my $line = <$fh>)) {
6101 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6102 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6103 die "archive does not contain data for drive '$virtdev'\n"
6104 if !$devinfo->{$devname};
6106 if (defined($options->{storage
})) {
6107 $storeid = $options->{storage
} || 'local';
6108 } elsif (!$storeid) {
6111 $format = 'raw' if !$format;
6112 $devinfo->{$devname}->{devname
} = $devname;
6113 $devinfo->{$devname}->{virtdev
} = $virtdev;
6114 $devinfo->{$devname}->{format
} = $format;
6115 $devinfo->{$devname}->{storeid
} = $storeid;
6117 # check permission on storage
6118 my $pool = $options->{pool
}; # todo: do we need that?
6119 if ($user ne 'root@pam') {
6120 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6123 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6124 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6126 my $drive = parse_drive
($virtdev, $2);
6127 if (drive_is_cloudinit
($drive)) {
6128 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6129 $storeid = $options->{storage
} if defined ($options->{storage
});
6130 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6131 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6133 $virtdev_hash->{$virtdev} = {
6135 storeid
=> $storeid,
6136 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6143 return $virtdev_hash;
6146 # Helper to allocate and activate all volumes required for a restore
6148 # $storecfg: Storage configuration
6149 # $virtdev_hash: as returned by parse_backup_hints()
6151 # Returns: { $virtdev => $volid }
6152 my $restore_allocate_devices = sub {
6153 my ($storecfg, $virtdev_hash, $vmid) = @_;
6156 foreach my $virtdev (sort keys %$virtdev_hash) {
6157 my $d = $virtdev_hash->{$virtdev};
6158 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6159 my $storeid = $d->{storeid
};
6160 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6162 # test if requested format is supported
6163 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6164 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6165 $d->{format
} = $defFormat if !$supported;
6168 if ($d->{is_cloudinit
}) {
6169 $name = "vm-$vmid-cloudinit";
6170 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6171 if ($scfg->{path
}) {
6172 $name .= ".$d->{format}";
6176 my $volid = PVE
::Storage
::vdisk_alloc
(
6177 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6179 print STDERR
"new volume ID is '$volid'\n";
6180 $d->{volid
} = $volid;
6182 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6184 $map->{$virtdev} = $volid;
6190 sub restore_update_config_line
{
6191 my ($cookie, $map, $line, $unique) = @_;
6193 return '' if $line =~ m/^\#qmdump\#/;
6194 return '' if $line =~ m/^\#vzdump\#/;
6195 return '' if $line =~ m/^lock:/;
6196 return '' if $line =~ m/^unused\d+:/;
6197 return '' if $line =~ m/^parent:/;
6201 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6202 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6203 # try to convert old 1.X settings
6204 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6205 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6206 my ($model, $macaddr) = split(/\=/, $devconfig);
6207 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6210 bridge
=> "vmbr$ind",
6211 macaddr
=> $macaddr,
6213 my $netstr = print_net
($net);
6215 $res .= "net$cookie->{netcount}: $netstr\n";
6216 $cookie->{netcount
}++;
6218 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6219 my ($id, $netstr) = ($1, $2);
6220 my $net = parse_net
($netstr);
6221 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6222 $netstr = print_net
($net);
6223 $res .= "$id: $netstr\n";
6224 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6227 my $di = parse_drive
($virtdev, $value);
6228 if (defined($di->{backup
}) && !$di->{backup
}) {
6230 } elsif ($map->{$virtdev}) {
6231 delete $di->{format
}; # format can change on restore
6232 $di->{file
} = $map->{$virtdev};
6233 $value = print_drive
($di);
6234 $res .= "$virtdev: $value\n";
6238 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6240 if ($vmgenid ne '0') {
6241 # always generate a new vmgenid if there was a valid one setup
6242 $vmgenid = generate_uuid
();
6244 $res .= "vmgenid: $vmgenid\n";
6245 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6246 my ($uuid, $uuid_str);
6247 UUID
::generate
($uuid);
6248 UUID
::unparse
($uuid, $uuid_str);
6249 my $smbios1 = parse_smbios1
($2);
6250 $smbios1->{uuid
} = $uuid_str;
6251 $res .= $1.print_smbios1
($smbios1)."\n";
6259 my $restore_deactivate_volumes = sub {
6260 my ($storecfg, $devinfo) = @_;
6263 foreach my $devname (keys %$devinfo) {
6264 my $volid = $devinfo->{$devname}->{volid
};
6265 push @$vollist, $volid if $volid;
6268 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6271 my $restore_destroy_volumes = sub {
6272 my ($storecfg, $devinfo) = @_;
6274 foreach my $devname (keys %$devinfo) {
6275 my $volid = $devinfo->{$devname}->{volid
};
6278 if ($volid =~ m
|^/|) {
6279 unlink $volid || die 'unlink failed\n';
6281 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6283 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6285 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6290 my ($cfg, $vmid) = @_;
6292 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6294 my $volid_hash = {};
6295 foreach my $storeid (keys %$info) {
6296 foreach my $item (@{$info->{$storeid}}) {
6297 next if !($item->{volid
} && $item->{size
});
6298 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6299 $volid_hash->{$item->{volid
}} = $item;
6306 sub update_disk_config
{
6307 my ($vmid, $conf, $volid_hash) = @_;
6310 my $prefix = "VM $vmid";
6312 # used and unused disks
6313 my $referenced = {};
6315 # Note: it is allowed to define multiple storages with same path (alias), so
6316 # we need to check both 'volid' and real 'path' (two different volid can point
6317 # to the same path).
6319 my $referencedpath = {};
6322 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6323 my ($opt, $drive) = @_;
6325 my $volid = $drive->{file
};
6327 my $volume = $volid_hash->{$volid};
6329 # mark volid as "in-use" for next step
6330 $referenced->{$volid} = 1;
6331 if ($volume && (my $path = $volume->{path
})) {
6332 $referencedpath->{$path} = 1;
6335 return if drive_is_cdrom
($drive);
6338 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6339 if (defined($updated)) {
6341 $conf->{$opt} = print_drive
($updated);
6342 print "$prefix ($opt): $msg\n";
6346 # remove 'unusedX' entry if volume is used
6347 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6348 my ($opt, $drive) = @_;
6350 my $volid = $drive->{file
};
6354 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6355 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6356 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6358 delete $conf->{$opt};
6361 $referenced->{$volid} = 1;
6362 $referencedpath->{$path} = 1 if $path;
6365 foreach my $volid (sort keys %$volid_hash) {
6366 next if $volid =~ m/vm-$vmid-state-/;
6367 next if $referenced->{$volid};
6368 my $path = $volid_hash->{$volid}->{path
};
6369 next if !$path; # just to be sure
6370 next if $referencedpath->{$path};
6372 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6373 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6374 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6381 my ($vmid, $nolock, $dryrun) = @_;
6383 my $cfg = PVE
::Storage
::config
();
6385 print "rescan volumes...\n";
6386 my $volid_hash = scan_volids
($cfg, $vmid);
6388 my $updatefn = sub {
6391 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6393 PVE
::QemuConfig-
>check_lock($conf);
6396 foreach my $volid (keys %$volid_hash) {
6397 my $info = $volid_hash->{$volid};
6398 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6401 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6403 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6406 if (defined($vmid)) {
6410 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6413 my $vmlist = config_list
();
6414 foreach my $vmid (keys %$vmlist) {
6418 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6424 sub restore_proxmox_backup_archive
{
6425 my ($archive, $vmid, $user, $options) = @_;
6427 my $storecfg = PVE
::Storage
::config
();
6429 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6430 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6432 my $fingerprint = $scfg->{fingerprint
};
6433 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6435 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6437 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6438 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6439 local $ENV{PBS_PASSWORD
} = $password;
6440 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6442 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6443 PVE
::Storage
::parse_volname
($storecfg, $archive);
6445 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6447 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6449 my $tmpdir = "/var/tmp/vzdumptmp$$";
6453 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6454 # disable interrupts (always do cleanups)
6458 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6460 # Note: $oldconf is undef if VM does not exists
6461 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6462 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6463 my $new_conf_raw = '';
6465 my $rpcenv = PVE
::RPCEnvironment
::get
();
6474 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6476 my $cfgfn = "$tmpdir/qemu-server.conf";
6477 my $firewall_config_fn = "$tmpdir/fw.conf";
6478 my $index_fn = "$tmpdir/index.json";
6480 my $cmd = "restore";
6482 my $param = [$pbs_backup_name, "index.json", $index_fn];
6483 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6484 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6485 $index = decode_json
($index);
6487 # print Dumper($index);
6488 foreach my $info (@{$index->{files
}}) {
6489 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6491 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6492 $devinfo->{$devname}->{size
} = $1;
6494 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6499 my $is_qemu_server_backup = scalar(
6500 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6502 if (!$is_qemu_server_backup) {
6503 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6505 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6507 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6508 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6510 if ($has_firewall_config) {
6511 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6512 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6514 my $pve_firewall_dir = '/etc/pve/firewall';
6515 mkdir $pve_firewall_dir; # make sure the dir exists
6516 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6519 my $fh = IO
::File-
>new($cfgfn, "r") ||
6520 die "unable to read qemu-server.conf - $!\n";
6522 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6524 # fixme: rate limit?
6526 # create empty/temp config
6527 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6529 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6532 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6534 foreach my $virtdev (sort keys %$virtdev_hash) {
6535 my $d = $virtdev_hash->{$virtdev};
6536 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6538 # this fails if storage is unavailable
6539 my $volid = $d->{volid
};
6540 my $path = PVE
::Storage
::path
($storecfg, $volid);
6542 # for live-restore we only want to preload the efidisk and TPM state
6543 next if $options->{live
} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6545 my $pbs_restore_cmd = [
6546 '/usr/bin/pbs-restore',
6547 '--repository', $repo,
6549 "$d->{devname}.img.fidx",
6554 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6555 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6557 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6558 push @$pbs_restore_cmd, '--skip-zero';
6561 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6562 print "restore proxmox backup image: $dbg_cmdstring\n";
6563 run_command
($pbs_restore_cmd);
6566 $fh->seek(0, 0) || die "seek failed - $!\n";
6568 my $cookie = { netcount
=> 0 };
6569 while (defined(my $line = <$fh>)) {
6570 $new_conf_raw .= restore_update_config_line
(
6582 if ($err || !$options->{live
}) {
6583 $restore_deactivate_volumes->($storecfg, $devinfo);
6589 $restore_destroy_volumes->($storecfg, $devinfo);
6593 if ($options->{live
}) {
6594 # keep lock during live-restore
6595 $new_conf_raw .= "\nlock: create";
6598 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6600 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6602 eval { rescan
($vmid, 1); };
6605 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
6607 if ($options->{live
}) {
6613 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
6615 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6616 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
6618 # these special drives are already restored before start
6619 delete $devinfo->{'drive-efidisk0'};
6620 delete $devinfo->{'drive-tpmstate0-backup'};
6621 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6623 PVE
::QemuConfig-
>remove_lock($vmid, "create");
6627 sub pbs_live_restore
{
6628 my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6630 print "starting VM for live-restore\n";
6631 print "repository: '$repo', snapshot: '$snap'\n";
6633 my $pbs_backing = {};
6634 for my $ds (keys %$restored_disks) {
6635 $ds =~ m/^drive-(.*)$/;
6637 $pbs_backing->{$confname} = {
6638 repository
=> $repo,
6640 archive
=> "$ds.img.fidx",
6642 $pbs_backing->{$confname}->{keyfile
} = $keyfile if -e
$keyfile;
6644 my $drive = parse_drive
($confname, $conf->{$confname});
6645 print "restoring '$ds' to '$drive->{file}'\n";
6648 my $drives_streamed = 0;
6650 # make sure HA doesn't interrupt our restore by stopping the VM
6651 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
6652 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
6655 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6656 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6657 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'pbs-backing' => $pbs_backing}, {});
6659 my $qmeventd_fd = register_qmeventd_handle
($vmid);
6661 # begin streaming, i.e. data copy from PBS to target disk for every vol,
6662 # this will effectively collapse the backing image chain consisting of
6663 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6664 # removes itself once all backing images vanish with 'auto-remove=on')
6666 for my $ds (sort keys %$restored_disks) {
6667 my $job_id = "restore-$ds";
6668 mon_cmd
($vmid, 'block-stream',
6669 'job-id' => $job_id,
6672 $jobs->{$job_id} = {};
6675 mon_cmd
($vmid, 'cont');
6676 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
6678 print "restore-drive jobs finished successfully, removing all tracking block devices"
6679 ." to disconnect from Proxmox Backup Server\n";
6681 for my $ds (sort keys %$restored_disks) {
6682 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6685 close($qmeventd_fd);
6691 warn "An error occured during live-restore: $err\n";
6692 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
6693 die "live-restore failed\n";
6697 sub restore_vma_archive
{
6698 my ($archive, $vmid, $user, $opts, $comp) = @_;
6700 my $readfrom = $archive;
6702 my $cfg = PVE
::Storage
::config
();
6704 my $bwlimit = $opts->{bwlimit
};
6706 my $dbg_cmdstring = '';
6707 my $add_pipe = sub {
6709 push @$commands, $cmd;
6710 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6711 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6716 if ($archive eq '-') {
6719 # If we use a backup from a PVE defined storage we also consider that
6720 # storage's rate limit:
6721 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6722 if (defined($volid)) {
6723 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6724 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6726 print STDERR
"applying read rate limit: $readlimit\n";
6727 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6728 $add_pipe->($cstream);
6734 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6735 my $cmd = $info->{decompressor
};
6736 push @$cmd, $readfrom;
6740 my $tmpdir = "/var/tmp/vzdumptmp$$";
6743 # disable interrupts (always do cleanups)
6747 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6749 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6750 POSIX
::mkfifo
($mapfifo, 0600);
6752 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6754 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6761 my $rpcenv = PVE
::RPCEnvironment
::get
();
6763 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6765 # Note: $oldconf is undef if VM does not exist
6766 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6767 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6768 my $new_conf_raw = '';
6772 my $print_devmap = sub {
6773 my $cfgfn = "$tmpdir/qemu-server.conf";
6775 # we can read the config - that is already extracted
6776 my $fh = IO
::File-
>new($cfgfn, "r") ||
6777 die "unable to read qemu-server.conf - $!\n";
6779 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6781 my $pve_firewall_dir = '/etc/pve/firewall';
6782 mkdir $pve_firewall_dir; # make sure the dir exists
6783 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6786 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6788 foreach my $info (values %{$virtdev_hash}) {
6789 my $storeid = $info->{storeid
};
6790 next if defined($storage_limits{$storeid});
6792 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
6793 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6794 $storage_limits{$storeid} = $limit * 1024;
6797 foreach my $devname (keys %$devinfo) {
6798 die "found no device mapping information for device '$devname'\n"
6799 if !$devinfo->{$devname}->{virtdev
};
6802 # create empty/temp config
6804 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6805 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6809 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6811 # print restore information to $fifofh
6812 foreach my $virtdev (sort keys %$virtdev_hash) {
6813 my $d = $virtdev_hash->{$virtdev};
6814 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6816 my $storeid = $d->{storeid
};
6817 my $volid = $d->{volid
};
6820 if (my $limit = $storage_limits{$storeid}) {
6821 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6824 my $write_zeros = 1;
6825 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6829 my $path = PVE
::Storage
::path
($cfg, $volid);
6831 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6833 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6836 $fh->seek(0, 0) || die "seek failed - $!\n";
6838 my $cookie = { netcount
=> 0 };
6839 while (defined(my $line = <$fh>)) {
6840 $new_conf_raw .= restore_update_config_line
(
6857 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6858 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6860 $oldtimeout = alarm($timeout);
6867 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6868 my ($dev_id, $size, $devname) = ($1, $2, $3);
6869 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6870 } elsif ($line =~ m/^CTIME: /) {
6871 # we correctly received the vma config, so we can disable
6872 # the timeout now for disk allocation (set to 10 minutes, so
6873 # that we always timeout if something goes wrong)
6876 print $fifofh "done\n";
6877 my $tmp = $oldtimeout || 0;
6878 $oldtimeout = undef;
6885 print "restore vma archive: $dbg_cmdstring\n";
6886 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6890 alarm($oldtimeout) if $oldtimeout;
6892 $restore_deactivate_volumes->($cfg, $devinfo);
6894 close($fifofh) if $fifofh;
6899 $restore_destroy_volumes->($cfg, $devinfo);
6903 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6905 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6907 eval { rescan
($vmid, 1); };
6910 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
6913 sub restore_tar_archive
{
6914 my ($archive, $vmid, $user, $opts) = @_;
6916 if ($archive ne '-') {
6917 my $firstfile = tar_archive_read_firstfile
($archive);
6918 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6919 if $firstfile ne 'qemu-server.conf';
6922 my $storecfg = PVE
::Storage
::config
();
6924 # avoid zombie disks when restoring over an existing VM -> cleanup first
6925 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6926 # skiplock=1 because qmrestore has set the 'create' lock itself already
6927 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6928 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6930 my $tocmd = "/usr/lib/qemu-server/qmextract";
6932 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6933 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6934 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6935 $tocmd .= ' --info' if $opts->{info
};
6937 # tar option "xf" does not autodetect compression when read from STDIN,
6938 # so we pipe to zcat
6939 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6940 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6942 my $tmpdir = "/var/tmp/vzdumptmp$$";
6945 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6946 local $ENV{VZDUMP_VMID
} = $vmid;
6947 local $ENV{VZDUMP_USER
} = $user;
6949 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6950 my $new_conf_raw = '';
6952 # disable interrupts (always do cleanups)
6956 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6964 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6966 if ($archive eq '-') {
6967 print "extracting archive from STDIN\n";
6968 run_command
($cmd, input
=> "<&STDIN");
6970 print "extracting archive '$archive'\n";
6974 return if $opts->{info
};
6978 my $statfile = "$tmpdir/qmrestore.stat";
6979 if (my $fd = IO
::File-
>new($statfile, "r")) {
6980 while (defined (my $line = <$fd>)) {
6981 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6982 $map->{$1} = $2 if $1;
6984 print STDERR
"unable to parse line in statfile - $line\n";
6990 my $confsrc = "$tmpdir/qemu-server.conf";
6992 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6994 my $cookie = { netcount
=> 0 };
6995 while (defined (my $line = <$srcfd>)) {
6996 $new_conf_raw .= restore_update_config_line
(
7007 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
7013 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7015 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7017 eval { rescan
($vmid, 1); };
7021 sub foreach_storage_used_by_vm
{
7022 my ($conf, $func) = @_;
7026 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7027 my ($ds, $drive) = @_;
7028 return if drive_is_cdrom
($drive);
7030 my $volid = $drive->{file
};
7032 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
7033 $sidhash->{$sid} = $sid if $sid;
7036 foreach my $sid (sort keys %$sidhash) {
7041 my $qemu_snap_storage = {
7044 sub do_snapshots_with_qemu
{
7045 my ($storecfg, $volid) = @_;
7047 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
7048 my $scfg = $storecfg->{ids
}->{$storage_name};
7049 die "could not find storage '$storage_name'\n" if !defined($scfg);
7051 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
7055 if ($volid =~ m/\.(qcow2|qed)$/){
7062 sub qga_check_running
{
7063 my ($vmid, $nowarn) = @_;
7065 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
7067 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7073 sub template_create
{
7074 my ($vmid, $conf, $disk) = @_;
7076 my $storecfg = PVE
::Storage
::config
();
7078 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7079 my ($ds, $drive) = @_;
7081 return if drive_is_cdrom
($drive);
7082 return if $disk && $ds ne $disk;
7084 my $volid = $drive->{file
};
7085 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
7087 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
7088 $drive->{file
} = $voliddst;
7089 $conf->{$ds} = print_drive
($drive);
7090 PVE
::QemuConfig-
>write_config($vmid, $conf);
7094 sub convert_iscsi_path
{
7097 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
7102 my $initiator_name = get_initiator_name
();
7104 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7105 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7108 die "cannot convert iscsi path '$path', unkown format\n";
7111 sub qemu_img_convert
{
7112 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7114 my $storecfg = PVE
::Storage
::config
();
7115 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
7116 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
7118 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7122 my $src_is_iscsi = 0;
7126 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
7127 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7128 $src_format = qemu_img_format
($src_scfg, $src_volname);
7129 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
7130 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
7131 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
7132 } elsif (-f
$src_volid) {
7133 $src_path = $src_volid;
7134 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7139 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7141 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7142 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7143 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7144 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7147 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7148 push @$cmd, '-l', "snapshot.name=$snapname"
7149 if $snapname && $src_format && $src_format eq "qcow2";
7150 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7151 push @$cmd, '-T', $cachemode if defined($cachemode);
7153 if ($src_is_iscsi) {
7154 push @$cmd, '--image-opts';
7155 $src_path = convert_iscsi_path
($src_path);
7156 } elsif ($src_format) {
7157 push @$cmd, '-f', $src_format;
7160 if ($dst_is_iscsi) {
7161 push @$cmd, '--target-image-opts';
7162 $dst_path = convert_iscsi_path
($dst_path);
7164 push @$cmd, '-O', $dst_format;
7167 push @$cmd, $src_path;
7169 if (!$dst_is_iscsi && $is_zero_initialized) {
7170 push @$cmd, "zeroinit:$dst_path";
7172 push @$cmd, $dst_path;
7177 if($line =~ m/\((\S+)\/100\
%\)/){
7179 my $transferred = int($size * $percent / 100);
7180 my $total_h = render_bytes
($size, 1);
7181 my $transferred_h = render_bytes
($transferred, 1);
7183 print "transferred $transferred_h of $total_h ($percent%)\n";
7188 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7190 die "copy failed: $err" if $err;
7193 sub qemu_img_format
{
7194 my ($scfg, $volname) = @_;
7196 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7203 sub qemu_drive_mirror
{
7204 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7206 $jobs = {} if !$jobs;
7210 $jobs->{"drive-$drive"} = {};
7212 if ($dst_volid =~ /^nbd:/) {
7213 $qemu_target = $dst_volid;
7216 my $storecfg = PVE
::Storage
::config
();
7217 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7219 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7221 $format = qemu_img_format
($dst_scfg, $dst_volname);
7223 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7225 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7228 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7229 $opts->{format
} = $format if $format;
7231 if (defined($src_bitmap)) {
7232 $opts->{sync
} = 'incremental';
7233 $opts->{bitmap
} = $src_bitmap;
7234 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7237 if (defined($bwlimit)) {
7238 $opts->{speed
} = $bwlimit * 1024;
7239 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7241 print "drive mirror is starting for drive-$drive\n";
7244 # if a job already runs for this device we get an error, catch it for cleanup
7245 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7247 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7249 die "mirroring error: $err\n";
7252 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7255 # $completion can be either
7256 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7257 # 'cancel': wait until all jobs are ready, block-job-cancel them
7258 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7259 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7260 sub qemu_drive_mirror_monitor
{
7261 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7263 $completion //= 'complete';
7267 my $err_complete = 0;
7269 my $starttime = time ();
7271 die "block job ('$op') timed out\n" if $err_complete > 300;
7273 my $stats = mon_cmd
($vmid, "query-block-jobs");
7276 my $running_jobs = {};
7277 for my $stat (@$stats) {
7278 next if $stat->{type
} ne $op;
7279 $running_jobs->{$stat->{device
}} = $stat;
7282 my $readycounter = 0;
7284 for my $job_id (sort keys %$jobs) {
7285 my $job = $running_jobs->{$job_id};
7287 my $vanished = !defined($job);
7288 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7289 if($complete || ($vanished && $completion eq 'auto')) {
7290 print "$job_id: $op-job finished\n";
7291 delete $jobs->{$job_id};
7295 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7297 my $busy = $job->{busy
};
7298 my $ready = $job->{ready
};
7299 if (my $total = $job->{len
}) {
7300 my $transferred = $job->{offset
} || 0;
7301 my $remaining = $total - $transferred;
7302 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7304 my $duration = $ctime - $starttime;
7305 my $total_h = render_bytes
($total, 1);
7306 my $transferred_h = render_bytes
($transferred, 1);
7308 my $status = sprintf(
7309 "transferred $transferred_h of $total_h ($percent%%) in %s",
7310 render_duration
($duration),
7315 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7317 $status .= ", ready";
7320 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
7321 $jobs->{$job_id}->{ready
} = $ready;
7324 $readycounter++ if $job->{ready
};
7327 last if scalar(keys %$jobs) == 0;
7329 if ($readycounter == scalar(keys %$jobs)) {
7330 print "all '$op' jobs are ready\n";
7332 # do the complete later (or has already been done)
7333 last if $completion eq 'skip' || $completion eq 'auto';
7335 if ($vmiddst && $vmiddst != $vmid) {
7336 my $agent_running = $qga && qga_check_running
($vmid);
7337 if ($agent_running) {
7338 print "freeze filesystem\n";
7339 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
7341 print "suspend vm\n";
7342 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
7345 # if we clone a disk for a new target vm, we don't switch the disk
7346 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
7348 if ($agent_running) {
7349 print "unfreeze filesystem\n";
7350 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
7352 print "resume vm\n";
7353 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
7359 for my $job_id (sort keys %$jobs) {
7360 # try to switch the disk if source and destination are on the same guest
7361 print "$job_id: Completing block job_id...\n";
7364 if ($completion eq 'complete') {
7365 $op = 'block-job-complete';
7366 } elsif ($completion eq 'cancel') {
7367 $op = 'block-job-cancel';
7369 die "invalid completion value: $completion\n";
7371 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
7372 if ($@ =~ m/cannot be completed/) {
7373 print "$job_id: block job cannot be completed, trying again.\n";
7376 print "$job_id: Completed successfully.\n";
7377 $jobs->{$job_id}->{complete
} = 1;
7388 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7389 die "block job ($op) error: $err";
7393 sub qemu_blockjobs_cancel
{
7394 my ($vmid, $jobs) = @_;
7396 foreach my $job (keys %$jobs) {
7397 print "$job: Cancelling block job\n";
7398 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
7399 $jobs->{$job}->{cancel
} = 1;
7403 my $stats = mon_cmd
($vmid, "query-block-jobs");
7405 my $running_jobs = {};
7406 foreach my $stat (@$stats) {
7407 $running_jobs->{$stat->{device
}} = $stat;
7410 foreach my $job (keys %$jobs) {
7412 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
7413 print "$job: Done.\n";
7414 delete $jobs->{$job};
7418 last if scalar(keys %$jobs) == 0;
7425 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7426 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7431 print "create linked clone of drive $drivename ($drive->{file})\n";
7432 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
7433 push @$newvollist, $newvolid;
7436 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
7437 $storeid = $storage if $storage;
7439 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
7441 print "create full clone of drive $drivename ($drive->{file})\n";
7444 if (drive_is_cloudinit
($drive)) {
7445 $name = "vm-$newvmid-cloudinit";
7446 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7447 if ($scfg->{path
}) {
7448 $name .= ".$dst_format";
7451 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
7452 } elsif ($drivename eq 'efidisk0') {
7453 $size = get_efivars_size
($conf);
7454 } elsif ($drivename eq 'tpmstate0') {
7455 $size = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
7457 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
7459 $newvolid = PVE
::Storage
::vdisk_alloc
(
7460 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7462 push @$newvollist, $newvolid;
7464 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
7466 if (drive_is_cloudinit
($drive)) {
7467 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7468 # if this is the case, we have to complete any block-jobs still there from
7469 # previous drive-mirrors
7470 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7471 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
7476 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
7477 if (!$running || $snapname) {
7478 # TODO: handle bwlimits
7479 if ($drivename eq 'efidisk0') {
7480 # the relevant data on the efidisk may be smaller than the source
7481 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7482 # that is given by the OVMF_VARS.fd
7483 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
7484 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
7486 # better for Ceph if block size is not too small, see bug #3324
7489 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7490 "if=$src_path", "of=$dst_path"]);
7492 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
7496 die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7498 my $kvmver = get_running_qemu_version
($vmid);
7499 if (!min_version
($kvmver, 2, 7)) {
7500 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7501 if $drive->{iothread
};
7504 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7505 $completion, $qga, $bwlimit);
7510 my ($size) = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
7513 $disk->{format
} = undef;
7514 $disk->{file
} = $newvolid;
7515 $disk->{size
} = $size if defined($size);
7520 sub get_running_qemu_version
{
7522 my $res = mon_cmd
($vmid, "query-version");
7523 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7526 sub qemu_use_old_bios_files
{
7527 my ($machine_type) = @_;
7529 return if !$machine_type;
7531 my $use_old_bios_files = undef;
7533 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7535 $use_old_bios_files = 1;
7537 my $version = extract_version
($machine_type, kvm_user_version
());
7538 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7539 # load new efi bios files on migration. So this hack is required to allow
7540 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7541 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7542 $use_old_bios_files = !min_version
($version, 2, 4);
7545 return ($use_old_bios_files, $machine_type);
7548 sub get_efivars_size
{
7550 my $arch = get_vm_arch
($conf);
7551 my $efidisk = $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
7552 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk);
7553 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
7554 return -s
$ovmf_vars;
7557 sub update_efidisk_size
{
7560 return if !defined($conf->{efidisk0
});
7562 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
7563 $disk->{size
} = get_efivars_size
($conf);
7564 $conf->{efidisk0
} = print_drive
($disk);
7569 sub update_tpmstate_size
{
7572 my $disk = PVE
::QemuServer
::parse_drive
('tpmstate0', $conf->{tpmstate0
});
7573 $disk->{size
} = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
7574 $conf->{tpmstate0
} = print_drive
($disk);
7577 sub create_efidisk
($$$$$$) {
7578 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk) = @_;
7580 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk);
7581 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7583 my $vars_size_b = -s
$ovmf_vars;
7584 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7585 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7586 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7588 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7589 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7591 return ($volid, $size/1024);
7594 sub vm_iothreads_list
{
7597 my $res = mon_cmd
($vmid, 'query-iothreads');
7600 foreach my $iothread (@$res) {
7601 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7608 my ($conf, $drive) = @_;
7612 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7614 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7620 my $controller = int($drive->{index} / $maxdev);
7621 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
7625 return ($maxdev, $controller, $controller_prefix);
7628 sub windows_version
{
7631 return 0 if !$ostype;
7635 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7637 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7639 } elsif ($ostype =~ m/^win(\d+)$/) {
7646 sub resolve_dst_disk_format
{
7647 my ($storecfg, $storeid, $src_volname, $format) = @_;
7648 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7651 # if no target format is specified, use the source disk format as hint
7653 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7654 $format = qemu_img_format
($scfg, $src_volname);
7660 # test if requested format is supported - else use default
7661 my $supported = grep { $_ eq $format } @$validFormats;
7662 $format = $defFormat if !$supported;
7666 # NOTE: if this logic changes, please update docs & possibly gui logic
7667 sub find_vmstate_storage
{
7668 my ($conf, $storecfg) = @_;
7670 # first, return storage from conf if set
7671 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7673 my ($target, $shared, $local);
7675 foreach_storage_used_by_vm
($conf, sub {
7677 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7678 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7679 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7682 # second, use shared storage where VM has at least one disk
7683 # third, use local storage where VM has at least one disk
7684 # fall back to local storage
7685 $target = $shared // $local // 'local';
7691 my ($uuid, $uuid_str);
7692 UUID
::generate
($uuid);
7693 UUID
::unparse
($uuid, $uuid_str);
7697 sub generate_smbios1_uuid
{
7698 return "uuid=".generate_uuid
();
7704 mon_cmd
($vmid, 'nbd-server-stop');
7707 sub create_reboot_request
{
7709 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7710 or die "failed to create reboot trigger file: $!\n";
7714 sub clear_reboot_request
{
7716 my $path = "/run/qemu-server/$vmid.reboot";
7719 $res = unlink($path);
7720 die "could not remove reboot request for $vmid: $!"
7721 if !$res && $! != POSIX
::ENOENT
;
7726 sub bootorder_from_legacy
{
7727 my ($conf, $bootcfg) = @_;
7729 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
7730 my $bootindex_hash = {};
7732 foreach my $o (split(//, $boot)) {
7733 $bootindex_hash->{$o} = $i*100;
7739 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7740 my ($ds, $drive) = @_;
7742 if (drive_is_cdrom
($drive, 1)) {
7743 if ($bootindex_hash->{d
}) {
7744 $bootorder->{$ds} = $bootindex_hash->{d
};
7745 $bootindex_hash->{d
} += 1;
7747 } elsif ($bootindex_hash->{c
}) {
7748 $bootorder->{$ds} = $bootindex_hash->{c
}
7749 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
7750 $bootindex_hash->{c
} += 1;
7754 if ($bootindex_hash->{n
}) {
7755 for (my $i = 0; $i < $MAX_NETS; $i++) {
7756 my $netname = "net$i";
7757 next if !$conf->{$netname};
7758 $bootorder->{$netname} = $bootindex_hash->{n
};
7759 $bootindex_hash->{n
} += 1;
7766 # Generate default device list for 'boot: order=' property. Matches legacy
7767 # default boot order, but with explicit device names. This is important, since
7768 # the fallback for when neither 'order' nor the old format is specified relies
7769 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7770 sub get_default_bootdevices
{
7776 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
7777 push @ret, $first if $first;
7780 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
7781 push @ret, $first if $first;
7784 for (my $i = 0; $i < $MAX_NETS; $i++) {
7785 my $netname = "net$i";
7786 next if !$conf->{$netname};
7787 push @ret, $netname;
7794 sub device_bootorder
{
7797 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
7799 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
7802 if (!defined($boot) || $boot->{legacy
}) {
7803 $bootorder = bootorder_from_legacy
($conf, $boot);
7804 } elsif ($boot->{order
}) {
7805 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7806 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
7807 $bootorder->{$dev} = $i++;
7814 sub register_qmeventd_handle
{
7818 my $peer = "/var/run/qmeventd.sock";
7823 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
7825 if ($! != EINTR
&& $! != EAGAIN
) {
7826 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7829 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7830 . "after $count retries\n";
7835 # send handshake to mark VM as backing up
7836 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
7838 # return handle to be closed later when inhibit is no longer required
7842 # bash completion helper
7844 sub complete_backup_archives
{
7845 my ($cmdname, $pname, $cvalue) = @_;
7847 my $cfg = PVE
::Storage
::config
();
7851 if ($cvalue =~ m/^([^:]+):/) {
7855 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7858 foreach my $id (keys %$data) {
7859 foreach my $item (@{$data->{$id}}) {
7860 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7861 push @$res, $item->{volid
} if defined($item->{volid
});
7868 my $complete_vmid_full = sub {
7871 my $idlist = vmstatus
();
7875 foreach my $id (keys %$idlist) {
7876 my $d = $idlist->{$id};
7877 if (defined($running)) {
7878 next if $d->{template
};
7879 next if $running && $d->{status
} ne 'running';
7880 next if !$running && $d->{status
} eq 'running';
7889 return &$complete_vmid_full();
7892 sub complete_vmid_stopped
{
7893 return &$complete_vmid_full(0);
7896 sub complete_vmid_running
{
7897 return &$complete_vmid_full(1);
7900 sub complete_storage
{
7902 my $cfg = PVE
::Storage
::config
();
7903 my $ids = $cfg->{ids
};
7906 foreach my $sid (keys %$ids) {
7907 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7908 next if !$ids->{$sid}->{content
}->{images
};
7915 sub complete_migration_storage
{
7916 my ($cmd, $param, $current_value, $all_args) = @_;
7918 my $targetnode = @$all_args[1];
7920 my $cfg = PVE
::Storage
::config
();
7921 my $ids = $cfg->{ids
};
7924 foreach my $sid (keys %$ids) {
7925 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7926 next if !$ids->{$sid}->{content
}->{images
};
7935 my $qmpstatus = eval {
7936 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
7937 mon_cmd
($vmid, "query-status");
7940 return $qmpstatus && $qmpstatus->{status
} eq "paused";
7943 sub check_volume_storage_type
{
7944 my ($storecfg, $vol) = @_;
7946 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
7947 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7948 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
7950 die "storage '$storeid' does not support content-type '$vtype'\n"
7951 if !$scfg->{content
}->{$vtype};