1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
30 use PVE
::DataCenterConfig
;
31 use PVE
::Exception
qw(raise raise_param_exc);
32 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
34 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
36 use PVE
::RPCEnvironment
;
40 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
44 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
45 use PVE
::QemuServer
::Cloudinit
;
46 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
47 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
48 use PVE
::QemuServer
::Machine
;
49 use PVE
::QemuServer
::Memory
;
50 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
51 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
52 use PVE
::QemuServer
::USB
qw(parse_usb_device);
56 require PVE
::Network
::SDN
::Zones
;
60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
63 "$EDK2_FW_BASE/OVMF_CODE.fd",
64 "$EDK2_FW_BASE/OVMF_VARS.fd"
67 "$EDK2_FW_BASE/AAVMF_CODE.fd",
68 "$EDK2_FW_BASE/AAVMF_VARS.fd"
72 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
74 # Note about locking: we use flock on the config file protect
75 # against concurent actions.
76 # Aditionaly, we have a 'lock' setting in the config file. This
77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
78 # allowed when such lock is set. But you can ignore this kind of
79 # lock with the --skiplock flag.
81 cfs_register_file
('/qemu-server/',
85 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
86 description
=> "Some command save/restore state from this location.",
92 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
93 description
=> "Specifies the Qemu machine type.",
95 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
102 my ($map, $source) = @_;
104 return $source if !defined($map);
106 return $map->{entries
}->{$source}
107 if $map->{entries
} && defined($map->{entries
}->{$source});
109 return $map->{default} if $map->{default};
111 # identity (fallback)
115 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
116 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
118 format
=> 'storagepair-list',
122 #no warnings 'redefine';
125 my ($controller, $vmid, $option, $value) = @_;
127 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
128 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
134 $nodename_cache //= PVE
::INotify
::nodename
();
135 return $nodename_cache;
142 enum
=> [qw(i6300esb ib700)],
143 description
=> "Watchdog type to emulate.",
144 default => 'i6300esb',
149 enum
=> [qw(reset shutdown poweroff pause debug none)],
150 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
154 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
158 description
=> "Enable/disable Qemu GuestAgent.",
163 fstrim_cloned_disks
=> {
164 description
=> "Run fstrim after cloning/moving a disk.",
170 description
=> "Select the agent type",
174 enum
=> [qw(virtio isa)],
180 description
=> "Select the VGA type.",
185 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
188 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
200 description
=> "The size of the file in MB.",
204 pattern
=> '[a-zA-Z0-9\-]+',
206 format_description
=> 'string',
207 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
214 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
215 description
=> "Configure an audio device."
222 description
=> "Driver backend for the audio device."
226 my $spice_enhancements_fmt = {
231 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
235 enum
=> ['off', 'all', 'filter'],
238 description
=> "Enable video streaming. Uses compression for detected video streams."
245 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
247 description
=> "The file on the host to gather entropy from. In most"
248 . " cases /dev/urandom should be preferred over /dev/random"
249 . " to avoid entropy-starvation issues on the host. Using"
250 . " urandom does *not* decrease security in any meaningful"
251 . " way, as it's still seeded from real entropy, and the"
252 . " bytes provided will most likely be mixed with real"
253 . " entropy on the guest as well. /dev/hwrng can be used"
254 . " to pass through a hardware RNG from the host.",
258 description
=> "Maximum bytes of entropy injected into the guest every"
259 . " 'period' milliseconds. Prefer a lower value when using"
260 . " /dev/random as source. Use 0 to disable limiting"
261 . " (potentially dangerous!).",
264 # default is 1 KiB/s, provides enough entropy to the guest to avoid
265 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
266 # of overwhelming the host, provided we're reading from /dev/urandom
271 description
=> "Every 'period' milliseconds the entropy-injection quota"
272 . " is reset, allowing the guest to retrieve another"
273 . " 'max_bytes' of entropy.",
283 description
=> "Specifies whether a VM will be started during system bootup.",
289 description
=> "Automatic restart after crash (currently ignored).",
294 type
=> 'string', format
=> 'pve-hotplug-features',
295 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
296 default => 'network,disk,usb',
301 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
307 description
=> "Lock/unlock the VM.",
308 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
313 description
=> "Limit of CPU usage.",
314 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
322 description
=> "CPU weight for a VM.",
323 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
331 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
338 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
344 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
352 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
353 "It should not be necessary to set it.",
354 enum
=> PVE
::Tools
::kvmkeymaplist
(),
359 type
=> 'string', format
=> 'dns-name',
360 description
=> "Set a name for the VM. Only used on the configuration web interface.",
365 description
=> "SCSI controller model",
366 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
372 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
377 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
378 description
=> "Specify guest operating system.",
379 verbose_description
=> <<EODESC,
380 Specify guest operating system. This is used to enable special
381 optimization/features for specific operating systems:
384 other;; unspecified OS
385 wxp;; Microsoft Windows XP
386 w2k;; Microsoft Windows 2000
387 w2k3;; Microsoft Windows 2003
388 w2k8;; Microsoft Windows 2008
389 wvista;; Microsoft Windows Vista
390 win7;; Microsoft Windows 7
391 win8;; Microsoft Windows 8/2012/2012r2
392 win10;; Microsoft Windows 10/2016
393 l24;; Linux 2.4 Kernel
394 l26;; Linux 2.6 - 5.X Kernel
395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
400 type
=> 'string', format
=> 'pve-qm-boot',
401 description
=> "Specify guest boot order. Use with 'order=', usage with"
402 . " no key or 'legacy=' is deprecated.",
406 type
=> 'string', format
=> 'pve-qm-bootdisk',
407 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
408 pattern
=> '(ide|sata|scsi|virtio)\d+',
413 description
=> "The number of CPUs. Please use option -sockets instead.",
420 description
=> "The number of CPU sockets.",
427 description
=> "The number of cores per socket.",
434 description
=> "Enable/disable NUMA.",
440 description
=> "Enable/disable hugepages memory.",
441 enum
=> [qw(any 2 1024)],
447 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
448 ." after VM shutdown and can be used for subsequent starts.",
453 description
=> "Number of hotplugged vcpus.",
460 description
=> "Enable/disable ACPI.",
465 description
=> "Enable/disable Qemu GuestAgent and its properties.",
467 format
=> $agent_fmt,
472 description
=> "Enable/disable KVM hardware virtualization.",
478 description
=> "Enable/disable time drift fix.",
484 description
=> "Set the real time clock to local time. This is enabled by default if ostype"
485 ." indicates a Microsoft OS.",
490 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
494 type
=> 'string', format
=> $vga_fmt,
495 description
=> "Configure the VGA hardware.",
496 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
497 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
498 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
499 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
500 ." display server. For win* OS you can select how many independent displays you want,"
501 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
502 ." using a serial device as terminal.",
506 type
=> 'string', format
=> 'pve-qm-watchdog',
507 description
=> "Create a virtual hardware watchdog device.",
508 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
509 ." action), the watchdog must be periodically polled by an agent inside the guest or"
510 ." else the watchdog will reset the guest (or execute the respective action specified)",
515 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
516 description
=> "Set the initial date of the real time clock. Valid format for date are:"
517 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
518 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
521 startup
=> get_standard_option
('pve-startup-order'),
525 description
=> "Enable/disable Template.",
531 description
=> "Arbitrary arguments passed to kvm.",
532 verbose_description
=> <<EODESCR,
533 Arbitrary arguments passed to kvm, for example:
535 args: -no-reboot -no-hpet
537 NOTE: this option is for experts only.
544 description
=> "Enable/disable the USB tablet device.",
545 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
546 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
547 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
548 ." may consider disabling this to save some context switches. This is turned off by"
549 ." default if you use spice (`qm set <vmid> --vga qxl`).",
554 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
558 migrate_downtime
=> {
561 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
567 type
=> 'string', format
=> 'pve-qm-ide',
568 typetext
=> '<volume>',
569 description
=> "This is an alias for option -ide2",
573 description
=> "Emulated CPU type.",
575 format
=> 'pve-vm-cpu-conf',
577 parent
=> get_standard_option
('pve-snapshot-name', {
579 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
583 description
=> "Timestamp for snapshots.",
589 type
=> 'string', format
=> 'pve-volume-id',
590 description
=> "Reference to a volume which stores the VM state. This is used internally"
593 vmstatestorage
=> get_standard_option
('pve-storage-id', {
594 description
=> "Default storage for VM state volumes/files.",
597 runningmachine
=> get_standard_option
('pve-qemu-machine', {
598 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
602 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
603 ." internally for snapshots.",
606 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
607 format_description
=> 'QEMU -cpu parameter'
609 machine
=> get_standard_option
('pve-qemu-machine'),
611 description
=> "Virtual processor architecture. Defaults to the host.",
614 enum
=> [qw(x86_64 aarch64)],
617 description
=> "Specify SMBIOS type 1 fields.",
618 type
=> 'string', format
=> 'pve-qm-smbios1',
625 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
626 ." remove disk operations.",
632 enum
=> [ qw(seabios ovmf) ],
633 description
=> "Select BIOS implementation.",
634 default => 'seabios',
638 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
639 format_description
=> 'UUID',
640 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
641 ." to disable explicitly.",
642 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
643 ." value identifier to the guest OS. This allows to notify the guest operating system"
644 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
645 ." execution or creation from a template). The guest operating system notices the"
646 ." change, and is then able to react as appropriate by marking its copies of"
647 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
648 ."Note that auto-creation only works when done through API/CLI create or update methods"
649 .", but not when manually editing the config file.",
650 default => "1 (autogenerated)",
655 format
=> 'pve-volume-id',
657 description
=> "Script that will be executed during various steps in the vms lifetime.",
661 format
=> $ivshmem_fmt,
662 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
668 format
=> $audio_fmt,
669 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
672 spice_enhancements
=> {
674 format
=> $spice_enhancements_fmt,
675 description
=> "Configure additional enhancements for SPICE.",
679 type
=> 'string', format
=> 'pve-tag-list',
680 description
=> 'Tags of the VM. This is only meta information.',
686 description
=> "Configure a VirtIO-based Random Number Generator.",
695 description
=> 'Specify a custom file containing all meta data passed to the VM via"
696 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
697 format
=> 'pve-volume-id',
698 format_description
=> 'volume',
703 description
=> 'Specify a custom file containing all network data passed to the VM via'
705 format
=> 'pve-volume-id',
706 format_description
=> 'volume',
711 description
=> 'Specify a custom file containing all user data passed to the VM via'
713 format
=> 'pve-volume-id',
714 format_description
=> 'volume',
717 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
719 my $confdesc_cloudinit = {
723 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
724 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
725 .' and `configdrive2` for windows.',
726 enum
=> ['configdrive2', 'nocloud'],
731 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
732 ." image's configured default user.",
737 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
738 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
739 .' support hashed passwords.',
744 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
746 format
=> 'pve-qm-cicustom',
751 description
=> "cloud-init: Sets DNS search domains for a container. Create will'
752 .' automatically use the setting from the host if neither searchdomain nor nameserver'
757 type
=> 'string', format
=> 'address-list',
758 description
=> "cloud-init: Sets DNS server IP address for a container. Create will'
759 .' automatically use the setting from the host if neither searchdomain nor nameserver'
765 format
=> 'urlencoded',
766 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
770 # what about other qemu settings ?
772 #machine => 'string',
785 ##soundhw => 'string',
787 while (my ($k, $v) = each %$confdesc) {
788 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
791 my $MAX_USB_DEVICES = 5;
793 my $MAX_SERIAL_PORTS = 4;
794 my $MAX_PARALLEL_PORTS = 3;
800 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
801 description
=> "CPUs accessing this NUMA node.",
802 format_description
=> "id[-id];...",
806 description
=> "Amount of memory this NUMA node provides.",
811 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
812 description
=> "Host NUMA nodes to use.",
813 format_description
=> "id[-id];...",
818 enum
=> [qw(preferred bind interleave)],
819 description
=> "NUMA allocation policy.",
823 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
826 type
=> 'string', format
=> $numa_fmt,
827 description
=> "NUMA topology.",
829 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
831 for (my $i = 0; $i < $MAX_NUMA; $i++) {
832 $confdesc->{"numa$i"} = $numadesc;
835 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
836 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
837 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
838 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
840 my $net_fmt_bridge_descr = <<__EOD__;
841 Bridge to attach the network device to. The Proxmox VE standard bridge
844 If you do not specify a bridge, we create a kvm user (NATed) network
845 device, which provides DHCP and DNS services. The following addresses
852 The DHCP server assign addresses to the guest starting from 10.0.2.15.
856 macaddr
=> get_standard_option
('mac-addr', {
857 description
=> "MAC address. That address must be unique withing your network. This is"
858 ." automatically generated if not specified.",
862 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
863 ." very low CPU overhead. If your guest does not support this driver, it is usually"
864 ." best to use 'e1000'.",
865 enum
=> $nic_model_list,
868 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
871 description
=> $net_fmt_bridge_descr,
872 format_description
=> 'bridge',
873 pattern
=> '[-_.\w\d]+',
878 minimum
=> 0, maximum
=> 16,
879 description
=> 'Number of packet queues to be used on the device.',
885 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
890 minimum
=> 1, maximum
=> 4094,
891 description
=> 'VLAN tag to apply to packets on this interface.',
896 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
897 description
=> 'VLAN trunks to pass through this interface.',
898 format_description
=> 'vlanid[;vlanid...]',
903 description
=> 'Whether this interface should be protected by the firewall.',
908 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
913 minimum
=> 1, maximum
=> 65520,
914 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
921 type
=> 'string', format
=> $net_fmt,
922 description
=> "Specify network devices.",
925 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
930 format
=> 'pve-ipv4-config',
931 format_description
=> 'IPv4Format/CIDR',
932 description
=> 'IPv4 address in CIDR format.',
939 format_description
=> 'GatewayIPv4',
940 description
=> 'Default gateway for IPv4 traffic.',
946 format
=> 'pve-ipv6-config',
947 format_description
=> 'IPv6Format/CIDR',
948 description
=> 'IPv6 address in CIDR format.',
955 format_description
=> 'GatewayIPv6',
956 description
=> 'Default gateway for IPv6 traffic.',
961 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
964 type
=> 'string', format
=> 'pve-qm-ipconfig',
965 description
=> <<'EODESCR',
966 cloud-init: Specify IP addresses and gateways for the corresponding interface.
968 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
970 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
971 gateway should be provided.
972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
978 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
980 for (my $i = 0; $i < $MAX_NETS; $i++) {
981 $confdesc->{"net$i"} = $netdesc;
982 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
985 foreach my $key (keys %$confdesc_cloudinit) {
986 $confdesc->{$key} = $confdesc_cloudinit->{$key};
989 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
990 sub verify_volume_id_or_qm_path
{
991 my ($volid, $noerr) = @_;
993 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
997 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
998 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1009 type
=> 'string', format
=> 'pve-qm-usb-device',
1010 format_description
=> 'HOSTUSBDEVICE|spice',
1011 description
=> <<EODESCR,
1012 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1014 'bus-port(.port)*' (decimal numbers) or
1015 'vendor_id:product_id' (hexadeciaml numbers) or
1018 You can use the 'lsusb -t' command to list existing usb devices.
1020 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1021 machines - use with special care.
1023 The value 'spice' can be used to add a usb redirection devices for spice.
1029 description
=> "Specifies whether if given host option is a USB3 device or port.",
1036 type
=> 'string', format
=> $usb_fmt,
1037 description
=> "Configure an USB device (n is 0 to 4).",
1039 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1044 pattern
=> '(/dev/.+|socket)',
1045 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1046 verbose_description
=> <<EODESCR,
1047 Create a serial device inside the VM (n is 0 to 3), and pass through a
1048 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1049 host side (use 'qm terminal' to open a terminal connection).
1051 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1052 use with special care.
1054 CAUTION: Experimental! User reported problems with this option.
1061 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1062 description
=> "Map host parallel devices (n is 0 to 2).",
1063 verbose_description
=> <<EODESCR,
1064 Map host parallel devices (n is 0 to 2).
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067 machines - use with special care.
1069 CAUTION: Experimental! User reported problems with this option.
1073 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1074 $confdesc->{"parallel$i"} = $paralleldesc;
1077 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1078 $confdesc->{"serial$i"} = $serialdesc;
1081 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1082 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1085 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1086 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1089 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1090 $confdesc->{"usb$i"} = $usbdesc;
1098 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1099 . " Deprecated, use 'order=' instead.",
1100 pattern
=> '[acdn]{1,4}',
1101 format_description
=> "[acdn]{1,4}",
1103 # note: this is also the fallback if boot: is not given at all
1109 format
=> 'pve-qm-bootdev-list',
1110 format_description
=> "device[;device...]",
1111 description
=> <<EODESC,
1112 The guest will attempt to boot from devices in the order they appear here.
1114 Disks, optical drives and passed-through storage USB devices will be directly
1115 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1116 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1118 Note that only devices in this list will be marked as bootable and thus loaded
1119 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1120 (e.g. software-raid), you need to specify all of them here.
1122 Overrides the deprecated 'legacy=[acdn]*' value when given.
1126 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1128 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1129 sub verify_bootdev
{
1130 my ($dev, $noerr) = @_;
1132 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && $dev !~ m/^efidisk/;
1136 return 0 if $dev !~ m/^$base\d+$/;
1137 return 0 if !$confdesc->{$dev};
1141 return $dev if $check->("net");
1142 return $dev if $check->("usb");
1143 return $dev if $check->("hostpci");
1146 die "invalid boot device '$dev'\n";
1149 sub print_bootorder
{
1151 my $data = { order
=> join(';', @$devs) };
1152 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1155 my $kvm_api_version = 0;
1158 return $kvm_api_version if $kvm_api_version;
1160 open my $fh, '<', '/dev/kvm'
1163 # 0xae00 => KVM_GET_API_VERSION
1164 $kvm_api_version = ioctl($fh, 0xae00, 0);
1166 return $kvm_api_version;
1169 my $kvm_user_version = {};
1172 sub kvm_user_version
{
1175 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1176 my $st = stat($binary);
1178 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1179 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1180 $cachedmtime == $st->mtime;
1182 $kvm_user_version->{$binary} = 'unknown';
1183 $kvm_mtime->{$binary} = $st->mtime;
1187 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188 $kvm_user_version->{$binary} = $2;
1192 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1195 return $kvm_user_version->{$binary};
1198 my sub extract_version
{
1199 my ($machine_type, $version) = @_;
1200 $version = kvm_user_version
() if !defined($version);
1201 PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1204 sub kernel_has_vhost_net
{
1205 return -c
'/dev/vhost-net';
1210 return defined($confdesc->{$key});
1214 sub get_cdrom_path
{
1216 return $cdrom_path if $cdrom_path;
1218 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1219 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1220 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1224 my ($storecfg, $vmid, $cdrom) = @_;
1226 if ($cdrom eq 'cdrom') {
1227 return get_cdrom_path
();
1228 } elsif ($cdrom eq 'none') {
1230 } elsif ($cdrom =~ m
|^/|) {
1233 return PVE
::Storage
::path
($storecfg, $cdrom);
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id
{
1239 my ($vmid, $file, $media) = @_;
1241 if (!($file eq 'none' || $file eq 'cdrom' ||
1242 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1244 return if $file =~ m
|/|;
1246 if ($media && $media eq 'cdrom') {
1247 $file = "local:iso/$file";
1249 $file = "local:$vmid/$file";
1256 sub verify_media_type
{
1257 my ($opt, $vtype, $media) = @_;
1262 if ($media eq 'disk') {
1264 } elsif ($media eq 'cdrom') {
1267 die "internal error";
1270 return if ($vtype eq $etype);
1272 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1275 sub cleanup_drive_path
{
1276 my ($opt, $storecfg, $drive) = @_;
1278 # try to convert filesystem paths to volume IDs
1280 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1281 ($drive->{file
} !~ m
|^/dev/.+|) &&
1282 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1283 ($drive->{file
} !~ m/^\d+$/)) {
1284 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1285 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1287 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1288 verify_media_type
($opt, $vtype, $drive->{media
});
1289 $drive->{file
} = $volid;
1292 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1295 sub parse_hotplug_features
{
1300 return $res if $data eq '0';
1302 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1304 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1305 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1308 die "invalid hotplug feature '$feature'\n";
1314 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1315 sub pve_verify_hotplug_features
{
1316 my ($value, $noerr) = @_;
1318 return $value if parse_hotplug_features
($value);
1322 die "unable to parse hotplug option\n";
1326 my($fh, $noerr) = @_;
1329 my $SG_GET_VERSION_NUM = 0x2282;
1331 my $versionbuf = "\x00" x
8;
1332 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1334 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1337 my $version = unpack("I", $versionbuf);
1338 if ($version < 30000) {
1339 die "scsi generic interface too old\n" if !$noerr;
1343 my $buf = "\x00" x
36;
1344 my $sensebuf = "\x00" x
8;
1345 my $cmd = pack("C x3 C x1", 0x12, 36);
1347 # see /usr/include/scsi/sg.h
1348 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1350 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351 length($sensebuf), 0, length($buf), $buf,
1352 $cmd, $sensebuf, 6000);
1354 $ret = ioctl($fh, $SG_IO, $packet);
1356 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1360 my @res = unpack($sg_io_hdr_t, $packet);
1361 if ($res[17] || $res[18]) {
1362 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1367 (my $byte0, my $byte1, $res->{vendor
},
1368 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1370 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1371 $res->{type
} = $byte0 & 31;
1379 my $fh = IO
::File-
>new("+<$path") || return;
1380 my $res = scsi_inquiry
($fh, 1);
1386 sub print_tabletdevice_full
{
1387 my ($conf, $arch) = @_;
1389 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1391 # we use uhci for old VMs because tablet driver was buggy in older qemu
1393 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1399 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1402 sub print_keyboarddevice_full
{
1403 my ($conf, $arch, $machine) = @_;
1405 return if $arch ne 'aarch64';
1407 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1410 sub print_drivedevice_full
{
1411 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1416 my $drive_id = "$drive->{interface}$drive->{index}";
1417 if ($drive->{interface
} eq 'virtio') {
1418 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1419 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1420 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1421 } elsif ($drive->{interface
} eq 'scsi') {
1423 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1424 my $unit = $drive->{index} % $maxdev;
1425 my $devicetype = 'hd';
1427 if (drive_is_cdrom
($drive)) {
1430 if ($drive->{file
} =~ m
|^/|) {
1431 $path = $drive->{file
};
1432 if (my $info = path_is_scsi
($path)) {
1433 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1434 $devicetype = 'block';
1435 } elsif ($info->{type
} == 1) { # tape
1436 $devicetype = 'generic';
1440 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1443 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1444 my $version = extract_version
($machine_type, kvm_user_version
());
1445 if ($path =~ m/^iscsi\:\/\
// &&
1446 !min_version
($version, 4, 1)) {
1447 $devicetype = 'generic';
1451 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1452 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1454 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1455 .",lun=$drive->{index}";
1457 $device .= ",drive=drive-$drive_id,id=$drive_id";
1459 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1460 $device .= ",rotation_rate=1";
1462 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1464 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1465 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1466 my $controller = int($drive->{index} / $maxdev);
1467 my $unit = $drive->{index} % $maxdev;
1468 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1470 $device = "ide-$devicetype";
1471 if ($drive->{interface
} eq 'ide') {
1472 $device .= ",bus=ide.$controller,unit=$unit";
1474 $device .= ",bus=ahci$controller.$unit";
1476 $device .= ",drive=drive-$drive_id,id=$drive_id";
1478 if ($devicetype eq 'hd') {
1479 if (my $model = $drive->{model
}) {
1480 $model = URI
::Escape
::uri_unescape
($model);
1481 $device .= ",model=$model";
1483 if ($drive->{ssd
}) {
1484 $device .= ",rotation_rate=1";
1487 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1488 } elsif ($drive->{interface
} eq 'usb') {
1490 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1492 die "unsupported interface type";
1495 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1497 if (my $serial = $drive->{serial
}) {
1498 $serial = URI
::Escape
::uri_unescape
($serial);
1499 $device .= ",serial=$serial";
1506 sub get_initiator_name
{
1509 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1510 while (defined(my $line = <$fh>)) {
1511 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1520 sub print_drive_commandline_full
{
1521 my ($storecfg, $vmid, $drive) = @_;
1524 my $volid = $drive->{file
};
1527 if (drive_is_cdrom
($drive)) {
1528 $path = get_iso_path
($storecfg, $vmid, $volid);
1530 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1532 $path = PVE
::Storage
::path
($storecfg, $volid);
1533 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1534 $format = qemu_img_format
($scfg, $volname);
1542 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1543 foreach my $o (@qemu_drive_options) {
1544 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1547 # snapshot only accepts on|off
1548 if (defined($drive->{snapshot
})) {
1549 my $v = $drive->{snapshot
} ?
'on' : 'off';
1550 $opts .= ",snapshot=$v";
1553 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1554 my ($dir, $qmpname) = @$type;
1555 if (my $v = $drive->{"mbps$dir"}) {
1556 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1558 if (my $v = $drive->{"mbps${dir}_max"}) {
1559 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1561 if (my $v = $drive->{"bps${dir}_max_length"}) {
1562 $opts .= ",throttling.bps$qmpname-max-length=$v";
1564 if (my $v = $drive->{"iops${dir}"}) {
1565 $opts .= ",throttling.iops$qmpname=$v";
1567 if (my $v = $drive->{"iops${dir}_max"}) {
1568 $opts .= ",throttling.iops$qmpname-max=$v";
1570 if (my $v = $drive->{"iops${dir}_max_length"}) {
1571 $opts .= ",throttling.iops$qmpname-max-length=$v";
1575 $opts .= ",format=$format" if $format && !$drive->{format
};
1577 my $cache_direct = 0;
1579 if (my $cache = $drive->{cache
}) {
1580 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1581 } elsif (!drive_is_cdrom
($drive)) {
1582 $opts .= ",cache=none";
1586 # aio native works only with O_DIRECT
1587 if (!$drive->{aio
}) {
1589 $opts .= ",aio=native";
1591 $opts .= ",aio=threads";
1595 if (!drive_is_cdrom
($drive)) {
1597 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1598 $detectzeroes = 'off';
1599 } elsif ($drive->{discard
}) {
1600 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1602 # This used to be our default with discard not being specified:
1603 $detectzeroes = 'on';
1605 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1608 my $pathinfo = $path ?
"file=$path," : '';
1610 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1613 sub print_netdevice_full
{
1614 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1616 my $device = $net->{model
};
1617 if ($net->{model
} eq 'virtio') {
1618 $device = 'virtio-net-pci';
1621 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1622 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1623 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1624 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1625 # and out of each queue plus one config interrupt and control vector queue
1626 my $vectors = $net->{queues
} * 2 + 2;
1627 $tmpstr .= ",vectors=$vectors,mq=on";
1629 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1631 if (my $mtu = $net->{mtu
}) {
1632 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1633 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1636 } elsif ($mtu < 576) {
1637 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1638 } elsif ($mtu > $bridge_mtu) {
1639 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1641 $tmpstr .= ",host_mtu=$mtu";
1643 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1647 if ($use_old_bios_files) {
1649 if ($device eq 'virtio-net-pci') {
1650 $romfile = 'pxe-virtio.rom';
1651 } elsif ($device eq 'e1000') {
1652 $romfile = 'pxe-e1000.rom';
1653 } elsif ($device eq 'ne2k') {
1654 $romfile = 'pxe-ne2k_pci.rom';
1655 } elsif ($device eq 'pcnet') {
1656 $romfile = 'pxe-pcnet.rom';
1657 } elsif ($device eq 'rtl8139') {
1658 $romfile = 'pxe-rtl8139.rom';
1660 $tmpstr .= ",romfile=$romfile" if $romfile;
1666 sub print_netdev_full
{
1667 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1670 if ($netid =~ m/^net(\d+)$/) {
1674 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1676 my $ifname = "tap${vmid}i$i";
1678 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1679 die "interface name '$ifname' is too long (max 15 character)\n"
1680 if length($ifname) >= 16;
1682 my $vhostparam = '';
1683 if (is_native
($arch)) {
1684 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1687 my $vmname = $conf->{name
} || "vm$vmid";
1690 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1692 if ($net->{bridge
}) {
1693 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1694 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1696 $netdev = "type=user,id=$netid,hostname=$vmname";
1699 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1705 'cirrus' => 'cirrus-vga',
1707 'vmware' => 'vmware-svga',
1708 'virtio' => 'virtio-vga',
1711 sub print_vga_device
{
1712 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1714 my $type = $vga_map->{$vga->{type
}};
1715 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1716 $type = 'virtio-gpu';
1718 my $vgamem_mb = $vga->{memory
};
1720 my $max_outputs = '';
1722 $type = $id ?
'qxl' : 'qxl-vga';
1724 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1725 # set max outputs so linux can have up to 4 qxl displays with one device
1726 if (min_version
($machine_version, 4, 1)) {
1727 $max_outputs = ",max_outputs=4";
1732 die "no devicetype for $vga->{type}\n" if !$type;
1736 if ($vga->{type
} eq 'virtio') {
1737 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1738 $memory = ",max_hostmem=$bytes";
1740 # from https://www.spice-space.org/multiple-monitors.html
1741 $memory = ",vgamem_mb=$vga->{memory}";
1742 my $ram = $vgamem_mb * 4;
1743 my $vram = $vgamem_mb * 2;
1744 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1746 $memory = ",vgamem_mb=$vga->{memory}";
1748 } elsif ($qxlnum && $id) {
1749 $memory = ",ram_size=67108864,vram_size=33554432";
1753 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1754 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1757 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1758 my $vgaid = "vga" . ($id // '');
1761 if ($q35 && $vgaid eq 'vga') {
1762 # the first display uses pcie.0 bus on q35 machines
1763 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1765 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1768 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1771 sub parse_number_sets
{
1774 foreach my $part (split(/;/, $set)) {
1775 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1776 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1777 push @$res, [ $1, $2 ];
1779 die "invalid range: $part\n";
1788 my $res = parse_property_string
($numa_fmt, $data);
1789 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1790 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1794 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1798 my $res = eval { parse_property_string
($net_fmt, $data) };
1803 if (!defined($res->{macaddr
})) {
1804 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1805 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1810 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1811 sub parse_ipconfig
{
1814 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1820 if ($res->{gw
} && !$res->{ip
}) {
1821 warn 'gateway specified without specifying an IP address';
1824 if ($res->{gw6
} && !$res->{ip6
}) {
1825 warn 'IPv6 gateway specified without specifying an IPv6 address';
1828 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1829 warn 'gateway specified together with DHCP';
1832 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1834 warn "IPv6 gateway specified together with $res->{ip6} address";
1838 if (!$res->{ip
} && !$res->{ip6
}) {
1839 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1848 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1851 sub add_random_macs
{
1852 my ($settings) = @_;
1854 foreach my $opt (keys %$settings) {
1855 next if $opt !~ m/^net(\d+)$/;
1856 my $net = parse_net
($settings->{$opt});
1858 $settings->{$opt} = print_net
($net);
1862 sub vm_is_volid_owner
{
1863 my ($storecfg, $vmid, $volid) = @_;
1865 if ($volid !~ m
|^/|) {
1867 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1868 if ($owner && ($owner == $vmid)) {
1876 sub vmconfig_register_unused_drive
{
1877 my ($storecfg, $vmid, $conf, $drive) = @_;
1879 if (drive_is_cloudinit
($drive)) {
1880 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1882 } elsif (!drive_is_cdrom
($drive)) {
1883 my $volid = $drive->{file
};
1884 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1885 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1890 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1894 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1895 format_description
=> 'UUID',
1896 description
=> "Set SMBIOS1 UUID.",
1901 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1902 format_description
=> 'Base64 encoded string',
1903 description
=> "Set SMBIOS1 version.",
1908 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1909 format_description
=> 'Base64 encoded string',
1910 description
=> "Set SMBIOS1 serial number.",
1915 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1916 format_description
=> 'Base64 encoded string',
1917 description
=> "Set SMBIOS1 manufacturer.",
1922 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1923 format_description
=> 'Base64 encoded string',
1924 description
=> "Set SMBIOS1 product ID.",
1929 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1930 format_description
=> 'Base64 encoded string',
1931 description
=> "Set SMBIOS1 SKU string.",
1936 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1937 format_description
=> 'Base64 encoded string',
1938 description
=> "Set SMBIOS1 family string.",
1943 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
1951 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
1958 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
1961 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
1963 sub parse_watchdog
{
1968 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
1973 sub parse_guest_agent
{
1976 return {} if !defined($value->{agent
});
1978 my $res = eval { parse_property_string
($agent_fmt, $value->{agent
}) };
1981 # if the agent is disabled ignore the other potentially set properties
1982 return {} if !$res->{enabled
};
1989 return {} if !$value;
1990 my $res = eval { parse_property_string
($vga_fmt, $value) };
2000 my $res = eval { parse_property_string
($rng_fmt, $value) };
2005 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2006 sub verify_usb_device
{
2007 my ($value, $noerr) = @_;
2009 return $value if parse_usb_device
($value);
2013 die "unable to parse usb device\n";
2016 # add JSON properties for create and set function
2017 sub json_config_properties
{
2020 foreach my $opt (keys %$confdesc) {
2021 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2022 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2023 $prop->{$opt} = $confdesc->{$opt};
2029 # return copy of $confdesc_cloudinit to generate documentation
2030 sub cloudinit_config_properties
{
2032 return dclone
($confdesc_cloudinit);
2036 my ($key, $value) = @_;
2038 die "unknown setting '$key'\n" if !$confdesc->{$key};
2040 my $type = $confdesc->{$key}->{type
};
2042 if (!defined($value)) {
2043 die "got undefined value\n";
2046 if ($value =~ m/[\n\r]/) {
2047 die "property contains a line feed\n";
2050 if ($type eq 'boolean') {
2051 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2052 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2053 die "type check ('boolean') failed - got '$value'\n";
2054 } elsif ($type eq 'integer') {
2055 return int($1) if $value =~ m/^(\d+)$/;
2056 die "type check ('integer') failed - got '$value'\n";
2057 } elsif ($type eq 'number') {
2058 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2059 die "type check ('number') failed - got '$value'\n";
2060 } elsif ($type eq 'string') {
2061 if (my $fmt = $confdesc->{$key}->{format
}) {
2062 PVE
::JSONSchema
::check_format
($fmt, $value);
2065 $value =~ s/^\"(.*)\"$/$1/;
2068 die "internal error"
2073 my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2075 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2077 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2079 if ($conf->{template
}) {
2080 # check if any base image is still used by a linked clone
2081 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2082 my ($ds, $drive) = @_;
2083 return if drive_is_cdrom
($drive);
2085 my $volid = $drive->{file
};
2086 return if !$volid || $volid =~ m
|^/|;
2088 die "base volume '$volid' is still in use by linked cloned\n"
2089 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2094 # only remove disks owned by this VM
2095 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2096 my ($ds, $drive) = @_;
2097 return if drive_is_cdrom
($drive, 1);
2099 my $volid = $drive->{file
};
2100 return if !$volid || $volid =~ m
|^/|;
2102 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2103 return if !$path || !$owner || ($owner != $vmid);
2105 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2106 warn "Could not remove disk '$volid', check manually: $@" if $@;
2109 # also remove unused disk
2110 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2111 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2112 my ($volid, $sid, $volname, $d) = @_;
2113 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2117 if (defined $replacement_conf) {
2118 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2120 PVE
::QemuConfig-
>destroy_config($vmid);
2124 sub parse_vm_config
{
2125 my ($filename, $raw) = @_;
2127 return if !defined($raw);
2130 digest
=> Digest
::SHA
::sha1_hex
($raw),
2135 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2136 || die "got strange filename '$filename'";
2144 my @lines = split(/\n/, $raw);
2145 foreach my $line (@lines) {
2146 next if $line =~ m/^\s*$/;
2148 if ($line =~ m/^\[PENDING\]\s*$/i) {
2149 $section = 'pending';
2150 if (defined($descr)) {
2152 $conf->{description
} = $descr;
2155 $conf = $res->{$section} = {};
2158 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2160 if (defined($descr)) {
2162 $conf->{description
} = $descr;
2165 $conf = $res->{snapshots
}->{$section} = {};
2169 if ($line =~ m/^\#(.*)\s*$/) {
2170 $descr = '' if !defined($descr);
2171 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2175 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2176 $descr = '' if !defined($descr);
2177 $descr .= PVE
::Tools
::decode_text
($2);
2178 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2179 $conf->{snapstate
} = $1;
2180 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2183 $conf->{$key} = $value;
2184 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2186 if ($section eq 'pending') {
2187 $conf->{delete} = $value; # we parse this later
2189 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2191 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2194 eval { $value = check_type
($key, $value); };
2196 warn "vm $vmid - unable to parse value of '$key' - $@";
2198 $key = 'ide2' if $key eq 'cdrom';
2199 my $fmt = $confdesc->{$key}->{format
};
2200 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2201 my $v = parse_drive
($key, $value);
2202 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2203 $v->{file
} = $volid;
2204 $value = print_drive
($v);
2206 warn "vm $vmid - unable to parse value of '$key'\n";
2211 $conf->{$key} = $value;
2216 if (defined($descr)) {
2218 $conf->{description
} = $descr;
2220 delete $res->{snapstate
}; # just to be sure
2225 sub write_vm_config
{
2226 my ($filename, $conf) = @_;
2228 delete $conf->{snapstate
}; # just to be sure
2230 if ($conf->{cdrom
}) {
2231 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2232 $conf->{ide2
} = $conf->{cdrom
};
2233 delete $conf->{cdrom
};
2236 # we do not use 'smp' any longer
2237 if ($conf->{sockets
}) {
2238 delete $conf->{smp
};
2239 } elsif ($conf->{smp
}) {
2240 $conf->{sockets
} = $conf->{smp
};
2241 delete $conf->{cores
};
2242 delete $conf->{smp
};
2245 my $used_volids = {};
2247 my $cleanup_config = sub {
2248 my ($cref, $pending, $snapname) = @_;
2250 foreach my $key (keys %$cref) {
2251 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2252 $key eq 'snapstate' || $key eq 'pending';
2253 my $value = $cref->{$key};
2254 if ($key eq 'delete') {
2255 die "propertry 'delete' is only allowed in [PENDING]\n"
2257 # fixme: check syntax?
2260 eval { $value = check_type
($key, $value); };
2261 die "unable to parse value of '$key' - $@" if $@;
2263 $cref->{$key} = $value;
2265 if (!$snapname && is_valid_drivename
($key)) {
2266 my $drive = parse_drive
($key, $value);
2267 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2272 &$cleanup_config($conf);
2274 &$cleanup_config($conf->{pending
}, 1);
2276 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2277 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2278 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2281 # remove 'unusedX' settings if we re-add a volume
2282 foreach my $key (keys %$conf) {
2283 my $value = $conf->{$key};
2284 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2285 delete $conf->{$key};
2289 my $generate_raw_config = sub {
2290 my ($conf, $pending) = @_;
2294 # add description as comment to top of file
2295 if (defined(my $descr = $conf->{description
})) {
2297 foreach my $cl (split(/\n/, $descr)) {
2298 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2301 $raw .= "#\n" if $pending;
2305 foreach my $key (sort keys %$conf) {
2306 next if $key =~ /^(digest|description|pending|snapshots)$/;
2307 $raw .= "$key: $conf->{$key}\n";
2312 my $raw = &$generate_raw_config($conf);
2314 if (scalar(keys %{$conf->{pending
}})){
2315 $raw .= "\n[PENDING]\n";
2316 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2319 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2320 $raw .= "\n[$snapname]\n";
2321 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2331 # we use static defaults from our JSON schema configuration
2332 foreach my $key (keys %$confdesc) {
2333 if (defined(my $default = $confdesc->{$key}->{default})) {
2334 $res->{$key} = $default;
2342 my $vmlist = PVE
::Cluster
::get_vmlist
();
2344 return $res if !$vmlist || !$vmlist->{ids
};
2345 my $ids = $vmlist->{ids
};
2346 my $nodename = nodename
();
2348 foreach my $vmid (keys %$ids) {
2349 my $d = $ids->{$vmid};
2350 next if !$d->{node
} || $d->{node
} ne $nodename;
2351 next if !$d->{type
} || $d->{type
} ne 'qemu';
2352 $res->{$vmid}->{exists} = 1;
2357 # test if VM uses local resources (to prevent migration)
2358 sub check_local_resources
{
2359 my ($conf, $noerr) = @_;
2363 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2364 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2366 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2368 foreach my $k (keys %$conf) {
2369 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2370 # sockets are safe: they will recreated be on the target side post-migrate
2371 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2372 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2375 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2380 # check if used storages are available on all nodes (use by migrate)
2381 sub check_storage_availability
{
2382 my ($storecfg, $conf, $node) = @_;
2384 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2385 my ($ds, $drive) = @_;
2387 my $volid = $drive->{file
};
2390 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2393 # check if storage is available on both nodes
2394 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2395 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2399 # list nodes where all VM images are available (used by has_feature API)
2401 my ($conf, $storecfg) = @_;
2403 my $nodelist = PVE
::Cluster
::get_nodelist
();
2404 my $nodehash = { map { $_ => 1 } @$nodelist };
2405 my $nodename = nodename
();
2407 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2408 my ($ds, $drive) = @_;
2410 my $volid = $drive->{file
};
2413 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2415 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2416 if ($scfg->{disable
}) {
2418 } elsif (my $avail = $scfg->{nodes
}) {
2419 foreach my $node (keys %$nodehash) {
2420 delete $nodehash->{$node} if !$avail->{$node};
2422 } elsif (!$scfg->{shared
}) {
2423 foreach my $node (keys %$nodehash) {
2424 delete $nodehash->{$node} if $node ne $nodename
2433 sub check_local_storage_availability
{
2434 my ($conf, $storecfg) = @_;
2436 my $nodelist = PVE
::Cluster
::get_nodelist
();
2437 my $nodehash = { map { $_ => {} } @$nodelist };
2439 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2440 my ($ds, $drive) = @_;
2442 my $volid = $drive->{file
};
2445 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2447 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2449 if ($scfg->{disable
}) {
2450 foreach my $node (keys %$nodehash) {
2451 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2453 } elsif (my $avail = $scfg->{nodes
}) {
2454 foreach my $node (keys %$nodehash) {
2455 if (!$avail->{$node}) {
2456 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2463 foreach my $node (values %$nodehash) {
2464 if (my $unavail = $node->{unavailable_storages
}) {
2465 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2472 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2474 my ($vmid, $nocheck, $node) = @_;
2476 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2477 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2482 my $vzlist = config_list
();
2484 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2486 while (defined(my $de = $fd->read)) {
2487 next if $de !~ m/^(\d+)\.pid$/;
2489 next if !defined($vzlist->{$vmid});
2490 if (my $pid = check_running
($vmid)) {
2491 $vzlist->{$vmid}->{pid
} = $pid;
2498 our $vmstatus_return_properties = {
2499 vmid
=> get_standard_option
('pve-vmid'),
2501 description
=> "Qemu process status.",
2503 enum
=> ['stopped', 'running'],
2506 description
=> "Maximum memory in bytes.",
2509 renderer
=> 'bytes',
2512 description
=> "Root disk size in bytes.",
2515 renderer
=> 'bytes',
2518 description
=> "VM name.",
2523 description
=> "Qemu QMP agent status.",
2528 description
=> "PID of running qemu process.",
2533 description
=> "Uptime.",
2536 renderer
=> 'duration',
2539 description
=> "Maximum usable CPUs.",
2544 description
=> "The current config lock, if any.",
2549 description
=> "The current configured tags, if any",
2555 my $last_proc_pid_stat;
2557 # get VM status information
2558 # This must be fast and should not block ($full == false)
2559 # We only query KVM using QMP if $full == true (this can be slow)
2561 my ($opt_vmid, $full) = @_;
2565 my $storecfg = PVE
::Storage
::config
();
2567 my $list = vzlist
();
2568 my $defaults = load_defaults
();
2570 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2572 my $cpucount = $cpuinfo->{cpus
} || 1;
2574 foreach my $vmid (keys %$list) {
2575 next if $opt_vmid && ($vmid ne $opt_vmid);
2577 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2579 my $d = { vmid
=> $vmid };
2580 $d->{pid
} = $list->{$vmid}->{pid
};
2582 # fixme: better status?
2583 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2585 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2586 if (defined($size)) {
2587 $d->{disk
} = 0; # no info available
2588 $d->{maxdisk
} = $size;
2594 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2595 * ($conf->{cores
} || $defaults->{cores
});
2596 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2597 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2599 $d->{name
} = $conf->{name
} || "VM $vmid";
2600 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2601 : $defaults->{memory
}*(1024*1024);
2603 if ($conf->{balloon
}) {
2604 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2605 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2606 : $defaults->{shares
};
2617 $d->{diskwrite
} = 0;
2619 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
2621 $d->{serial
} = 1 if conf_has_serial
($conf);
2622 $d->{lock} = $conf->{lock} if $conf->{lock};
2623 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2628 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2629 foreach my $dev (keys %$netdev) {
2630 next if $dev !~ m/^tap([1-9]\d*)i/;
2632 my $d = $res->{$vmid};
2635 $d->{netout
} += $netdev->{$dev}->{receive
};
2636 $d->{netin
} += $netdev->{$dev}->{transmit
};
2639 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
2640 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
2645 my $ctime = gettimeofday
;
2647 foreach my $vmid (keys %$list) {
2649 my $d = $res->{$vmid};
2650 my $pid = $d->{pid
};
2653 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2654 next if !$pstat; # not running
2656 my $used = $pstat->{utime} + $pstat->{stime
};
2658 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2660 if ($pstat->{vsize
}) {
2661 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2664 my $old = $last_proc_pid_stat->{$pid};
2666 $last_proc_pid_stat->{$pid} = {
2674 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2676 if ($dtime > 1000) {
2677 my $dutime = $used - $old->{used
};
2679 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2680 $last_proc_pid_stat->{$pid} = {
2686 $d->{cpu
} = $old->{cpu
};
2690 return $res if !$full;
2692 my $qmpclient = PVE
::QMPClient-
>new();
2694 my $ballooncb = sub {
2695 my ($vmid, $resp) = @_;
2697 my $info = $resp->{'return'};
2698 return if !$info->{max_mem
};
2700 my $d = $res->{$vmid};
2702 # use memory assigned to VM
2703 $d->{maxmem
} = $info->{max_mem
};
2704 $d->{balloon
} = $info->{actual
};
2706 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2707 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2708 $d->{freemem
} = $info->{free_mem
};
2711 $d->{ballooninfo
} = $info;
2714 my $blockstatscb = sub {
2715 my ($vmid, $resp) = @_;
2716 my $data = $resp->{'return'} || [];
2717 my $totalrdbytes = 0;
2718 my $totalwrbytes = 0;
2720 for my $blockstat (@$data) {
2721 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2722 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2724 $blockstat->{device
} =~ s/drive-//;
2725 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2727 $res->{$vmid}->{diskread
} = $totalrdbytes;
2728 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2731 my $statuscb = sub {
2732 my ($vmid, $resp) = @_;
2734 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2735 # this fails if ballon driver is not loaded, so this must be
2736 # the last commnand (following command are aborted if this fails).
2737 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2739 my $status = 'unknown';
2740 if (!defined($status = $resp->{'return'}->{status
})) {
2741 warn "unable to get VM status\n";
2745 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2748 foreach my $vmid (keys %$list) {
2749 next if $opt_vmid && ($vmid ne $opt_vmid);
2750 next if !$res->{$vmid}->{pid
}; # not running
2751 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2754 $qmpclient->queue_execute(undef, 2);
2756 foreach my $vmid (keys %$list) {
2757 next if $opt_vmid && ($vmid ne $opt_vmid);
2758 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2764 sub conf_has_serial
{
2767 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2768 if ($conf->{"serial$i"}) {
2776 sub conf_has_audio
{
2777 my ($conf, $id) = @_;
2780 my $audio = $conf->{"audio$id"};
2781 return if !defined($audio);
2783 my $audioproperties = parse_property_string
($audio_fmt, $audio);
2784 my $audiodriver = $audioproperties->{driver
} // 'spice';
2787 dev
=> $audioproperties->{device
},
2788 dev_id
=> "audiodev$id",
2789 backend
=> $audiodriver,
2790 backend_id
=> "$audiodriver-backend${id}",
2795 my ($audio, $audiopciaddr, $machine_version) = @_;
2799 my $id = $audio->{dev_id
};
2801 if (min_version
($machine_version, 4, 2)) {
2802 $audiodev = ",audiodev=$audio->{backend_id}";
2805 if ($audio->{dev
} eq 'AC97') {
2806 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2807 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
2808 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2809 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2810 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2812 die "unkown audio device '$audio->{dev}', implement me!";
2815 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2820 sub vga_conf_has_spice
{
2823 my $vgaconf = parse_vga
($vga);
2824 my $vgatype = $vgaconf->{type
};
2825 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2832 return get_host_arch
() eq $arch;
2837 return $conf->{arch
} // get_host_arch
();
2840 my $default_machines = {
2845 sub get_vm_machine
{
2846 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2848 my $machine = $forcemachine || $conf->{machine
};
2850 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2852 $machine ||= $default_machines->{$arch};
2853 if ($add_pve_version) {
2854 $kvmversion //= kvm_user_version
();
2855 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
2856 $machine .= "+pve$pvever";
2860 if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2861 # for version-pinned machines that do not include a pve-version (e.g.
2862 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2863 $machine .= '+pve0';
2869 sub get_ovmf_files
($) {
2872 my $ovmf = $OVMF->{$arch}
2873 or die "no OVMF images known for architecture '$arch'\n";
2879 aarch64
=> '/usr/bin/qemu-system-aarch64',
2880 x86_64
=> '/usr/bin/qemu-system-x86_64',
2882 sub get_command_for_arch
($) {
2884 return '/usr/bin/kvm' if is_native
($arch);
2886 my $cmd = $Arch2Qemu->{$arch}
2887 or die "don't know how to emulate architecture '$arch'\n";
2891 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2892 # to use in a QEMU command line (-cpu element), first array_intersect the result
2893 # of query_supported_ with query_understood_. This is necessary because:
2895 # a) query_understood_ returns flags the host cannot use and
2896 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2897 # flags, but CPU settings - with most of them being flags. Those settings
2898 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
2900 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2901 # expensive. If you need the value returned from this, you can get it much
2902 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2903 # $accel being 'kvm' or 'tcg'.
2905 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2906 # changes, automatically populating pmxcfs.
2908 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2909 # since kvm and tcg machines support different flags
2911 sub query_supported_cpu_flags
{
2914 $arch //= get_host_arch
();
2915 my $default_machine = $default_machines->{$arch};
2919 # FIXME: Once this is merged, the code below should work for ARM as well:
2920 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2921 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2924 my $kvm_supported = defined(kvm_version
());
2925 my $qemu_cmd = get_command_for_arch
($arch);
2927 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
2929 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2930 my $query_supported_run_qemu = sub {
2936 '-machine', $default_machine,
2938 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2939 '-mon', 'chardev=qmp,mode=control',
2940 '-pidfile', $pidfile,
2945 push @$cmd, '-accel', 'tcg';
2948 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
2949 die "QEMU flag querying VM exited with code " . $rc if $rc;
2952 my $cmd_result = mon_cmd
(
2954 'query-cpu-model-expansion',
2956 model
=> { name
=> 'host' }
2959 my $props = $cmd_result->{model
}->{props
};
2960 foreach my $prop (keys %$props) {
2961 next if $props->{$prop} ne '1';
2962 # QEMU returns some flags multiple times, with '_', '.' or '-'
2963 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2964 # We only keep those with underscores, to match /proc/cpuinfo
2965 $prop =~ s/\.|-/_/g;
2966 $flags->{$prop} = 1;
2971 # force stop with 10 sec timeout and 'nocheck'
2972 # always stop, even if QMP failed
2973 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
2977 return [ sort keys %$flags ];
2980 # We need to query QEMU twice, since KVM and TCG have different supported flags
2981 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
2982 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
2983 warn "warning: failed querying supported tcg flags: $@\n" if $@;
2985 if ($kvm_supported) {
2986 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
2987 warn "warning: failed querying supported kvm flags: $@\n" if $@;
2994 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2995 my $understood_cpu_flag_dir = "/usr/share/kvm";
2996 sub query_understood_cpu_flags
{
2997 my $arch = get_host_arch
();
2998 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3000 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3003 my $raw = file_get_contents
($filepath);
3004 $raw =~ s/^\s+|\s+$//g;
3005 my @flags = split(/\s+/, $raw);
3010 sub config_to_command
{
3011 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3014 my $globalFlags = [];
3015 my $machineFlags = [];
3020 my $ostype = $conf->{ostype
};
3021 my $winversion = windows_version
($ostype);
3022 my $kvm = $conf->{kvm
};
3023 my $nodename = nodename
();
3025 my $arch = get_vm_arch
($conf);
3026 my $kvm_binary = get_command_for_arch
($arch);
3027 my $kvmver = kvm_user_version
($kvm_binary);
3029 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3030 $kvmver //= "undefined";
3031 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3034 my $add_pve_version = min_version
($kvmver, 4, 1);
3036 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3037 my $machine_version = extract_version
($machine_type, $kvmver);
3038 $kvm //= 1 if is_native
($arch);
3040 $machine_version =~ m/(\d+)\.(\d+)/;
3041 my ($machine_major, $machine_minor) = ($1, $2);
3043 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3044 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3045 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3046 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3047 ." please upgrade node '$nodename'\n"
3048 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3049 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3050 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3051 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3052 ." node '$nodename'\n";
3055 # if a specific +pve version is required for a feature, use $version_guard
3056 # instead of min_version to allow machines to be run with the minimum
3058 my $required_pve_version = 0;
3059 my $version_guard = sub {
3060 my ($major, $minor, $pve) = @_;
3061 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3062 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3063 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3064 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3068 if ($kvm && !defined kvm_version
()) {
3069 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3070 ." or enable in BIOS.\n";
3073 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3074 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3075 my $use_old_bios_files = undef;
3076 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3078 my $cpuunits = defined($conf->{cpuunits
}) ?
3079 $conf->{cpuunits
} : $defaults->{cpuunits
};
3081 push @$cmd, $kvm_binary;
3083 push @$cmd, '-id', $vmid;
3085 my $vmname = $conf->{name
} || "vm$vmid";
3087 push @$cmd, '-name', $vmname;
3091 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3092 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3093 push @$cmd, '-mon', "chardev=qmp,mode=control";
3095 if (min_version
($machine_version, 2, 12)) {
3096 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3097 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3100 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3102 push @$cmd, '-daemonize';
3104 if ($conf->{smbios1
}) {
3105 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3106 if ($smbios_conf->{base64
}) {
3107 # Do not pass base64 flag to qemu
3108 delete $smbios_conf->{base64
};
3109 my $smbios_string = "";
3110 foreach my $key (keys %$smbios_conf) {
3112 if ($key eq "uuid") {
3113 $value = $smbios_conf->{uuid
}
3115 $value = decode_base64
($smbios_conf->{$key});
3117 # qemu accepts any binary data, only commas need escaping by double comma
3119 $smbios_string .= "," . $key . "=" . $value if $value;
3121 push @$cmd, '-smbios', "type=1" . $smbios_string;
3123 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3127 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3128 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3129 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3131 my ($path, $format);
3132 if (my $efidisk = $conf->{efidisk0
}) {
3133 my $d = parse_drive
('efidisk0', $efidisk);
3134 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3135 $format = $d->{format
};
3137 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3138 if (!defined($format)) {
3139 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3140 $format = qemu_img_format
($scfg, $volname);
3144 die "efidisk format must be specified\n"
3145 if !defined($format);
3148 warn "no efidisk configured! Using temporary efivars disk.\n";
3149 $path = "/tmp/$vmid-ovmf.fd";
3150 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3156 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3157 $size_str = ",size=" . (-s
$ovmf_vars);
3160 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3161 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3166 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3167 if (min_version
($machine_version, 4, 0)) {
3168 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3170 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3174 if ($conf->{vmgenid
}) {
3175 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3178 # add usb controllers
3179 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3180 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3181 push @$devices, @usbcontrollers if @usbcontrollers;
3182 my $vga = parse_vga
($conf->{vga
});
3184 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3185 $vga->{type
} = 'qxl' if $qxlnum;
3187 if (!$vga->{type
}) {
3188 if ($arch eq 'aarch64') {
3189 $vga->{type
} = 'virtio';
3190 } elsif (min_version
($machine_version, 2, 9)) {
3191 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3193 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3197 # enable absolute mouse coordinates (needed by vnc)
3199 if (defined($conf->{tablet
})) {
3200 $tablet = $conf->{tablet
};
3202 $tablet = $defaults->{tablet
};
3203 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3204 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3208 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3209 my $kbd = print_keyboarddevice_full
($conf, $arch);
3210 push @$devices, '-device', $kbd if defined($kbd);
3214 my $boot = parse_property_string
($boot_fmt, $conf->{boot
}) if $conf->{boot
};
3215 if (!defined($boot) || $boot->{legacy
}) {
3216 $bootorder = bootorder_from_legacy
($conf, $boot);
3217 } elsif ($boot->{order
}) {
3218 # start at 100 to allow user to insert devices before us with -args
3220 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
3221 $bootorder->{$dev} = $i++;
3225 # host pci device passthrough
3226 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3227 $vmid, $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3230 my $usb_dev_features = {};
3231 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3233 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3234 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3235 push @$devices, @usbdevices if @usbdevices;
3238 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3239 if (my $path = $conf->{"serial$i"}) {
3240 if ($path eq 'socket') {
3241 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3242 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3243 # On aarch64, serial0 is the UART device. Qemu only allows
3244 # connecting UART devices via the '-serial' command line, as
3245 # the device has a fixed slot on the hardware...
3246 if ($arch eq 'aarch64' && $i == 0) {
3247 push @$devices, '-serial', "chardev:serial$i";
3249 push @$devices, '-device', "isa-serial,chardev=serial$i";
3252 die "no such serial device\n" if ! -c
$path;
3253 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3254 push @$devices, '-device', "isa-serial,chardev=serial$i";
3260 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3261 if (my $path = $conf->{"parallel$i"}) {
3262 die "no such parallel device\n" if ! -c
$path;
3263 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3264 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3265 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3269 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3270 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3271 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3272 push @$devices, @$audio_devs;
3276 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3277 $sockets = $conf->{sockets
} if $conf->{sockets
};
3279 my $cores = $conf->{cores
} || 1;
3281 my $maxcpus = $sockets * $cores;
3283 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3285 my $allowed_vcpus = $cpuinfo->{cpus
};
3287 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3288 if ($allowed_vcpus < $maxcpus);
3290 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3292 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3293 for (my $i = 2; $i <= $vcpus; $i++) {
3294 my $cpustr = print_cpu_device
($conf,$i);
3295 push @$cmd, '-device', $cpustr;
3300 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3302 push @$cmd, '-nodefaults';
3304 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3306 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3308 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3310 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3311 push @$devices, '-device', print_vga_device
(
3312 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3313 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3314 push @$cmd, '-vnc', "unix:$socket,password";
3316 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3317 push @$cmd, '-nographic';
3321 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3322 my $useLocaltime = $conf->{localtime};
3324 if ($winversion >= 5) { # windows
3325 $useLocaltime = 1 if !defined($conf->{localtime});
3327 # use time drift fix when acpi is enabled
3328 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3329 $tdf = 1 if !defined($conf->{tdf
});
3333 if ($winversion >= 6) {
3334 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3335 push @$cmd, '-no-hpet';
3338 push @$rtcFlags, 'driftfix=slew' if $tdf;
3340 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3341 push @$rtcFlags, "base=$conf->{startdate}";
3342 } elsif ($useLocaltime) {
3343 push @$rtcFlags, 'base=localtime';
3347 push @$cmd, '-cpu', $forcecpu;
3349 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3352 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3354 push @$cmd, '-S' if $conf->{freeze
};
3356 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3358 my $guest_agent = parse_guest_agent
($conf);
3360 if ($guest_agent->{enabled
}) {
3361 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3362 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3364 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3365 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3366 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3367 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3368 } elsif ($guest_agent->{type
} eq 'isa') {
3369 push @$devices, '-device', "isa-serial,chardev=qga0";
3373 my $rng = parse_rng
($conf->{rng0
}) if $conf->{rng0
};
3374 if ($rng && &$version_guard(4, 1, 2)) {
3375 check_rng_source
($rng->{source
});
3377 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3378 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3379 my $limiter_str = "";
3381 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3384 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3385 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3386 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3394 for (my $i = 1; $i < $qxlnum; $i++){
3395 push @$devices, '-device', print_vga_device
(
3396 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3399 # assume other OS works like Linux
3400 my ($ram, $vram) = ("134217728", "67108864");
3401 if ($vga->{memory
}) {
3402 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3403 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3405 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3406 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3410 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3412 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3413 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3414 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3416 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3417 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3418 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3420 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3421 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3423 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3424 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3425 if ($spice_enhancement->{foldersharing
}) {
3426 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3427 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3430 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3431 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3432 if $spice_enhancement->{videostreaming
};
3434 push @$devices, '-spice', "$spice_opts";
3437 # enable balloon by default, unless explicitly disabled
3438 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3439 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3440 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3443 if ($conf->{watchdog
}) {
3444 my $wdopts = parse_watchdog
($conf->{watchdog
});
3445 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3446 my $watchdog = $wdopts->{model
} || 'i6300esb';
3447 push @$devices, '-device', "$watchdog$pciaddr";
3448 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3452 my $scsicontroller = {};
3453 my $ahcicontroller = {};
3454 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3456 # Add iscsi initiator name if available
3457 if (my $initiator = get_initiator_name
()) {
3458 push @$devices, '-iscsi', "initiator-name=$initiator";
3461 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3462 my ($ds, $drive) = @_;
3464 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3465 push @$vollist, $drive->{file
};
3468 # ignore efidisk here, already added in bios/fw handling code above
3469 return if $drive->{interface
} eq 'efidisk';
3471 $use_virtio = 1 if $ds =~ m/^virtio/;
3473 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3475 if ($drive->{interface
} eq 'virtio'){
3476 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3479 if ($drive->{interface
} eq 'scsi') {
3481 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3483 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3484 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3486 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3487 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3490 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3491 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3492 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3493 } elsif ($drive->{iothread
}) {
3494 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3498 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3499 $queues = ",num_queues=$drive->{queues}";
3502 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3503 if !$scsicontroller->{$controller};
3504 $scsicontroller->{$controller}=1;
3507 if ($drive->{interface
} eq 'sata') {
3508 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3509 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3510 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3511 if !$ahcicontroller->{$controller};
3512 $ahcicontroller->{$controller}=1;
3515 my $drive_cmd = print_drive_commandline_full
($storecfg, $vmid, $drive);
3516 $drive_cmd .= ',readonly' if PVE
::QemuConfig-
>is_template($conf);
3518 push @$devices, '-drive',$drive_cmd;
3519 push @$devices, '-device', print_drivedevice_full
(
3520 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3523 for (my $i = 0; $i < $MAX_NETS; $i++) {
3524 my $netname = "net$i";
3526 next if !$conf->{$netname};
3527 my $d = parse_net
($conf->{$netname});
3530 $use_virtio = 1 if $d->{model
} eq 'virtio';
3532 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
3534 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
3535 push @$devices, '-netdev', $netdevfull;
3537 my $netdevicefull = print_netdevice_full
(
3538 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3540 push @$devices, '-device', $netdevicefull;
3543 if ($conf->{ivshmem
}) {
3544 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3548 $bus = print_pcie_addr
("ivshmem");
3550 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3553 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3554 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3556 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3557 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3558 .",size=$ivshmem->{size}M";
3561 # pci.4 is nested in pci.1
3562 $bridges->{1} = 1 if $bridges->{4};
3566 if (min_version
($machine_version, 2, 3)) {
3571 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3575 for my $k (sort {$b cmp $a} keys %$bridges) {
3576 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3579 if ($k == 2 && $legacy_igd) {
3582 $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
3584 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3586 # add after -readconfig pve-q35.cfg
3587 splice @$devices, 2, 0, '-device', $devstr;
3589 unshift @$devices, '-device', $devstr if $k > 0;
3594 push @$machineFlags, 'accel=tcg';
3597 my $machine_type_min = $machine_type;
3598 if ($add_pve_version) {
3599 $machine_type_min =~ s/\+pve\d+$//;
3600 $machine_type_min .= "+pve$required_pve_version";
3602 push @$machineFlags, "type=${machine_type_min}";
3604 push @$cmd, @$devices;
3605 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3606 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3607 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3609 if (my $vmstate = $conf->{vmstate
}) {
3610 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3611 push @$vollist, $vmstate;
3612 push @$cmd, '-loadstate', $statepath;
3613 print "activating and using '$vmstate' as vmstate\n";
3617 if ($conf->{args
}) {
3618 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3622 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3625 sub check_rng_source
{
3628 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3629 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3632 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3633 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
3634 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3635 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3636 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3637 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3645 my $res = mon_cmd
($vmid, 'query-spice');
3647 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3650 sub vm_devices_list
{
3653 my $res = mon_cmd
($vmid, 'query-pci');
3654 my $devices_to_check = [];
3656 foreach my $pcibus (@$res) {
3657 push @$devices_to_check, @{$pcibus->{devices
}},
3660 while (@$devices_to_check) {
3662 for my $d (@$devices_to_check) {
3663 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3664 next if !$d->{'pci_bridge'};
3666 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3667 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3669 $devices_to_check = $to_check;
3672 my $resblock = mon_cmd
($vmid, 'query-block');
3673 foreach my $block (@$resblock) {
3674 if($block->{device
} =~ m/^drive-(\S+)/){
3679 my $resmice = mon_cmd
($vmid, 'query-mice');
3680 foreach my $mice (@$resmice) {
3681 if ($mice->{name
} eq 'QEMU HID Tablet') {
3682 $devices->{tablet
} = 1;
3687 # for usb devices there is no query-usb
3688 # but we can iterate over the entries in
3689 # qom-list path=/machine/peripheral
3690 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3691 foreach my $per (@$resperipheral) {
3692 if ($per->{name
} =~ m/^usb\d+$/) {
3693 $devices->{$per->{name
}} = 1;
3701 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3703 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3705 my $devices_list = vm_devices_list
($vmid);
3706 return 1 if defined($devices_list->{$deviceid});
3708 # add PCI bridge if we need it for the device
3709 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3711 if ($deviceid eq 'tablet') {
3713 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
3715 } elsif ($deviceid eq 'keyboard') {
3717 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
3719 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3721 die "usb hotplug currently not reliable\n";
3722 # since we can't reliably hot unplug all added usb devices
3723 # and usb passthrough disables live migration
3724 # we disable usb hotplugging for now
3725 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device));
3727 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3729 qemu_iothread_add
($vmid, $deviceid, $device);
3731 qemu_driveadd
($storecfg, $vmid, $device);
3732 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3734 qemu_deviceadd
($vmid, $devicefull);
3735 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3737 eval { qemu_drivedel
($vmid, $deviceid); };
3742 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3745 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3746 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3747 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3749 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3751 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3752 qemu_iothread_add
($vmid, $deviceid, $device);
3753 $devicefull .= ",iothread=iothread-$deviceid";
3756 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3757 $devicefull .= ",num_queues=$device->{queues}";
3760 qemu_deviceadd
($vmid, $devicefull);
3761 qemu_deviceaddverify
($vmid, $deviceid);
3763 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3765 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3766 qemu_driveadd
($storecfg, $vmid, $device);
3768 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3769 eval { qemu_deviceadd
($vmid, $devicefull); };
3771 eval { qemu_drivedel
($vmid, $deviceid); };
3776 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3778 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
3780 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
3781 my $use_old_bios_files = undef;
3782 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3784 my $netdevicefull = print_netdevice_full
(
3785 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3786 qemu_deviceadd
($vmid, $netdevicefull);
3788 qemu_deviceaddverify
($vmid, $deviceid);
3789 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
3792 eval { qemu_netdevdel
($vmid, $deviceid); };
3797 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3800 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3801 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3803 qemu_deviceadd
($vmid, $devicefull);
3804 qemu_deviceaddverify
($vmid, $deviceid);
3807 die "can't hotplug device '$deviceid'\n";
3813 # fixme: this should raise exceptions on error!
3814 sub vm_deviceunplug
{
3815 my ($vmid, $conf, $deviceid) = @_;
3817 my $devices_list = vm_devices_list
($vmid);
3818 return 1 if !defined($devices_list->{$deviceid});
3820 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
3821 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3823 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3825 qemu_devicedel
($vmid, $deviceid);
3827 } elsif ($deviceid =~ m/^usb\d+$/) {
3829 die "usb hotplug currently not reliable\n";
3830 # when unplugging usb devices this way,
3831 # there may be remaining usb controllers/hubs
3832 # so we disable it for now
3833 qemu_devicedel
($vmid, $deviceid);
3834 qemu_devicedelverify
($vmid, $deviceid);
3836 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3838 qemu_devicedel
($vmid, $deviceid);
3839 qemu_devicedelverify
($vmid, $deviceid);
3840 qemu_drivedel
($vmid, $deviceid);
3841 qemu_iothread_del
($conf, $vmid, $deviceid);
3843 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3845 qemu_devicedel
($vmid, $deviceid);
3846 qemu_devicedelverify
($vmid, $deviceid);
3847 qemu_iothread_del
($conf, $vmid, $deviceid);
3849 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3851 qemu_devicedel
($vmid, $deviceid);
3852 qemu_drivedel
($vmid, $deviceid);
3853 qemu_deletescsihw
($conf, $vmid, $deviceid);
3855 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3857 qemu_devicedel
($vmid, $deviceid);
3858 qemu_devicedelverify
($vmid, $deviceid);
3859 qemu_netdevdel
($vmid, $deviceid);
3862 die "can't unplug device '$deviceid'\n";
3868 sub qemu_deviceadd
{
3869 my ($vmid, $devicefull) = @_;
3871 $devicefull = "driver=".$devicefull;
3872 my %options = split(/[=,]/, $devicefull);
3874 mon_cmd
($vmid, "device_add" , %options);
3877 sub qemu_devicedel
{
3878 my ($vmid, $deviceid) = @_;
3880 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
3883 sub qemu_iothread_add
{
3884 my($vmid, $deviceid, $device) = @_;
3886 if ($device->{iothread
}) {
3887 my $iothreads = vm_iothreads_list
($vmid);
3888 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3892 sub qemu_iothread_del
{
3893 my($conf, $vmid, $deviceid) = @_;
3895 my $confid = $deviceid;
3896 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3897 $confid = 'scsi' . $1;
3899 my $device = parse_drive
($confid, $conf->{$confid});
3900 if ($device->{iothread
}) {
3901 my $iothreads = vm_iothreads_list
($vmid);
3902 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3906 sub qemu_objectadd
{
3907 my($vmid, $objectid, $qomtype) = @_;
3909 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
3914 sub qemu_objectdel
{
3915 my($vmid, $objectid) = @_;
3917 mon_cmd
($vmid, "object-del", id
=> $objectid);
3923 my ($storecfg, $vmid, $device) = @_;
3925 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device);
3926 $drive =~ s/\\/\\\\/g;
3927 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
3929 # If the command succeeds qemu prints: "OK
"
3930 return 1 if $ret =~ m/OK/s;
3932 die "adding drive failed
: $ret\n";
3936 my($vmid, $deviceid) = @_;
3938 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
3941 return 1 if $ret eq "";
3943 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3944 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3946 die "deleting drive
$deviceid failed
: $ret\n";
3949 sub qemu_deviceaddverify {
3950 my ($vmid, $deviceid) = @_;
3952 for (my $i = 0; $i <= 5; $i++) {
3953 my $devices_list = vm_devices_list($vmid);
3954 return 1 if defined($devices_list->{$deviceid});
3958 die "error on hotplug device
'$deviceid'\n";
3962 sub qemu_devicedelverify {
3963 my ($vmid, $deviceid) = @_;
3965 # need to verify that the device is correctly removed as device_del
3966 # is async and empty return is not reliable
3968 for (my $i = 0; $i <= 5; $i++) {
3969 my $devices_list = vm_devices_list($vmid);
3970 return 1 if !defined($devices_list->{$deviceid});
3974 die "error on hot-unplugging device
'$deviceid'\n";
3977 sub qemu_findorcreatescsihw {
3978 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3980 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3982 my $scsihwid="$controller_prefix$controller";
3983 my $devices_list = vm_devices_list($vmid);
3985 if(!defined($devices_list->{$scsihwid})) {
3986 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3992 sub qemu_deletescsihw {
3993 my ($conf, $vmid, $opt) = @_;
3995 my $device = parse_drive($opt, $conf->{$opt});
3997 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3998 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4002 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4004 my $devices_list = vm_devices_list($vmid);
4005 foreach my $opt (keys %{$devices_list}) {
4006 if (is_valid_drivename($opt)) {
4007 my $drive = parse_drive($opt, $conf->{$opt});
4008 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4014 my $scsihwid="scsihw
$controller";
4016 vm_deviceunplug($vmid, $conf, $scsihwid);
4021 sub qemu_add_pci_bridge {
4022 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4028 print_pci_addr($device, $bridges, $arch, $machine_type);
4030 while (my ($k, $v) = each %$bridges) {
4033 return 1 if !defined($bridgeid) || $bridgeid < 1;
4035 my $bridge = "pci
.$bridgeid";
4036 my $devices_list = vm_devices_list($vmid);
4038 if (!defined($devices_list->{$bridge})) {
4039 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4045 sub qemu_set_link_status {
4046 my ($vmid, $device, $up) = @_;
4048 mon_cmd($vmid, "set_link
", name => $device,
4049 up => $up ? JSON::true : JSON::false);
4052 sub qemu_netdevadd {
4053 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4055 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4056 my %options = split(/[=,]/, $netdev);
4058 if (defined(my $vhost = $options{vhost})) {
4059 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4062 if (defined(my $queues = $options{queues})) {
4063 $options{queues} = $queues + 0;
4066 mon_cmd($vmid, "netdev_add
", %options);
4070 sub qemu_netdevdel {
4071 my ($vmid, $deviceid) = @_;
4073 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4076 sub qemu_usb_hotplug {
4077 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4081 # remove the old one first
4082 vm_deviceunplug($vmid, $conf, $deviceid);
4084 # check if xhci controller is necessary and available
4085 if ($device->{usb3}) {
4087 my $devicelist = vm_devices_list($vmid);
4089 if (!$devicelist->{xhci}) {
4090 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4091 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4094 my $d = parse_usb_device($device->{host});
4095 $d->{usb3} = $device->{usb3};
4098 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4101 sub qemu_cpu_hotplug {
4102 my ($vmid, $conf, $vcpus) = @_;
4104 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4107 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4108 $sockets = $conf->{sockets} if $conf->{sockets};
4109 my $cores = $conf->{cores} || 1;
4110 my $maxcpus = $sockets * $cores;
4112 $vcpus = $maxcpus if !$vcpus;
4114 die "you can
't add more vcpus than maxcpus\n"
4115 if $vcpus > $maxcpus;
4117 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4119 if ($vcpus < $currentvcpus) {
4121 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4123 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4124 qemu_devicedel($vmid, "cpu$i");
4126 my $currentrunningvcpus = undef;
4128 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4129 last if scalar(@{$currentrunningvcpus}) == $i-1;
4130 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4134 #update conf after each succesfull cpu unplug
4135 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4136 PVE::QemuConfig->write_config($vmid, $conf);
4139 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4145 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4146 die "vcpus in running vm does not match its configuration\n"
4147 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4149 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4151 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4152 my $cpustr = print_cpu_device($conf, $i);
4153 qemu_deviceadd($vmid, $cpustr);
4156 my $currentrunningvcpus = undef;
4158 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4159 last if scalar(@{$currentrunningvcpus}) == $i;
4160 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4164 #update conf after each succesfull cpu hotplug
4165 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4166 PVE::QemuConfig->write_config($vmid, $conf);
4170 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4171 mon_cmd($vmid, "cpu-add", id => int($i));
4176 sub qemu_block_set_io_throttle {
4177 my ($vmid, $deviceid,
4178 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4179 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4180 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4181 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4183 return if !check_running($vmid) ;
4185 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4187 bps_rd => int($bps_rd),
4188 bps_wr => int($bps_wr),
4190 iops_rd => int($iops_rd),
4191 iops_wr => int($iops_wr),
4192 bps_max => int($bps_max),
4193 bps_rd_max => int($bps_rd_max),
4194 bps_wr_max => int($bps_wr_max),
4195 iops_max => int($iops_max),
4196 iops_rd_max => int($iops_rd_max),
4197 iops_wr_max => int($iops_wr_max),
4198 bps_max_length => int($bps_max_length),
4199 bps_rd_max_length => int($bps_rd_max_length),
4200 bps_wr_max_length => int($bps_wr_max_length),
4201 iops_max_length => int($iops_max_length),
4202 iops_rd_max_length => int($iops_rd_max_length),
4203 iops_wr_max_length => int($iops_wr_max_length),
4208 # old code, only used to shutdown old VM after update
4210 my ($fh, $timeout) = @_;
4212 my $sel = new IO::Select;
4219 while (scalar (@ready = $sel->can_read($timeout))) {
4221 if ($count = $fh->sysread($buf, 8192)) {
4222 if ($buf =~ /^(.*)\(qemu\) $/s) {
4229 if (!defined($count)) {
4236 die "monitor read timeout\n" if !scalar(@ready);
4241 sub qemu_block_resize {
4242 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4244 my $running = check_running($vmid);
4246 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4248 return if !$running;
4250 my $padding = (1024 - $size % 1024) % 1024;
4251 $size = $size + $padding;
4253 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4257 sub qemu_volume_snapshot {
4258 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4260 my $running = check_running($vmid);
4262 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4263 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4265 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4269 sub qemu_volume_snapshot_delete {
4270 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4272 my $running = check_running($vmid);
4277 my $conf = PVE::QemuConfig->load_config($vmid);
4278 PVE::QemuConfig->foreach_volume($conf, sub {
4279 my ($ds, $drive) = @_;
4280 $running = 1 if $drive->{file} eq $volid;
4284 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4285 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4287 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4291 sub set_migration_caps {
4297 "auto-converge" => 1,
4299 "x-rdma-pin-all" => 0,
4304 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4306 for my $supported_capability (@$supported_capabilities) {
4308 capability => $supported_capability->{capability},
4309 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4313 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4317 my ($conf, $func, @param) = @_;
4321 my $test_volid = sub {
4322 my ($key, $drive, $snapname) = @_;
4324 my $volid = $drive->{file};
4327 $volhash->{$volid}->{cdrom} //= 1;
4328 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4330 my $replicate = $drive->{replicate} // 1;
4331 $volhash->{$volid}->{replicate} //= 0;
4332 $volhash->{$volid}->{replicate} = 1 if $replicate;
4334 $volhash->{$volid}->{shared} //= 0;
4335 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4337 $volhash->{$volid}->{referenced_in_config} //= 0;
4338 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4340 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4341 if defined($snapname);
4343 my $size = $drive->{size};
4344 $volhash->{$volid}->{size} //= $size if $size;
4346 $volhash->{$volid}->{is_vmstate} //= 0;
4347 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4349 $volhash->{$volid}->{is_unused} //= 0;
4350 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4353 my $include_opts = {
4354 extra_keys => ['vmstate
'],
4355 include_unused => 1,
4358 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4359 foreach my $snapname (keys %{$conf->{snapshots}}) {
4360 my $snap = $conf->{snapshots}->{$snapname};
4361 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4364 foreach my $volid (keys %$volhash) {
4365 &$func($volid, $volhash->{$volid}, @param);
4369 my $fast_plug_option = {
4377 'vmstatestorage
' => 1,
4382 # hotplug changes in [PENDING]
4383 # $selection hash can be used to only apply specified options, for
4384 # example: { cores => 1 } (only apply changed 'cores
')
4385 # $errors ref is used to return error messages
4386 sub vmconfig_hotplug_pending {
4387 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4389 my $defaults = load_defaults();
4390 my $arch = get_vm_arch($conf);
4391 my $machine_type = get_vm_machine($conf, undef, $arch);
4393 # commit values which do not have any impact on running VM first
4394 # Note: those option cannot raise errors, we we do not care about
4395 # $selection and always apply them.
4397 my $add_error = sub {
4398 my ($opt, $msg) = @_;
4399 $errors->{$opt} = "hotplug problem - $msg";
4403 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4404 if ($fast_plug_option->{$opt}) {
4405 $conf->{$opt} = $conf->{pending}->{$opt};
4406 delete $conf->{pending}->{$opt};
4412 PVE::QemuConfig->write_config($vmid, $conf);
4415 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4417 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4418 foreach my $opt (sort keys %$pending_delete_hash) {
4419 next if $selection && !$selection->{$opt};
4420 my $force = $pending_delete_hash->{$opt}->{force};
4422 if ($opt eq 'hotplug
') {
4423 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4424 } elsif ($opt eq 'tablet
') {
4425 die "skip\n" if !$hotplug_features->{usb};
4426 if ($defaults->{tablet}) {
4427 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4428 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4429 if $arch eq 'aarch64
';
4431 vm_deviceunplug($vmid, $conf, 'tablet
');
4432 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4434 } elsif ($opt =~ m/^usb\d+/) {
4436 # since we cannot reliably hot unplug usb devices
4437 # we are disabling it
4438 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4439 vm_deviceunplug($vmid, $conf, $opt);
4440 } elsif ($opt eq 'vcpus
') {
4441 die "skip\n" if !$hotplug_features->{cpu};
4442 qemu_cpu_hotplug($vmid, $conf, undef);
4443 } elsif ($opt eq 'balloon
') {
4444 # enable balloon device is not hotpluggable
4445 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4446 # here we reset the ballooning value to memory
4447 my $balloon = $conf->{memory} || $defaults->{memory};
4448 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4449 } elsif ($fast_plug_option->{$opt}) {
4451 } elsif ($opt =~ m/^net(\d+)$/) {
4452 die "skip\n" if !$hotplug_features->{network};
4453 vm_deviceunplug($vmid, $conf, $opt);
4454 } elsif (is_valid_drivename($opt)) {
4455 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4456 vm_deviceunplug($vmid, $conf, $opt);
4457 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4458 } elsif ($opt =~ m/^memory$/) {
4459 die "skip\n" if !$hotplug_features->{memory};
4460 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4461 } elsif ($opt eq 'cpuunits
') {
4462 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4463 } elsif ($opt eq 'cpulimit
') {
4464 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4470 &$add_error($opt, $err) if $err ne "skip\n";
4472 delete $conf->{$opt};
4473 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4477 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4478 $apply_pending_cloudinit = sub {
4479 return if $apply_pending_cloudinit_done; # once is enough
4480 $apply_pending_cloudinit_done = 1; # once is enough
4482 my ($key, $value) = @_;
4484 my @cloudinit_opts = keys %$confdesc_cloudinit;
4485 foreach my $opt (keys %{$conf->{pending}}) {
4486 next if !grep { $_ eq $opt } @cloudinit_opts;
4487 $conf->{$opt} = delete $conf->{pending}->{$opt};
4490 my $new_conf = { %$conf };
4491 $new_conf->{$key} = $value;
4492 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4495 foreach my $opt (keys %{$conf->{pending}}) {
4496 next if $selection && !$selection->{$opt};
4497 my $value = $conf->{pending}->{$opt};
4499 if ($opt eq 'hotplug
') {
4500 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4501 } elsif ($opt eq 'tablet
') {
4502 die "skip\n" if !$hotplug_features->{usb};
4504 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4505 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4506 if $arch eq 'aarch64
';
4507 } elsif ($value == 0) {
4508 vm_deviceunplug($vmid, $conf, 'tablet
');
4509 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4511 } elsif ($opt =~ m/^usb\d+$/) {
4513 # since we cannot reliably hot unplug usb devices
4514 # we are disabling it
4515 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4516 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4517 die "skip\n" if !$d;
4518 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4519 } elsif ($opt eq 'vcpus
') {
4520 die "skip\n" if !$hotplug_features->{cpu};
4521 qemu_cpu_hotplug($vmid, $conf, $value);
4522 } elsif ($opt eq 'balloon
') {
4523 # enable/disable balloning device is not hotpluggable
4524 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4525 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4526 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4528 # allow manual ballooning if shares is set to zero
4529 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4530 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4531 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4533 } elsif ($opt =~ m/^net(\d+)$/) {
4534 # some changes can be done without hotplug
4535 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4536 $vmid, $opt, $value, $arch, $machine_type);
4537 } elsif (is_valid_drivename($opt)) {
4538 die "skip\n" if $opt eq 'efidisk0
';
4539 # some changes can be done without hotplug
4540 my $drive = parse_drive($opt, $value);
4541 if (drive_is_cloudinit($drive)) {
4542 &$apply_pending_cloudinit($opt, $value);
4544 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4545 $vmid, $opt, $value, $arch, $machine_type);
4546 } elsif ($opt =~ m/^memory$/) { #dimms
4547 die "skip\n" if !$hotplug_features->{memory};
4548 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4549 } elsif ($opt eq 'cpuunits
') {
4550 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4551 } elsif ($opt eq 'cpulimit
') {
4552 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4553 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4555 die "skip\n"; # skip non-hot-pluggable options
4559 &$add_error($opt, $err) if $err ne "skip\n";
4561 $conf->{$opt} = $value;
4562 delete $conf->{pending}->{$opt};
4566 PVE::QemuConfig->write_config($vmid, $conf);
4569 sub try_deallocate_drive {
4570 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4572 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4573 my $volid = $drive->{file};
4574 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4575 my $sid = PVE::Storage::parse_volume_id($volid);
4576 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4578 # check if the disk is really unused
4579 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4580 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4581 PVE::Storage::vdisk_free($storecfg, $volid);
4584 # If vm is not owner of this disk remove from config
4592 sub vmconfig_delete_or_detach_drive {
4593 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4595 my $drive = parse_drive($opt, $conf->{$opt});
4597 my $rpcenv = PVE::RPCEnvironment::get();
4598 my $authuser = $rpcenv->get_user();
4601 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4602 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4604 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4610 sub vmconfig_apply_pending {
4611 my ($vmid, $conf, $storecfg, $errors) = @_;
4613 my $add_apply_error = sub {
4614 my ($opt, $msg) = @_;
4615 my $err_msg = "unable to apply pending change $opt : $msg";
4616 $errors->{$opt} = $err_msg;
4622 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4623 foreach my $opt (sort keys %$pending_delete_hash) {
4624 my $force = $pending_delete_hash->{$opt}->{force};
4626 if ($opt =~ m/^unused/) {
4627 die "internal error";
4628 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4629 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4633 $add_apply_error->($opt, $err);
4635 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4636 delete $conf->{$opt};
4640 PVE::QemuConfig->cleanup_pending($conf);
4642 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4643 next if $opt eq 'delete'; # just to be sure
4645 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4646 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4650 $add_apply_error->($opt, $err);
4652 $conf->{$opt} = delete $conf->{pending}->{$opt};
4656 # write all changes at once to avoid unnecessary i/o
4657 PVE::QemuConfig->write_config($vmid, $conf);
4660 sub vmconfig_update_net {
4661 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4663 my $newnet = parse_net($value);
4665 if ($conf->{$opt}) {
4666 my $oldnet = parse_net($conf->{$opt});
4668 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4669 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4670 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4671 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4673 # for non online change, we try to hot-unplug
4674 die "skip\n" if !$hotplug;
4675 vm_deviceunplug($vmid, $conf, $opt);
4678 die "internal error" if $opt !~ m/net(\d+)/;
4679 my $iface = "tap${vmid}i$1";
4681 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4682 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4683 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4684 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4685 PVE::Network::tap_unplug($iface);
4688 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4690 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4692 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4693 # Rate can be applied on its own but any change above needs to
4694 # include the rate in tap_plug since OVS resets everything.
4695 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4698 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4699 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4707 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4713 sub vmconfig_update_disk {
4714 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4716 my $drive = parse_drive($opt, $value);
4718 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4719 my $media = $drive->{media} || 'disk
';
4720 my $oldmedia = $old_drive->{media} || 'disk
';
4721 die "unable to change media type\n" if $media ne $oldmedia;
4723 if (!drive_is_cdrom($old_drive)) {
4725 if ($drive->{file} ne $old_drive->{file}) {
4727 die "skip\n" if !$hotplug;
4729 # unplug and register as unused
4730 vm_deviceunplug($vmid, $conf, $opt);
4731 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4734 # update existing disk
4736 # skip non hotpluggable value
4737 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4738 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4739 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4740 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4741 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4746 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4747 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4748 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4749 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4750 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4751 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4752 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4753 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4754 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4755 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4756 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4757 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4758 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4759 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4760 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4761 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4762 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4763 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4765 qemu_block_set_io_throttle(
4767 ($drive->{mbps} || 0)*1024*1024,
4768 ($drive->{mbps_rd} || 0)*1024*1024,
4769 ($drive->{mbps_wr} || 0)*1024*1024,
4770 $drive->{iops} || 0,
4771 $drive->{iops_rd} || 0,
4772 $drive->{iops_wr} || 0,
4773 ($drive->{mbps_max} || 0)*1024*1024,
4774 ($drive->{mbps_rd_max} || 0)*1024*1024,
4775 ($drive->{mbps_wr_max} || 0)*1024*1024,
4776 $drive->{iops_max} || 0,
4777 $drive->{iops_rd_max} || 0,
4778 $drive->{iops_wr_max} || 0,
4779 $drive->{bps_max_length} || 1,
4780 $drive->{bps_rd_max_length} || 1,
4781 $drive->{bps_wr_max_length} || 1,
4782 $drive->{iops_max_length} || 1,
4783 $drive->{iops_rd_max_length} || 1,
4784 $drive->{iops_wr_max_length} || 1,
4794 if ($drive->{file} eq 'none
') {
4795 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4796 if (drive_is_cloudinit($old_drive)) {
4797 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4800 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4802 # force eject if locked
4803 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4806 mon_cmd($vmid, "blockdev-change-medium",
4807 id => "$opt", filename => "$path");
4815 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4817 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4818 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4821 # called in locked context by incoming migration
4822 sub vm_migrate_get_nbd_disks {
4823 my ($storecfg, $conf, $replicated_volumes) = @_;
4825 my $local_volumes = {};
4826 PVE::QemuConfig->foreach_volume($conf, sub {
4827 my ($ds, $drive) = @_;
4829 return if drive_is_cdrom($drive);
4831 my $volid = $drive->{file};
4835 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4837 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4838 return if $scfg->{shared};
4840 # replicated disks re-use existing state via bitmap
4841 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4842 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4844 return $local_volumes;
4847 # called in locked context by incoming migration
4848 sub vm_migrate_alloc_nbd_disks {
4849 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4854 foreach my $opt (sort keys %$source_volumes) {
4855 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4857 if ($use_existing) {
4858 $nbd->{$opt}->{drivestr} = print_drive($drive);
4859 $nbd->{$opt}->{volid} = $volid;
4860 $nbd->{$opt}->{replicated} = 1;
4864 # If a remote storage is specified and the format of the original
4865 # volume is not available there, fall back to the default format.
4866 # Otherwise use the same format as the original.
4867 if (!$storagemap->{identity}) {
4868 $storeid = map_storage($storagemap, $storeid);
4869 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4870 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4871 my $fileFormat = qemu_img_format($scfg, $volname);
4872 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4874 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4875 $format = qemu_img_format($scfg, $volname);
4878 my $size = $drive->{size} / 1024;
4879 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4880 my $newdrive = $drive;
4881 $newdrive->{format} = $format;
4882 $newdrive->{file} = $newvolid;
4883 my $drivestr = print_drive($newdrive);
4884 $nbd->{$opt}->{drivestr} = $drivestr;
4885 $nbd->{$opt}->{volid} = $newvolid;
4891 # see vm_start_nolock for parameters, additionally:
4893 # storagemap = parsed storage map for allocating NBD disks
4895 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4897 return PVE::QemuConfig->lock_config($vmid, sub {
4898 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4900 die "you can't start a vm
if it
's a template\n"
4901 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4903 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended
');
4905 PVE::QemuConfig->check_lock($conf)
4906 if !($params->{skiplock} || $has_suspended_lock);
4908 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4910 die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4912 if (my $storagemap = $migrate_opts->{storagemap}) {
4913 my $replicated = $migrate_opts->{replicated_volumes};
4914 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4915 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4917 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4918 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4922 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4928 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
4929 # skiplock => 0/1, skip checking for config lock
4930 # skiptemplate => 0/1, skip checking whether VM is template
4931 # forcemachine => to force Qemu machine (rollback/migration)
4932 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
4933 # timeout => in seconds
4934 # paused => start VM in paused state (backup)
4935 # resume => resume from hibernation
4937 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4938 # migratedfrom => source node
4939 # spice_ticket => used for spice migration, passed via tunnel/stdin
4940 # network => CIDR of migration network
4941 # type => secure/insecure - tunnel over encrypted connection or plain-text
4942 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
4943 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4944 sub vm_start_nolock {
4945 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4947 my $statefile = $params->{statefile};
4948 my $resume = $params->{resume};
4950 my $migratedfrom = $migrate_opts->{migratedfrom};
4951 my $migration_type = $migrate_opts->{type};
4955 # clean up leftover reboot request files
4956 eval { clear_reboot_request($vmid); };
4959 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4960 vmconfig_apply_pending($vmid, $conf, $storecfg);
4961 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4964 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4966 my $defaults = load_defaults();
4968 # set environment variable useful inside network script
4969 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4971 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
4973 my $forcemachine = $params->{forcemachine};
4974 my $forcecpu = $params->{forcecpu};
4976 # enforce machine and CPU type on suspended vm to ensure HW compatibility
4977 $forcemachine = $conf->{runningmachine};
4978 $forcecpu = $conf->{runningcpu};
4979 print "Resuming suspended VM\n";
4982 my ($cmd, $vollist, $spice_port) =
4983 config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4986 my $get_migration_ip = sub {
4987 my ($nodename) = @_;
4989 return $migration_ip if defined($migration_ip);
4991 my $cidr = $migrate_opts->{network};
4993 if (!defined($cidr)) {
4994 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
4995 $cidr = $dc_conf->{migration}->{network};
4998 if (defined($cidr)) {
4999 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5001 die "could not get IP: no address configured on local " .
5002 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5004 die "could not get IP: multiple addresses configured on local " .
5005 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5007 $migration_ip = @$ips[0];
5010 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5011 if !defined($migration_ip);
5013 return $migration_ip;
5018 if ($statefile eq 'tcp
') {
5019 my $localip = "localhost";
5020 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5021 my $nodename = nodename();
5023 if (!defined($migration_type)) {
5024 if (defined($datacenterconf->{migration}->{type})) {
5025 $migration_type = $datacenterconf->{migration}->{type};
5027 $migration_type = 'secure
';
5031 if ($migration_type eq 'insecure
') {
5032 $localip = $get_migration_ip->($nodename);
5033 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5036 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5037 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5038 $migrate_uri = "tcp:${localip}:${migrate_port}";
5039 push @$cmd, '-incoming
', $migrate_uri;
5042 } elsif ($statefile eq 'unix
') {
5043 # should be default for secure migrations as a ssh TCP forward
5044 # tunnel is not deterministic reliable ready and fails regurarly
5045 # to set up in time, so use UNIX socket forwards
5046 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5047 unlink $socket_addr;
5049 $migrate_uri = "unix:$socket_addr";
5051 push @$cmd, '-incoming
', $migrate_uri;
5054 } elsif (-e $statefile) {
5055 push @$cmd, '-loadstate
', $statefile;
5057 my $statepath = PVE::Storage::path($storecfg, $statefile);
5058 push @$vollist, $statefile;
5059 push @$cmd, '-loadstate
', $statepath;
5061 } elsif ($params->{paused}) {
5066 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
5067 my $d = parse_hostpci($conf->{"hostpci$i"});
5069 my $pcidevices = $d->{pciid};
5070 foreach my $pcidevice (@$pcidevices) {
5071 my $pciid = $pcidevice->{id};
5073 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5074 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5075 die "no pci device info for device '$pciid'\n" if !$info;
5078 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5079 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5081 die "can't unbind
/bind pci group to vfio
'$pciid'\n"
5082 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5083 die "can
't reset pci device '$pciid'\n"
5084 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5089 PVE::Storage::activate_volumes($storecfg, $vollist);
5092 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5093 outfunc => sub {}, errfunc => sub {});
5095 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5096 # timeout should be more than enough here...
5097 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5099 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5100 : $defaults->{cpuunits};
5102 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5104 timeout => $statefile ? undef : $start_timeout,
5109 # when migrating, prefix QEMU output so other side can pick up any
5110 # errors that might occur and show the user
5111 if ($migratedfrom) {
5112 $run_params{quiet} = 1;
5113 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5117 Slice => 'qemu
.slice
',
5119 CPUShares => $cpuunits
5122 if (my $cpulimit = $conf->{cpulimit}) {
5123 $properties{CPUQuota} = int($cpulimit * 100);
5125 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5127 my $run_qemu = sub {
5128 PVE::Tools::run_fork sub {
5129 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5131 my $exitcode = run_command($cmd, %run_params);
5132 die "QEMU exited with code $exitcode\n" if $exitcode;
5136 if ($conf->{hugepages}) {
5139 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5140 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5142 PVE::QemuServer::Memory::hugepages_mount();
5143 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5145 eval { $run_qemu->() };
5147 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5148 if !$conf->{keephugepages};
5152 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5153 if !$conf->{keephugepages};
5155 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5158 eval { $run_qemu->() };
5162 # deactivate volumes if start fails
5163 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5164 die "start failed: $err";
5167 print "migration listens on $migrate_uri\n" if $migrate_uri;
5168 $res->{migrate_uri} = $migrate_uri;
5170 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5171 eval { mon_cmd($vmid, "cont"); };
5175 #start nbd server for storage migration
5176 if (my $nbd = $migrate_opts->{nbd}) {
5177 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5179 my $migrate_storage_uri;
5180 # nbd_protocol_version > 0 for unix socket support
5181 if ($nbd_protocol_version > 0 && $migration_type eq 'secure
') {
5182 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5183 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix
', data => { path => $socket_path } } );
5184 $migrate_storage_uri = "nbd:unix:$socket_path";
5186 my $nodename = nodename();
5187 my $localip = $get_migration_ip->($nodename);
5188 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5189 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5191 mon_cmd($vmid, "nbd-server-start", addr => {
5194 host => "${localip}",
5195 port => "${storage_migrate_port}",
5198 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5199 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5202 $res->{migrate_storage_uri} = $migrate_storage_uri;
5204 foreach my $opt (sort keys %$nbd) {
5205 my $drivestr = $nbd->{$opt}->{drivestr};
5206 my $volid = $nbd->{$opt}->{volid};
5207 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5208 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5209 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5210 print "re-using replicated volume: $opt - $volid\n"
5211 if $nbd->{$opt}->{replicated};
5213 $res->{drives}->{$opt} = $nbd->{$opt};
5214 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5218 if ($migratedfrom) {
5220 set_migration_caps($vmid);
5225 print "spice listens on port $spice_port\n";
5226 $res->{spice_port} = $spice_port;
5227 if ($migrate_opts->{spice_ticket}) {
5228 mon_cmd($vmid, "set_password", protocol => 'spice
', password =>
5229 $migrate_opts->{spice_ticket});
5230 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5235 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5236 if !$statefile && $conf->{balloon};
5238 foreach my $opt (keys %$conf) {
5239 next if $opt !~ m/^net\d+$/;
5240 my $nicconf = parse_net($conf->{$opt});
5241 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5245 mon_cmd($vmid, 'qom-set
',
5246 path => "machine/peripheral/balloon0",
5247 property => "guest-stats-polling-interval",
5248 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5251 print "Resumed VM, removing state\n";
5252 if (my $vmstate = $conf->{vmstate}) {
5253 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5254 PVE::Storage::vdisk_free($storecfg, $vmstate);
5256 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5257 PVE
::QemuConfig-
>write_config($vmid, $conf);
5260 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5265 sub vm_commandline
{
5266 my ($storecfg, $vmid, $snapname) = @_;
5268 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5273 my $snapshot = $conf->{snapshots
}->{$snapname};
5274 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5276 # check for machine or CPU overrides in snapshot
5277 $forcemachine = $snapshot->{runningmachine
};
5278 $forcecpu = $snapshot->{runningcpu
};
5280 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5285 my $defaults = load_defaults
();
5287 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults,
5288 $forcemachine, $forcecpu);
5290 return PVE
::Tools
::cmd2string
($cmd);
5294 my ($vmid, $skiplock) = @_;
5296 PVE
::QemuConfig-
>lock_config($vmid, sub {
5298 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5300 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5302 mon_cmd
($vmid, "system_reset");
5306 sub get_vm_volumes
{
5310 foreach_volid
($conf, sub {
5311 my ($volid, $attr) = @_;
5313 return if $volid =~ m
|^/|;
5315 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5318 push @$vollist, $volid;
5324 sub vm_stop_cleanup
{
5325 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5330 my $vollist = get_vm_volumes
($conf);
5331 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5334 foreach my $ext (qw(mon qmp pid vnc qga)) {
5335 unlink "/var/run/qemu-server/${vmid}.$ext";
5338 if ($conf->{ivshmem
}) {
5339 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5340 # just delete it for now, VMs which have this already open do not
5341 # are affected, but new VMs will get a separated one. If this
5342 # becomes an issue we either add some sort of ref-counting or just
5343 # add a "don't delete on stop" flag to the ivshmem format.
5344 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5347 foreach my $key (keys %$conf) {
5348 next if $key !~ m/^hostpci(\d+)$/;
5349 my $hostpciindex = $1;
5350 my $d = parse_hostpci
($conf->{$key});
5351 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5353 foreach my $pci (@{$d->{pciid
}}) {
5354 my $pciid = $pci->{id
};
5355 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5359 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5361 warn $@ if $@; # avoid errors - just warn
5364 # call only in locked context
5366 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5368 my $pid = check_running
($vmid, $nocheck);
5373 $conf = PVE
::QemuConfig-
>load_config($vmid);
5374 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5375 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5376 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5377 $timeout = $opts->{down
} if $opts->{down
};
5379 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5384 if (defined($conf) && parse_guest_agent
($conf)->{enabled
}) {
5385 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5387 mon_cmd
($vmid, "system_powerdown");
5390 mon_cmd
($vmid, "quit");
5396 $timeout = 60 if !defined($timeout);
5399 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5404 if ($count >= $timeout) {
5406 warn "VM still running - terminating now with SIGTERM\n";
5409 die "VM quit/powerdown failed - got timeout\n";
5412 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5416 if (!check_running
($vmid, $nocheck)) {
5417 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5421 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5424 die "VM quit/powerdown failed\n";
5432 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5437 if ($count >= $timeout) {
5438 warn "VM still running - terminating now with SIGKILL\n";
5443 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5446 # Note: use $nocheck to skip tests if VM configuration file exists.
5447 # We need that when migration VMs to other nodes (files already moved)
5448 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5450 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5452 $force = 1 if !defined($force) && !$shutdown;
5455 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5456 kill 15, $pid if $pid;
5457 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5458 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5462 PVE
::QemuConfig-
>lock_config($vmid, sub {
5463 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5468 my ($vmid, $timeout) = @_;
5470 PVE
::QemuConfig-
>lock_config($vmid, sub {
5473 # only reboot if running, as qmeventd starts it again on a stop event
5474 return if !check_running
($vmid);
5476 create_reboot_request
($vmid);
5478 my $storecfg = PVE
::Storage
::config
();
5479 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5483 # avoid that the next normal shutdown will be confused for a reboot
5484 clear_reboot_request
($vmid);
5490 # note: if using the statestorage parameter, the caller has to check privileges
5492 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5499 PVE
::QemuConfig-
>lock_config($vmid, sub {
5501 $conf = PVE
::QemuConfig-
>load_config($vmid);
5503 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5504 PVE
::QemuConfig-
>check_lock($conf)
5505 if !($skiplock || $is_backing_up);
5507 die "cannot suspend to disk during backup\n"
5508 if $is_backing_up && $includestate;
5510 if ($includestate) {
5511 $conf->{lock} = 'suspending';
5512 my $date = strftime
("%Y-%m-%d", localtime(time()));
5513 $storecfg = PVE
::Storage
::config
();
5514 if (!$statestorage) {
5515 $statestorage = find_vmstate_storage
($conf, $storecfg);
5516 # check permissions for the storage
5517 my $rpcenv = PVE
::RPCEnvironment
::get
();
5518 if ($rpcenv->{type
} ne 'cli') {
5519 my $authuser = $rpcenv->get_user();
5520 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5525 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
5526 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5527 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5528 PVE
::QemuConfig-
>write_config($vmid, $conf);
5530 mon_cmd
($vmid, "stop");
5534 if ($includestate) {
5536 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5539 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5541 my $state = mon_cmd
($vmid, "query-savevm");
5542 if (!$state->{status
}) {
5543 die "savevm not active\n";
5544 } elsif ($state->{status
} eq 'active') {
5547 } elsif ($state->{status
} eq 'completed') {
5548 print "State saved, quitting\n";
5550 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5551 die "query-savevm failed with error '$state->{error}'\n"
5553 die "query-savevm returned status '$state->{status}'\n";
5559 PVE
::QemuConfig-
>lock_config($vmid, sub {
5560 $conf = PVE
::QemuConfig-
>load_config($vmid);
5562 # cleanup, but leave suspending lock, to indicate something went wrong
5564 mon_cmd
($vmid, "savevm-end");
5565 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5566 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5567 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5568 PVE
::QemuConfig-
>write_config($vmid, $conf);
5574 die "lock changed unexpectedly\n"
5575 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5577 mon_cmd
($vmid, "quit");
5578 $conf->{lock} = 'suspended';
5579 PVE
::QemuConfig-
>write_config($vmid, $conf);
5585 my ($vmid, $skiplock, $nocheck) = @_;
5587 PVE
::QemuConfig-
>lock_config($vmid, sub {
5588 my $res = mon_cmd
($vmid, 'query-status');
5589 my $resume_cmd = 'cont';
5591 if ($res->{status
} && $res->{status
} eq 'suspended') {
5592 $resume_cmd = 'system_wakeup';
5597 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5599 PVE
::QemuConfig-
>check_lock($conf)
5600 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5603 mon_cmd
($vmid, $resume_cmd);
5608 my ($vmid, $skiplock, $key) = @_;
5610 PVE
::QemuConfig-
>lock_config($vmid, sub {
5612 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5614 # there is no qmp command, so we use the human monitor command
5615 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5616 die $res if $res ne '';
5620 # vzdump restore implementaion
5622 sub tar_archive_read_firstfile
{
5623 my $archive = shift;
5625 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5627 # try to detect archive type first
5628 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5629 die "unable to open file '$archive'\n";
5630 my $firstfile = <$fh>;
5634 die "ERROR: archive contaions no data\n" if !$firstfile;
5640 sub tar_restore_cleanup
{
5641 my ($storecfg, $statfile) = @_;
5643 print STDERR
"starting cleanup\n";
5645 if (my $fd = IO
::File-
>new($statfile, "r")) {
5646 while (defined(my $line = <$fd>)) {
5647 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5650 if ($volid =~ m
|^/|) {
5651 unlink $volid || die 'unlink failed\n';
5653 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5655 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5657 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5659 print STDERR
"unable to parse line in statfile - $line";
5666 sub restore_file_archive
{
5667 my ($archive, $vmid, $user, $opts) = @_;
5669 return restore_vma_archive
($archive, $vmid, $user, $opts)
5672 my $info = PVE
::Storage
::archive_info
($archive);
5673 my $format = $opts->{format
} // $info->{format
};
5674 my $comp = $info->{compression
};
5676 # try to detect archive format
5677 if ($format eq 'tar') {
5678 return restore_tar_archive
($archive, $vmid, $user, $opts);
5680 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5684 # hepler to remove disks that will not be used after restore
5685 my $restore_cleanup_oldconf = sub {
5686 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5688 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
5689 my ($ds, $drive) = @_;
5691 return if drive_is_cdrom
($drive, 1);
5693 my $volid = $drive->{file
};
5694 return if !$volid || $volid =~ m
|^/|;
5696 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
5697 return if !$path || !$owner || ($owner != $vmid);
5699 # Note: only delete disk we want to restore
5700 # other volumes will become unused
5701 if ($virtdev_hash->{$ds}) {
5702 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
5709 # delete vmstate files, after the restore we have no snapshots anymore
5710 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5711 my $snap = $oldconf->{snapshots
}->{$snapname};
5712 if ($snap->{vmstate
}) {
5713 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
5721 # Helper to parse vzdump backup device hints
5723 # $rpcenv: Environment, used to ckeck storage permissions
5724 # $user: User ID, to check storage permissions
5725 # $storecfg: Storage configuration
5726 # $fh: the file handle for reading the configuration
5727 # $devinfo: should contain device sizes for all backu-up'ed devices
5728 # $options: backup options (pool, default storage)
5730 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5731 my $parse_backup_hints = sub {
5732 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5734 my $virtdev_hash = {};
5736 while (defined(my $line = <$fh>)) {
5737 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5738 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5739 die "archive does not contain data for drive '$virtdev'\n"
5740 if !$devinfo->{$devname};
5742 if (defined($options->{storage
})) {
5743 $storeid = $options->{storage
} || 'local';
5744 } elsif (!$storeid) {
5747 $format = 'raw' if !$format;
5748 $devinfo->{$devname}->{devname
} = $devname;
5749 $devinfo->{$devname}->{virtdev
} = $virtdev;
5750 $devinfo->{$devname}->{format
} = $format;
5751 $devinfo->{$devname}->{storeid
} = $storeid;
5753 # check permission on storage
5754 my $pool = $options->{pool
}; # todo: do we need that?
5755 if ($user ne 'root@pam') {
5756 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5759 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5760 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5762 my $drive = parse_drive
($virtdev, $2);
5763 if (drive_is_cloudinit
($drive)) {
5764 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
5765 $storeid = $options->{storage
} if defined ($options->{storage
});
5766 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5767 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
5769 $virtdev_hash->{$virtdev} = {
5771 storeid
=> $storeid,
5772 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
5779 return $virtdev_hash;
5782 # Helper to allocate and activate all volumes required for a restore
5784 # $storecfg: Storage configuration
5785 # $virtdev_hash: as returned by parse_backup_hints()
5787 # Returns: { $virtdev => $volid }
5788 my $restore_allocate_devices = sub {
5789 my ($storecfg, $virtdev_hash, $vmid) = @_;
5792 foreach my $virtdev (sort keys %$virtdev_hash) {
5793 my $d = $virtdev_hash->{$virtdev};
5794 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5795 my $storeid = $d->{storeid
};
5796 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5798 # test if requested format is supported
5799 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5800 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
5801 $d->{format
} = $defFormat if !$supported;
5804 if ($d->{is_cloudinit
}) {
5805 $name = "vm-$vmid-cloudinit";
5806 $name .= ".$d->{format}" if $d->{format
} ne 'raw';
5809 my $volid = PVE
::Storage
::vdisk_alloc
(
5810 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
5812 print STDERR
"new volume ID is '$volid'\n";
5813 $d->{volid
} = $volid;
5815 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
5817 $map->{$virtdev} = $volid;
5823 my $restore_update_config_line = sub {
5824 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5826 return if $line =~ m/^\#qmdump\#/;
5827 return if $line =~ m/^\#vzdump\#/;
5828 return if $line =~ m/^lock:/;
5829 return if $line =~ m/^unused\d+:/;
5830 return if $line =~ m/^parent:/;
5832 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5833 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5834 # try to convert old 1.X settings
5835 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5836 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5837 my ($model, $macaddr) = split(/\=/, $devconfig);
5838 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5841 bridge
=> "vmbr$ind",
5842 macaddr
=> $macaddr,
5844 my $netstr = print_net
($net);
5846 print $outfd "net$cookie->{netcount}: $netstr\n";
5847 $cookie->{netcount
}++;
5849 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5850 my ($id, $netstr) = ($1, $2);
5851 my $net = parse_net
($netstr);
5852 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5853 $netstr = print_net
($net);
5854 print $outfd "$id: $netstr\n";
5855 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5858 my $di = parse_drive
($virtdev, $value);
5859 if (defined($di->{backup
}) && !$di->{backup
}) {
5860 print $outfd "#$line";
5861 } elsif ($map->{$virtdev}) {
5862 delete $di->{format
}; # format can change on restore
5863 $di->{file
} = $map->{$virtdev};
5864 $value = print_drive
($di);
5865 print $outfd "$virtdev: $value\n";
5869 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5871 if ($vmgenid ne '0') {
5872 # always generate a new vmgenid if there was a valid one setup
5873 $vmgenid = generate_uuid
();
5875 print $outfd "vmgenid: $vmgenid\n";
5876 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5877 my ($uuid, $uuid_str);
5878 UUID
::generate
($uuid);
5879 UUID
::unparse
($uuid, $uuid_str);
5880 my $smbios1 = parse_smbios1
($2);
5881 $smbios1->{uuid
} = $uuid_str;
5882 print $outfd $1.print_smbios1
($smbios1)."\n";
5888 my $restore_deactivate_volumes = sub {
5889 my ($storecfg, $devinfo) = @_;
5892 foreach my $devname (keys %$devinfo) {
5893 my $volid = $devinfo->{$devname}->{volid
};
5894 push @$vollist, $volid if $volid;
5897 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5900 my $restore_destroy_volumes = sub {
5901 my ($storecfg, $devinfo) = @_;
5903 foreach my $devname (keys %$devinfo) {
5904 my $volid = $devinfo->{$devname}->{volid
};
5907 if ($volid =~ m
|^/|) {
5908 unlink $volid || die 'unlink failed\n';
5910 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5912 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5914 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5919 my ($cfg, $vmid) = @_;
5921 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5923 my $volid_hash = {};
5924 foreach my $storeid (keys %$info) {
5925 foreach my $item (@{$info->{$storeid}}) {
5926 next if !($item->{volid
} && $item->{size
});
5927 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5928 $volid_hash->{$item->{volid
}} = $item;
5935 sub update_disk_config
{
5936 my ($vmid, $conf, $volid_hash) = @_;
5939 my $prefix = "VM $vmid";
5941 # used and unused disks
5942 my $referenced = {};
5944 # Note: it is allowed to define multiple storages with same path (alias), so
5945 # we need to check both 'volid' and real 'path' (two different volid can point
5946 # to the same path).
5948 my $referencedpath = {};
5951 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5952 my ($opt, $drive) = @_;
5954 my $volid = $drive->{file
};
5956 my $volume = $volid_hash->{$volid};
5958 # mark volid as "in-use" for next step
5959 $referenced->{$volid} = 1;
5960 if ($volume && (my $path = $volume->{path
})) {
5961 $referencedpath->{$path} = 1;
5964 return if drive_is_cdrom
($drive);
5967 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
5968 if (defined($updated)) {
5970 $conf->{$opt} = print_drive
($updated);
5971 print "$prefix ($opt): $msg\n";
5975 # remove 'unusedX' entry if volume is used
5976 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
5977 my ($opt, $drive) = @_;
5979 my $volid = $drive->{file
};
5982 my $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
5983 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5984 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5986 delete $conf->{$opt};
5989 $referenced->{$volid} = 1;
5990 $referencedpath->{$path} = 1 if $path;
5993 foreach my $volid (sort keys %$volid_hash) {
5994 next if $volid =~ m/vm-$vmid-state-/;
5995 next if $referenced->{$volid};
5996 my $path = $volid_hash->{$volid}->{path
};
5997 next if !$path; # just to be sure
5998 next if $referencedpath->{$path};
6000 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6001 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6002 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6009 my ($vmid, $nolock, $dryrun) = @_;
6011 my $cfg = PVE
::Storage
::config
();
6013 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6014 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6015 foreach my $stor (keys %{$cfg->{ids
}}) {
6016 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
6019 print "rescan volumes...\n";
6020 my $volid_hash = scan_volids
($cfg, $vmid);
6022 my $updatefn = sub {
6025 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6027 PVE
::QemuConfig-
>check_lock($conf);
6030 foreach my $volid (keys %$volid_hash) {
6031 my $info = $volid_hash->{$volid};
6032 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6035 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6037 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6040 if (defined($vmid)) {
6044 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6047 my $vmlist = config_list
();
6048 foreach my $vmid (keys %$vmlist) {
6052 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6058 sub restore_proxmox_backup_archive
{
6059 my ($archive, $vmid, $user, $options) = @_;
6061 my $storecfg = PVE
::Storage
::config
();
6063 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6064 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6066 my $server = $scfg->{server
};
6067 my $datastore = $scfg->{datastore
};
6068 my $username = $scfg->{username
} // 'root@pam';
6069 my $fingerprint = $scfg->{fingerprint
};
6070 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6072 my $repo = "$username\@$server:$datastore";
6074 # This is only used for `pbs-restore`!
6075 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6076 local $ENV{PBS_PASSWORD
} = $password;
6077 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6079 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6080 PVE
::Storage
::parse_volname
($storecfg, $archive);
6082 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6084 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6086 my $tmpdir = "/var/tmp/vzdumptmp$$";
6090 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6091 my $tmpfn = "$conffile.$$.tmp";
6092 # disable interrupts (always do cleanups)
6096 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6098 # Note: $oldconf is undef if VM does not exists
6099 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6100 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6102 my $rpcenv = PVE
::RPCEnvironment
::get
();
6111 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6113 my $cfgfn = "$tmpdir/qemu-server.conf";
6114 my $firewall_config_fn = "$tmpdir/fw.conf";
6115 my $index_fn = "$tmpdir/index.json";
6117 my $cmd = "restore";
6119 my $param = [$pbs_backup_name, "index.json", $index_fn];
6120 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6121 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6122 $index = decode_json
($index);
6124 # print Dumper($index);
6125 foreach my $info (@{$index->{files
}}) {
6126 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6128 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6129 $devinfo->{$devname}->{size
} = $1;
6131 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6136 my $is_qemu_server_backup = scalar(
6137 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6139 if (!$is_qemu_server_backup) {
6140 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6142 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6144 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6145 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6147 if ($has_firewall_config) {
6148 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6149 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6151 my $pve_firewall_dir = '/etc/pve/firewall';
6152 mkdir $pve_firewall_dir; # make sure the dir exists
6153 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6156 my $fh = IO
::File-
>new($cfgfn, "r") ||
6157 die "unable to read qemu-server.conf - $!\n";
6159 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6161 # fixme: rate limit?
6163 # create empty/temp config
6164 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6166 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6169 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6171 foreach my $virtdev (sort keys %$virtdev_hash) {
6172 my $d = $virtdev_hash->{$virtdev};
6173 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6175 my $volid = $d->{volid
};
6177 my $path = PVE
::Storage
::path
($storecfg, $volid);
6179 # This is the ONLY user of the PBS_ env vars set on top of this function!
6180 my $pbs_restore_cmd = [
6181 '/usr/bin/pbs-restore',
6182 '--repository', $repo,
6184 "$d->{devname}.img.fidx",
6189 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6190 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6192 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6193 push @$pbs_restore_cmd, '--skip-zero';
6196 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6197 print "restore proxmox backup image: $dbg_cmdstring\n";
6198 run_command
($pbs_restore_cmd);
6201 $fh->seek(0, 0) || die "seek failed - $!\n";
6203 my $outfd = new IO
::File
($tmpfn, "w") ||
6204 die "unable to write config for VM $vmid\n";
6206 my $cookie = { netcount
=> 0 };
6207 while (defined(my $line = <$fh>)) {
6208 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique
});
6216 $restore_deactivate_volumes->($storecfg, $devinfo);
6222 $restore_destroy_volumes->($storecfg, $devinfo);
6226 rename($tmpfn, $conffile) ||
6227 die "unable to commit configuration file '$conffile'\n";
6229 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6231 eval { rescan
($vmid, 1); };
6235 sub restore_vma_archive
{
6236 my ($archive, $vmid, $user, $opts, $comp) = @_;
6238 my $readfrom = $archive;
6240 my $cfg = PVE
::Storage
::config
();
6242 my $bwlimit = $opts->{bwlimit
};
6244 my $dbg_cmdstring = '';
6245 my $add_pipe = sub {
6247 push @$commands, $cmd;
6248 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6249 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6254 if ($archive eq '-') {
6257 # If we use a backup from a PVE defined storage we also consider that
6258 # storage's rate limit:
6259 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6260 if (defined($volid)) {
6261 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6262 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6264 print STDERR
"applying read rate limit: $readlimit\n";
6265 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6266 $add_pipe->($cstream);
6272 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6273 my $cmd = $info->{decompressor
};
6274 push @$cmd, $readfrom;
6278 my $tmpdir = "/var/tmp/vzdumptmp$$";
6281 # disable interrupts (always do cleanups)
6285 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6287 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6288 POSIX
::mkfifo
($mapfifo, 0600);
6291 my $openfifo = sub {
6292 open($fifofh, '>', $mapfifo) || die $!;
6295 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6302 my $rpcenv = PVE
::RPCEnvironment
::get
();
6304 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6305 my $tmpfn = "$conffile.$$.tmp";
6307 # Note: $oldconf is undef if VM does not exist
6308 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6309 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6313 my $print_devmap = sub {
6314 my $cfgfn = "$tmpdir/qemu-server.conf";
6316 # we can read the config - that is already extracted
6317 my $fh = IO
::File-
>new($cfgfn, "r") ||
6318 die "unable to read qemu-server.conf - $!\n";
6320 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6322 my $pve_firewall_dir = '/etc/pve/firewall';
6323 mkdir $pve_firewall_dir; # make sure the dir exists
6324 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6327 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6329 foreach my $key (keys %storage_limits) {
6330 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6332 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6333 $storage_limits{$key} = $limit * 1024;
6336 foreach my $devname (keys %$devinfo) {
6337 die "found no device mapping information for device '$devname'\n"
6338 if !$devinfo->{$devname}->{virtdev
};
6341 # create empty/temp config
6343 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6344 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6348 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6350 # print restore information to $fifofh
6351 foreach my $virtdev (sort keys %$virtdev_hash) {
6352 my $d = $virtdev_hash->{$virtdev};
6353 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6355 my $storeid = $d->{storeid
};
6356 my $volid = $d->{volid
};
6359 if (my $limit = $storage_limits{$storeid}) {
6360 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6363 my $write_zeros = 1;
6364 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6368 my $path = PVE
::Storage
::path
($cfg, $volid);
6370 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6372 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6375 $fh->seek(0, 0) || die "seek failed - $!\n";
6377 my $outfd = new IO
::File
($tmpfn, "w") ||
6378 die "unable to write config for VM $vmid\n";
6380 my $cookie = { netcount
=> 0 };
6381 while (defined(my $line = <$fh>)) {
6382 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6395 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6396 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6398 $oldtimeout = alarm($timeout);
6405 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6406 my ($dev_id, $size, $devname) = ($1, $2, $3);
6407 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6408 } elsif ($line =~ m/^CTIME: /) {
6409 # we correctly received the vma config, so we can disable
6410 # the timeout now for disk allocation (set to 10 minutes, so
6411 # that we always timeout if something goes wrong)
6414 print $fifofh "done\n";
6415 my $tmp = $oldtimeout || 0;
6416 $oldtimeout = undef;
6422 print "restore vma archive: $dbg_cmdstring\n";
6423 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6427 alarm($oldtimeout) if $oldtimeout;
6429 $restore_deactivate_volumes->($cfg, $devinfo);
6436 $restore_destroy_volumes->($cfg, $devinfo);
6440 rename($tmpfn, $conffile) ||
6441 die "unable to commit configuration file '$conffile'\n";
6443 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6445 eval { rescan
($vmid, 1); };
6449 sub restore_tar_archive
{
6450 my ($archive, $vmid, $user, $opts) = @_;
6452 if ($archive ne '-') {
6453 my $firstfile = tar_archive_read_firstfile
($archive);
6454 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6455 if $firstfile ne 'qemu-server.conf';
6458 my $storecfg = PVE
::Storage
::config
();
6460 # avoid zombie disks when restoring over an existing VM -> cleanup first
6461 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6462 # skiplock=1 because qmrestore has set the 'create' lock itself already
6463 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6464 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6466 my $tocmd = "/usr/lib/qemu-server/qmextract";
6468 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6469 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6470 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6471 $tocmd .= ' --info' if $opts->{info
};
6473 # tar option "xf" does not autodetect compression when read from STDIN,
6474 # so we pipe to zcat
6475 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6476 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6478 my $tmpdir = "/var/tmp/vzdumptmp$$";
6481 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6482 local $ENV{VZDUMP_VMID
} = $vmid;
6483 local $ENV{VZDUMP_USER
} = $user;
6485 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6486 my $tmpfn = "$conffile.$$.tmp";
6488 # disable interrupts (always do cleanups)
6492 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6500 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6502 if ($archive eq '-') {
6503 print "extracting archive from STDIN\n";
6504 run_command
($cmd, input
=> "<&STDIN");
6506 print "extracting archive '$archive'\n";
6510 return if $opts->{info
};
6514 my $statfile = "$tmpdir/qmrestore.stat";
6515 if (my $fd = IO
::File-
>new($statfile, "r")) {
6516 while (defined (my $line = <$fd>)) {
6517 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6518 $map->{$1} = $2 if $1;
6520 print STDERR
"unable to parse line in statfile - $line\n";
6526 my $confsrc = "$tmpdir/qemu-server.conf";
6528 my $srcfd = new IO
::File
($confsrc, "r") ||
6529 die "unable to open file '$confsrc'\n";
6531 my $outfd = new IO
::File
($tmpfn, "w") ||
6532 die "unable to write config for VM $vmid\n";
6534 my $cookie = { netcount
=> 0 };
6535 while (defined (my $line = <$srcfd>)) {
6536 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6544 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6550 rename $tmpfn, $conffile ||
6551 die "unable to commit configuration file '$conffile'\n";
6553 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6555 eval { rescan
($vmid, 1); };
6559 sub foreach_storage_used_by_vm
{
6560 my ($conf, $func) = @_;
6564 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6565 my ($ds, $drive) = @_;
6566 return if drive_is_cdrom
($drive);
6568 my $volid = $drive->{file
};
6570 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6571 $sidhash->{$sid} = $sid if $sid;
6574 foreach my $sid (sort keys %$sidhash) {
6579 my $qemu_snap_storage = {
6582 sub do_snapshots_with_qemu
{
6583 my ($storecfg, $volid) = @_;
6585 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6586 my $scfg = $storecfg->{ids
}->{$storage_name};
6588 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6592 if ($volid =~ m/\.(qcow2|qed)$/){
6599 sub qga_check_running
{
6600 my ($vmid, $nowarn) = @_;
6602 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6604 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6610 sub template_create
{
6611 my ($vmid, $conf, $disk) = @_;
6613 my $storecfg = PVE
::Storage
::config
();
6615 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6616 my ($ds, $drive) = @_;
6618 return if drive_is_cdrom
($drive);
6619 return if $disk && $ds ne $disk;
6621 my $volid = $drive->{file
};
6622 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6624 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6625 $drive->{file
} = $voliddst;
6626 $conf->{$ds} = print_drive
($drive);
6627 PVE
::QemuConfig-
>write_config($vmid, $conf);
6631 sub convert_iscsi_path
{
6634 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6639 my $initiator_name = get_initiator_name
();
6641 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6642 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6645 die "cannot convert iscsi path '$path', unkown format\n";
6648 sub qemu_img_convert
{
6649 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6651 my $storecfg = PVE
::Storage
::config
();
6652 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6653 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6655 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6659 my $src_is_iscsi = 0;
6663 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6664 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6665 $src_format = qemu_img_format
($src_scfg, $src_volname);
6666 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6667 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6668 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6669 } elsif (-f
$src_volid) {
6670 $src_path = $src_volid;
6671 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6676 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6678 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6679 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6680 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6681 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6684 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6685 push @$cmd, '-l', "snapshot.name=$snapname"
6686 if $snapname && $src_format && $src_format eq "qcow2";
6687 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6688 push @$cmd, '-T', $cachemode if defined($cachemode);
6690 if ($src_is_iscsi) {
6691 push @$cmd, '--image-opts';
6692 $src_path = convert_iscsi_path
($src_path);
6693 } elsif ($src_format) {
6694 push @$cmd, '-f', $src_format;
6697 if ($dst_is_iscsi) {
6698 push @$cmd, '--target-image-opts';
6699 $dst_path = convert_iscsi_path
($dst_path);
6701 push @$cmd, '-O', $dst_format;
6704 push @$cmd, $src_path;
6706 if (!$dst_is_iscsi && $is_zero_initialized) {
6707 push @$cmd, "zeroinit:$dst_path";
6709 push @$cmd, $dst_path;
6714 if($line =~ m/\((\S+)\/100\
%\)/){
6716 my $transferred = int($size * $percent / 100);
6717 my $remaining = $size - $transferred;
6719 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6724 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6726 die "copy failed: $err" if $err;
6729 sub qemu_img_format
{
6730 my ($scfg, $volname) = @_;
6732 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6739 sub qemu_drive_mirror
{
6740 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6742 $jobs = {} if !$jobs;
6746 $jobs->{"drive-$drive"} = {};
6748 if ($dst_volid =~ /^nbd:/) {
6749 $qemu_target = $dst_volid;
6752 my $storecfg = PVE
::Storage
::config
();
6753 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6755 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6757 $format = qemu_img_format
($dst_scfg, $dst_volname);
6759 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6761 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6764 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6765 $opts->{format
} = $format if $format;
6767 if (defined($src_bitmap)) {
6768 $opts->{sync
} = 'incremental';
6769 $opts->{bitmap
} = $src_bitmap;
6770 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6773 if (defined($bwlimit)) {
6774 $opts->{speed
} = $bwlimit * 1024;
6775 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6777 print "drive mirror is starting for drive-$drive\n";
6780 # if a job already runs for this device we get an error, catch it for cleanup
6781 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
6783 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6785 die "mirroring error: $err\n";
6788 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
6791 # $completion can be either
6792 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6793 # 'cancel': wait until all jobs are ready, block-job-cancel them
6794 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6795 sub qemu_drive_mirror_monitor
{
6796 my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6798 $completion //= 'complete';
6801 my $err_complete = 0;
6804 die "storage migration timed out\n" if $err_complete > 300;
6806 my $stats = mon_cmd
($vmid, "query-block-jobs");
6808 my $running_mirror_jobs = {};
6809 foreach my $stat (@$stats) {
6810 next if $stat->{type
} ne 'mirror';
6811 $running_mirror_jobs->{$stat->{device
}} = $stat;
6814 my $readycounter = 0;
6816 foreach my $job (keys %$jobs) {
6818 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6819 print "$job : finished\n";
6820 delete $jobs->{$job};
6824 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6826 my $busy = $running_mirror_jobs->{$job}->{busy
};
6827 my $ready = $running_mirror_jobs->{$job}->{ready
};
6828 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6829 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6830 my $remaining = $total - $transferred;
6831 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6833 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6836 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6839 last if scalar(keys %$jobs) == 0;
6841 if ($readycounter == scalar(keys %$jobs)) {
6842 print "all mirroring jobs are ready \n";
6843 last if $completion eq 'skip'; #do the complete later
6845 if ($vmiddst && $vmiddst != $vmid) {
6846 my $agent_running = $qga && qga_check_running
($vmid);
6847 if ($agent_running) {
6848 print "freeze filesystem\n";
6849 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6851 print "suspend vm\n";
6852 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6855 # if we clone a disk for a new target vm, we don't switch the disk
6856 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6858 if ($agent_running) {
6859 print "unfreeze filesystem\n";
6860 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6862 print "resume vm\n";
6863 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6869 foreach my $job (keys %$jobs) {
6870 # try to switch the disk if source and destination are on the same guest
6871 print "$job: Completing block job...\n";
6874 if ($completion eq 'complete') {
6875 $op = 'block-job-complete';
6876 } elsif ($completion eq 'cancel') {
6877 $op = 'block-job-cancel';
6879 die "invalid completion value: $completion\n";
6881 eval { mon_cmd
($vmid, $op, device
=> $job) };
6882 if ($@ =~ m/cannot be completed/) {
6883 print "$job: Block job cannot be completed, try again.\n";
6886 print "$job: Completed successfully.\n";
6887 $jobs->{$job}->{complete
} = 1;
6898 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6899 die "mirroring error: $err";
6904 sub qemu_blockjobs_cancel
{
6905 my ($vmid, $jobs) = @_;
6907 foreach my $job (keys %$jobs) {
6908 print "$job: Cancelling block job\n";
6909 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6910 $jobs->{$job}->{cancel
} = 1;
6914 my $stats = mon_cmd
($vmid, "query-block-jobs");
6916 my $running_jobs = {};
6917 foreach my $stat (@$stats) {
6918 $running_jobs->{$stat->{device
}} = $stat;
6921 foreach my $job (keys %$jobs) {
6923 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6924 print "$job: Done.\n";
6925 delete $jobs->{$job};
6929 last if scalar(keys %$jobs) == 0;
6936 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6937 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6942 print "create linked clone of drive $drivename ($drive->{file})\n";
6943 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6944 push @$newvollist, $newvolid;
6947 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6948 $storeid = $storage if $storage;
6950 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6952 print "create full clone of drive $drivename ($drive->{file})\n";
6955 if (drive_is_cloudinit
($drive)) {
6956 $name = "vm-$newvmid-cloudinit";
6957 $name .= ".$dst_format" if $dst_format ne 'raw';
6959 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
6960 } elsif ($drivename eq 'efidisk0') {
6961 $size = get_efivars_size
($conf);
6963 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
6966 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $dst_format, $name, $size);
6967 push @$newvollist, $newvolid;
6969 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
6971 if (drive_is_cloudinit
($drive)) {
6975 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6976 if (!$running || $snapname) {
6977 # TODO: handle bwlimits
6978 if ($drivename eq 'efidisk0') {
6979 # the relevant data on the efidisk may be smaller than the source
6980 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6981 # that is given by the OVMF_VARS.fd
6982 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
6983 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
6984 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
6985 "if=$src_path", "of=$dst_path"]);
6987 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
6991 my $kvmver = get_running_qemu_version
($vmid);
6992 if (!min_version
($kvmver, 2, 7)) {
6993 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6994 if $drive->{iothread
};
6997 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
6998 $completion, $qga, $bwlimit);
7003 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
7006 $disk->{format
} = undef;
7007 $disk->{file
} = $newvolid;
7008 $disk->{size
} = $size;
7013 sub get_running_qemu_version
{
7015 my $res = mon_cmd
($vmid, "query-version");
7016 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7019 sub qemu_use_old_bios_files
{
7020 my ($machine_type) = @_;
7022 return if !$machine_type;
7024 my $use_old_bios_files = undef;
7026 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7028 $use_old_bios_files = 1;
7030 my $version = extract_version
($machine_type, kvm_user_version
());
7031 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7032 # load new efi bios files on migration. So this hack is required to allow
7033 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7034 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7035 $use_old_bios_files = !min_version
($version, 2, 4);
7038 return ($use_old_bios_files, $machine_type);
7041 sub get_efivars_size
{
7043 my $arch = get_vm_arch
($conf);
7044 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7045 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
7046 return -s
$ovmf_vars;
7049 sub update_efidisk_size
{
7052 return if !defined($conf->{efidisk0
});
7054 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
7055 $disk->{size
} = get_efivars_size
($conf);
7056 $conf->{efidisk0
} = print_drive
($disk);
7061 sub create_efidisk
($$$$$) {
7062 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7064 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7065 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7067 my $vars_size_b = -s
$ovmf_vars;
7068 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7069 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7070 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7072 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7073 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7075 return ($volid, $size/1024);
7078 sub vm_iothreads_list
{
7081 my $res = mon_cmd
($vmid, 'query-iothreads');
7084 foreach my $iothread (@$res) {
7085 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7092 my ($conf, $drive) = @_;
7096 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7098 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7104 my $controller = int($drive->{index} / $maxdev);
7105 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
7109 return ($maxdev, $controller, $controller_prefix);
7112 sub windows_version
{
7115 return 0 if !$ostype;
7119 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7121 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7123 } elsif ($ostype =~ m/^win(\d+)$/) {
7130 sub resolve_dst_disk_format
{
7131 my ($storecfg, $storeid, $src_volname, $format) = @_;
7132 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7135 # if no target format is specified, use the source disk format as hint
7137 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7138 $format = qemu_img_format
($scfg, $src_volname);
7144 # test if requested format is supported - else use default
7145 my $supported = grep { $_ eq $format } @$validFormats;
7146 $format = $defFormat if !$supported;
7150 # NOTE: if this logic changes, please update docs & possibly gui logic
7151 sub find_vmstate_storage
{
7152 my ($conf, $storecfg) = @_;
7154 # first, return storage from conf if set
7155 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7157 my ($target, $shared, $local);
7159 foreach_storage_used_by_vm
($conf, sub {
7161 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7162 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7163 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7166 # second, use shared storage where VM has at least one disk
7167 # third, use local storage where VM has at least one disk
7168 # fall back to local storage
7169 $target = $shared // $local // 'local';
7175 my ($uuid, $uuid_str);
7176 UUID
::generate
($uuid);
7177 UUID
::unparse
($uuid, $uuid_str);
7181 sub generate_smbios1_uuid
{
7182 return "uuid=".generate_uuid
();
7188 mon_cmd
($vmid, 'nbd-server-stop');
7191 sub create_reboot_request
{
7193 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7194 or die "failed to create reboot trigger file: $!\n";
7198 sub clear_reboot_request
{
7200 my $path = "/run/qemu-server/$vmid.reboot";
7203 $res = unlink($path);
7204 die "could not remove reboot request for $vmid: $!"
7205 if !$res && $! != POSIX
::ENOENT
;
7210 sub bootorder_from_legacy
{
7211 my ($conf, $bootcfg) = @_;
7213 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
7214 my $bootindex_hash = {};
7216 foreach my $o (split(//, $boot)) {
7217 $bootindex_hash->{$o} = $i*100;
7223 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7224 my ($ds, $drive) = @_;
7226 if (drive_is_cdrom
($drive, 1)) {
7227 if ($bootindex_hash->{d
}) {
7228 $bootorder->{$ds} = $bootindex_hash->{d
};
7229 $bootindex_hash->{d
} += 1;
7231 } elsif ($bootindex_hash->{c
}) {
7232 $bootorder->{$ds} = $bootindex_hash->{c
}
7233 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
7234 $bootindex_hash->{c
} += 1;
7238 if ($bootindex_hash->{n
}) {
7239 for (my $i = 0; $i < $MAX_NETS; $i++) {
7240 my $netname = "net$i";
7241 next if !$conf->{$netname};
7242 $bootorder->{$netname} = $bootindex_hash->{n
};
7243 $bootindex_hash->{n
} += 1;
7250 # Generate default device list for 'boot: order=' property. Matches legacy
7251 # default boot order, but with explicit device names. This is important, since
7252 # the fallback for when neither 'order' nor the old format is specified relies
7253 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7254 sub get_default_bootdevices
{
7260 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
7261 push @ret, $first if $first;
7264 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
7265 push @ret, $first if $first;
7268 for (my $i = 0; $i < $MAX_NETS; $i++) {
7269 my $netname = "net$i";
7270 next if !$conf->{$netname};
7271 push @ret, $netname;
7278 # bash completion helper
7280 sub complete_backup_archives
{
7281 my ($cmdname, $pname, $cvalue) = @_;
7283 my $cfg = PVE
::Storage
::config
();
7287 if ($cvalue =~ m/^([^:]+):/) {
7291 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7294 foreach my $id (keys %$data) {
7295 foreach my $item (@{$data->{$id}}) {
7296 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7297 push @$res, $item->{volid
} if defined($item->{volid
});
7304 my $complete_vmid_full = sub {
7307 my $idlist = vmstatus
();
7311 foreach my $id (keys %$idlist) {
7312 my $d = $idlist->{$id};
7313 if (defined($running)) {
7314 next if $d->{template
};
7315 next if $running && $d->{status
} ne 'running';
7316 next if !$running && $d->{status
} eq 'running';
7325 return &$complete_vmid_full();
7328 sub complete_vmid_stopped
{
7329 return &$complete_vmid_full(0);
7332 sub complete_vmid_running
{
7333 return &$complete_vmid_full(1);
7336 sub complete_storage
{
7338 my $cfg = PVE
::Storage
::config
();
7339 my $ids = $cfg->{ids
};
7342 foreach my $sid (keys %$ids) {
7343 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7344 next if !$ids->{$sid}->{content
}->{images
};
7351 sub complete_migration_storage
{
7352 my ($cmd, $param, $current_value, $all_args) = @_;
7354 my $targetnode = @$all_args[1];
7356 my $cfg = PVE
::Storage
::config
();
7357 my $ids = $cfg->{ids
};
7360 foreach my $sid (keys %$ids) {
7361 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7362 next if !$ids->{$sid}->{content
}->{images
};