1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
30 use PVE
::DataCenterConfig
;
31 use PVE
::Exception
qw(raise raise_param_exc);
32 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
34 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
36 use PVE
::RPCEnvironment
;
40 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
44 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
45 use PVE
::QemuServer
::Cloudinit
;
46 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
47 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
48 use PVE
::QemuServer
::Machine
;
49 use PVE
::QemuServer
::Memory
;
50 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
51 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
52 use PVE
::QemuServer
::USB
qw(parse_usb_device);
56 require PVE
::Network
::SDN
::Zones
;
60 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
63 "$EDK2_FW_BASE/OVMF_CODE.fd",
64 "$EDK2_FW_BASE/OVMF_VARS.fd"
67 "$EDK2_FW_BASE/AAVMF_CODE.fd",
68 "$EDK2_FW_BASE/AAVMF_VARS.fd"
72 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
74 # Note about locking: we use flock on the config file protect
75 # against concurent actions.
76 # Aditionaly, we have a 'lock' setting in the config file. This
77 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
78 # allowed when such lock is set. But you can ignore this kind of
79 # lock with the --skiplock flag.
81 cfs_register_file
('/qemu-server/',
85 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
86 description
=> "Some command save/restore state from this location.",
92 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
93 description
=> "Specifies the Qemu machine type.",
95 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
102 my ($map, $source) = @_;
104 return $source if !defined($map);
106 return $map->{entries
}->{$source}
107 if $map->{entries
} && defined($map->{entries
}->{$source});
109 return $map->{default} if $map->{default};
111 # identity (fallback)
115 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
116 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
118 format
=> 'storagepair-list',
122 #no warnings 'redefine';
125 my ($controller, $vmid, $option, $value) = @_;
127 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
128 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
134 $nodename_cache //= PVE
::INotify
::nodename
();
135 return $nodename_cache;
142 enum
=> [qw(i6300esb ib700)],
143 description
=> "Watchdog type to emulate.",
144 default => 'i6300esb',
149 enum
=> [qw(reset shutdown poweroff pause debug none)],
150 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
154 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
158 description
=> "Enable/disable Qemu GuestAgent.",
163 fstrim_cloned_disks
=> {
164 description
=> "Run fstrim after cloning/moving a disk.",
170 description
=> "Select the agent type",
174 enum
=> [qw(virtio isa)],
180 description
=> "Select the VGA type.",
185 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
188 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
200 description
=> "The size of the file in MB.",
204 pattern
=> '[a-zA-Z0-9\-]+',
206 format_description
=> 'string',
207 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
214 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
215 description
=> "Configure an audio device."
222 description
=> "Driver backend for the audio device."
226 my $spice_enhancements_fmt = {
231 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
235 enum
=> ['off', 'all', 'filter'],
238 description
=> "Enable video streaming. Uses compression for detected video streams."
245 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
247 description
=> "The file on the host to gather entropy from. In most"
248 . " cases /dev/urandom should be preferred over /dev/random"
249 . " to avoid entropy-starvation issues on the host. Using"
250 . " urandom does *not* decrease security in any meaningful"
251 . " way, as it's still seeded from real entropy, and the"
252 . " bytes provided will most likely be mixed with real"
253 . " entropy on the guest as well. /dev/hwrng can be used"
254 . " to pass through a hardware RNG from the host.",
258 description
=> "Maximum bytes of entropy injected into the guest every"
259 . " 'period' milliseconds. Prefer a lower value when using"
260 . " /dev/random as source. Use 0 to disable limiting"
261 . " (potentially dangerous!).",
264 # default is 1 KiB/s, provides enough entropy to the guest to avoid
265 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
266 # of overwhelming the host, provided we're reading from /dev/urandom
271 description
=> "Every 'period' milliseconds the entropy-injection quota"
272 . " is reset, allowing the guest to retrieve another"
273 . " 'max_bytes' of entropy.",
283 description
=> "Specifies whether a VM will be started during system bootup.",
289 description
=> "Automatic restart after crash (currently ignored).",
294 type
=> 'string', format
=> 'pve-hotplug-features',
295 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
296 default => 'network,disk,usb',
301 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
307 description
=> "Lock/unlock the VM.",
308 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
313 description
=> "Limit of CPU usage.",
314 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
322 description
=> "CPU weight for a VM.",
323 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
331 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
338 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
344 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
352 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
353 "It should not be necessary to set it.",
354 enum
=> PVE
::Tools
::kvmkeymaplist
(),
359 type
=> 'string', format
=> 'dns-name',
360 description
=> "Set a name for the VM. Only used on the configuration web interface.",
365 description
=> "SCSI controller model",
366 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
372 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
377 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
378 description
=> "Specify guest operating system.",
379 verbose_description
=> <<EODESC,
380 Specify guest operating system. This is used to enable special
381 optimization/features for specific operating systems:
384 other;; unspecified OS
385 wxp;; Microsoft Windows XP
386 w2k;; Microsoft Windows 2000
387 w2k3;; Microsoft Windows 2003
388 w2k8;; Microsoft Windows 2008
389 wvista;; Microsoft Windows Vista
390 win7;; Microsoft Windows 7
391 win8;; Microsoft Windows 8/2012/2012r2
392 win10;; Microsoft Windows 10/2016
393 l24;; Linux 2.4 Kernel
394 l26;; Linux 2.6 - 5.X Kernel
395 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
400 type
=> 'string', format
=> 'pve-qm-boot',
401 description
=> "Specify guest boot order. Use with 'order=', usage with"
402 . " no key or 'legacy=' is deprecated.",
406 type
=> 'string', format
=> 'pve-qm-bootdisk',
407 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
408 pattern
=> '(ide|sata|scsi|virtio)\d+',
413 description
=> "The number of CPUs. Please use option -sockets instead.",
420 description
=> "The number of CPU sockets.",
427 description
=> "The number of cores per socket.",
434 description
=> "Enable/disable NUMA.",
440 description
=> "Enable/disable hugepages memory.",
441 enum
=> [qw(any 2 1024)],
447 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
448 ." after VM shutdown and can be used for subsequent starts.",
453 description
=> "Number of hotplugged vcpus.",
460 description
=> "Enable/disable ACPI.",
465 description
=> "Enable/disable Qemu GuestAgent and its properties.",
467 format
=> $agent_fmt,
472 description
=> "Enable/disable KVM hardware virtualization.",
478 description
=> "Enable/disable time drift fix.",
484 description
=> "Set the real time clock to local time. This is enabled by default if ostype"
485 ." indicates a Microsoft OS.",
490 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
494 type
=> 'string', format
=> $vga_fmt,
495 description
=> "Configure the VGA hardware.",
496 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
497 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
498 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
499 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
500 ." display server. For win* OS you can select how many independent displays you want,"
501 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
502 ." using a serial device as terminal.",
506 type
=> 'string', format
=> 'pve-qm-watchdog',
507 description
=> "Create a virtual hardware watchdog device.",
508 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
509 ." action), the watchdog must be periodically polled by an agent inside the guest or"
510 ." else the watchdog will reset the guest (or execute the respective action specified)",
515 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
516 description
=> "Set the initial date of the real time clock. Valid format for date are:"
517 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
518 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
521 startup
=> get_standard_option
('pve-startup-order'),
525 description
=> "Enable/disable Template.",
531 description
=> "Arbitrary arguments passed to kvm.",
532 verbose_description
=> <<EODESCR,
533 Arbitrary arguments passed to kvm, for example:
535 args: -no-reboot -no-hpet
537 NOTE: this option is for experts only.
544 description
=> "Enable/disable the USB tablet device.",
545 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
546 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
547 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
548 ." may consider disabling this to save some context switches. This is turned off by"
549 ." default if you use spice (`qm set <vmid> --vga qxl`).",
554 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
558 migrate_downtime
=> {
561 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
567 type
=> 'string', format
=> 'pve-qm-ide',
568 typetext
=> '<volume>',
569 description
=> "This is an alias for option -ide2",
573 description
=> "Emulated CPU type.",
575 format
=> 'pve-vm-cpu-conf',
577 parent
=> get_standard_option
('pve-snapshot-name', {
579 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
583 description
=> "Timestamp for snapshots.",
589 type
=> 'string', format
=> 'pve-volume-id',
590 description
=> "Reference to a volume which stores the VM state. This is used internally"
593 vmstatestorage
=> get_standard_option
('pve-storage-id', {
594 description
=> "Default storage for VM state volumes/files.",
597 runningmachine
=> get_standard_option
('pve-qemu-machine', {
598 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
602 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
603 ." internally for snapshots.",
606 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
607 format_description
=> 'QEMU -cpu parameter'
609 machine
=> get_standard_option
('pve-qemu-machine'),
611 description
=> "Virtual processor architecture. Defaults to the host.",
614 enum
=> [qw(x86_64 aarch64)],
617 description
=> "Specify SMBIOS type 1 fields.",
618 type
=> 'string', format
=> 'pve-qm-smbios1',
625 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
626 ." remove disk operations.",
632 enum
=> [ qw(seabios ovmf) ],
633 description
=> "Select BIOS implementation.",
634 default => 'seabios',
638 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
639 format_description
=> 'UUID',
640 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
641 ." to disable explicitly.",
642 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
643 ." value identifier to the guest OS. This allows to notify the guest operating system"
644 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
645 ." execution or creation from a template). The guest operating system notices the"
646 ." change, and is then able to react as appropriate by marking its copies of"
647 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
648 ."Note that auto-creation only works when done through API/CLI create or update methods"
649 .", but not when manually editing the config file.",
650 default => "1 (autogenerated)",
655 format
=> 'pve-volume-id',
657 description
=> "Script that will be executed during various steps in the vms lifetime.",
661 format
=> $ivshmem_fmt,
662 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
668 format
=> $audio_fmt,
669 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
672 spice_enhancements
=> {
674 format
=> $spice_enhancements_fmt,
675 description
=> "Configure additional enhancements for SPICE.",
679 type
=> 'string', format
=> 'pve-tag-list',
680 description
=> 'Tags of the VM. This is only meta information.',
686 description
=> "Configure a VirtIO-based Random Number Generator.",
695 description
=> 'Specify a custom file containing all meta data passed to the VM via"
696 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
697 format
=> 'pve-volume-id',
698 format_description
=> 'volume',
703 description
=> 'Specify a custom file containing all network data passed to the VM via'
705 format
=> 'pve-volume-id',
706 format_description
=> 'volume',
711 description
=> 'Specify a custom file containing all user data passed to the VM via'
713 format
=> 'pve-volume-id',
714 format_description
=> 'volume',
717 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
719 my $confdesc_cloudinit = {
723 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
724 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
725 .' and `configdrive2` for windows.',
726 enum
=> ['configdrive2', 'nocloud'],
731 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
732 ." image's configured default user.",
737 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
738 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
739 .' support hashed passwords.',
744 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
746 format
=> 'pve-qm-cicustom',
751 description
=> "cloud-init: Sets DNS search domains for a container. Create will'
752 .' automatically use the setting from the host if neither searchdomain nor nameserver'
757 type
=> 'string', format
=> 'address-list',
758 description
=> "cloud-init: Sets DNS server IP address for a container. Create will'
759 .' automatically use the setting from the host if neither searchdomain nor nameserver'
765 format
=> 'urlencoded',
766 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
770 # what about other qemu settings ?
772 #machine => 'string',
785 ##soundhw => 'string',
787 while (my ($k, $v) = each %$confdesc) {
788 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
791 my $MAX_USB_DEVICES = 5;
793 my $MAX_SERIAL_PORTS = 4;
794 my $MAX_PARALLEL_PORTS = 3;
800 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
801 description
=> "CPUs accessing this NUMA node.",
802 format_description
=> "id[-id];...",
806 description
=> "Amount of memory this NUMA node provides.",
811 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
812 description
=> "Host NUMA nodes to use.",
813 format_description
=> "id[-id];...",
818 enum
=> [qw(preferred bind interleave)],
819 description
=> "NUMA allocation policy.",
823 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
826 type
=> 'string', format
=> $numa_fmt,
827 description
=> "NUMA topology.",
829 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
831 for (my $i = 0; $i < $MAX_NUMA; $i++) {
832 $confdesc->{"numa$i"} = $numadesc;
835 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
836 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
837 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
838 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
840 my $net_fmt_bridge_descr = <<__EOD__;
841 Bridge to attach the network device to. The Proxmox VE standard bridge
844 If you do not specify a bridge, we create a kvm user (NATed) network
845 device, which provides DHCP and DNS services. The following addresses
852 The DHCP server assign addresses to the guest starting from 10.0.2.15.
856 macaddr
=> get_standard_option
('mac-addr', {
857 description
=> "MAC address. That address must be unique withing your network. This is"
858 ." automatically generated if not specified.",
862 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
863 ." very low CPU overhead. If your guest does not support this driver, it is usually"
864 ." best to use 'e1000'.",
865 enum
=> $nic_model_list,
868 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
871 description
=> $net_fmt_bridge_descr,
872 format_description
=> 'bridge',
873 pattern
=> '[-_.\w\d]+',
878 minimum
=> 0, maximum
=> 16,
879 description
=> 'Number of packet queues to be used on the device.',
885 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
890 minimum
=> 1, maximum
=> 4094,
891 description
=> 'VLAN tag to apply to packets on this interface.',
896 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
897 description
=> 'VLAN trunks to pass through this interface.',
898 format_description
=> 'vlanid[;vlanid...]',
903 description
=> 'Whether this interface should be protected by the firewall.',
908 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
913 minimum
=> 1, maximum
=> 65520,
914 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
921 type
=> 'string', format
=> $net_fmt,
922 description
=> "Specify network devices.",
925 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
930 format
=> 'pve-ipv4-config',
931 format_description
=> 'IPv4Format/CIDR',
932 description
=> 'IPv4 address in CIDR format.',
939 format_description
=> 'GatewayIPv4',
940 description
=> 'Default gateway for IPv4 traffic.',
946 format
=> 'pve-ipv6-config',
947 format_description
=> 'IPv6Format/CIDR',
948 description
=> 'IPv6 address in CIDR format.',
955 format_description
=> 'GatewayIPv6',
956 description
=> 'Default gateway for IPv6 traffic.',
961 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
964 type
=> 'string', format
=> 'pve-qm-ipconfig',
965 description
=> <<'EODESCR',
966 cloud-init: Specify IP addresses and gateways for the corresponding interface.
968 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
970 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
971 gateway should be provided.
972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
978 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
980 for (my $i = 0; $i < $MAX_NETS; $i++) {
981 $confdesc->{"net$i"} = $netdesc;
982 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
985 foreach my $key (keys %$confdesc_cloudinit) {
986 $confdesc->{$key} = $confdesc_cloudinit->{$key};
989 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
990 sub verify_volume_id_or_qm_path
{
991 my ($volid, $noerr) = @_;
993 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
997 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
998 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1009 type
=> 'string', format
=> 'pve-qm-usb-device',
1010 format_description
=> 'HOSTUSBDEVICE|spice',
1011 description
=> <<EODESCR,
1012 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1014 'bus-port(.port)*' (decimal numbers) or
1015 'vendor_id:product_id' (hexadeciaml numbers) or
1018 You can use the 'lsusb -t' command to list existing usb devices.
1020 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1021 machines - use with special care.
1023 The value 'spice' can be used to add a usb redirection devices for spice.
1029 description
=> "Specifies whether if given host option is a USB3 device or port.",
1036 type
=> 'string', format
=> $usb_fmt,
1037 description
=> "Configure an USB device (n is 0 to 4).",
1039 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1044 pattern
=> '(/dev/.+|socket)',
1045 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1046 verbose_description
=> <<EODESCR,
1047 Create a serial device inside the VM (n is 0 to 3), and pass through a
1048 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1049 host side (use 'qm terminal' to open a terminal connection).
1051 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1052 use with special care.
1054 CAUTION: Experimental! User reported problems with this option.
1061 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1062 description
=> "Map host parallel devices (n is 0 to 2).",
1063 verbose_description
=> <<EODESCR,
1064 Map host parallel devices (n is 0 to 2).
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067 machines - use with special care.
1069 CAUTION: Experimental! User reported problems with this option.
1073 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1074 $confdesc->{"parallel$i"} = $paralleldesc;
1077 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1078 $confdesc->{"serial$i"} = $serialdesc;
1081 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1082 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1085 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1086 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1089 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1090 $confdesc->{"usb$i"} = $usbdesc;
1098 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1099 . " Deprecated, use 'order=' instead.",
1100 pattern
=> '[acdn]{1,4}',
1101 format_description
=> "[acdn]{1,4}",
1103 # note: this is also the fallback if boot: is not given at all
1109 format
=> 'pve-qm-bootdev-list',
1110 format_description
=> "device[;device...]",
1111 description
=> <<EODESC,
1112 The guest will attempt to boot from devices in the order they appear here.
1114 Disks, optical drives and passed-through storage USB devices will be directly
1115 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1116 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1118 Note that only devices in this list will be marked as bootable and thus loaded
1119 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1120 (e.g. software-raid), you need to specify all of them here.
1122 Overrides the deprecated 'legacy=[acdn]*' value when given.
1126 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1128 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1129 sub verify_bootdev
{
1130 my ($dev, $noerr) = @_;
1132 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && $dev !~ m/^efidisk/;
1136 return 0 if $dev !~ m/^$base\d+$/;
1137 return 0 if !$confdesc->{$dev};
1141 return $dev if $check->("net");
1142 return $dev if $check->("usb");
1143 return $dev if $check->("hostpci");
1146 die "invalid boot device '$dev'\n";
1149 sub print_bootorder
{
1151 my $data = { order
=> join(';', @$devs) };
1152 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1155 my $kvm_api_version = 0;
1158 return $kvm_api_version if $kvm_api_version;
1160 open my $fh, '<', '/dev/kvm' or return;
1162 # 0xae00 => KVM_GET_API_VERSION
1163 $kvm_api_version = ioctl($fh, 0xae00, 0);
1166 return $kvm_api_version;
1169 my $kvm_user_version = {};
1172 sub kvm_user_version
{
1175 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1176 my $st = stat($binary);
1178 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1179 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1180 $cachedmtime == $st->mtime;
1182 $kvm_user_version->{$binary} = 'unknown';
1183 $kvm_mtime->{$binary} = $st->mtime;
1187 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188 $kvm_user_version->{$binary} = $2;
1192 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1195 return $kvm_user_version->{$binary};
1198 my sub extract_version
{
1199 my ($machine_type, $version) = @_;
1200 $version = kvm_user_version
() if !defined($version);
1201 PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1204 sub kernel_has_vhost_net
{
1205 return -c
'/dev/vhost-net';
1210 return defined($confdesc->{$key});
1214 sub get_cdrom_path
{
1216 return $cdrom_path if $cdrom_path;
1218 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1219 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1220 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1224 my ($storecfg, $vmid, $cdrom) = @_;
1226 if ($cdrom eq 'cdrom') {
1227 return get_cdrom_path
();
1228 } elsif ($cdrom eq 'none') {
1230 } elsif ($cdrom =~ m
|^/|) {
1233 return PVE
::Storage
::path
($storecfg, $cdrom);
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id
{
1239 my ($vmid, $file, $media) = @_;
1241 if (!($file eq 'none' || $file eq 'cdrom' ||
1242 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1244 return if $file =~ m
|/|;
1246 if ($media && $media eq 'cdrom') {
1247 $file = "local:iso/$file";
1249 $file = "local:$vmid/$file";
1256 sub verify_media_type
{
1257 my ($opt, $vtype, $media) = @_;
1262 if ($media eq 'disk') {
1264 } elsif ($media eq 'cdrom') {
1267 die "internal error";
1270 return if ($vtype eq $etype);
1272 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1275 sub cleanup_drive_path
{
1276 my ($opt, $storecfg, $drive) = @_;
1278 # try to convert filesystem paths to volume IDs
1280 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1281 ($drive->{file
} !~ m
|^/dev/.+|) &&
1282 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1283 ($drive->{file
} !~ m/^\d+$/)) {
1284 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1285 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1287 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1288 verify_media_type
($opt, $vtype, $drive->{media
});
1289 $drive->{file
} = $volid;
1292 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1295 sub parse_hotplug_features
{
1300 return $res if $data eq '0';
1302 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1304 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1305 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1308 die "invalid hotplug feature '$feature'\n";
1314 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1315 sub pve_verify_hotplug_features
{
1316 my ($value, $noerr) = @_;
1318 return $value if parse_hotplug_features
($value);
1322 die "unable to parse hotplug option\n";
1326 my($fh, $noerr) = @_;
1329 my $SG_GET_VERSION_NUM = 0x2282;
1331 my $versionbuf = "\x00" x
8;
1332 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1334 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1337 my $version = unpack("I", $versionbuf);
1338 if ($version < 30000) {
1339 die "scsi generic interface too old\n" if !$noerr;
1343 my $buf = "\x00" x
36;
1344 my $sensebuf = "\x00" x
8;
1345 my $cmd = pack("C x3 C x1", 0x12, 36);
1347 # see /usr/include/scsi/sg.h
1348 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1350 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351 length($sensebuf), 0, length($buf), $buf,
1352 $cmd, $sensebuf, 6000);
1354 $ret = ioctl($fh, $SG_IO, $packet);
1356 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1360 my @res = unpack($sg_io_hdr_t, $packet);
1361 if ($res[17] || $res[18]) {
1362 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1367 (my $byte0, my $byte1, $res->{vendor
},
1368 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1370 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1371 $res->{type
} = $byte0 & 31;
1379 my $fh = IO
::File-
>new("+<$path") || return;
1380 my $res = scsi_inquiry
($fh, 1);
1386 sub print_tabletdevice_full
{
1387 my ($conf, $arch) = @_;
1389 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1391 # we use uhci for old VMs because tablet driver was buggy in older qemu
1393 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1399 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1402 sub print_keyboarddevice_full
{
1403 my ($conf, $arch, $machine) = @_;
1405 return if $arch ne 'aarch64';
1407 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1410 sub print_drivedevice_full
{
1411 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1416 my $drive_id = "$drive->{interface}$drive->{index}";
1417 if ($drive->{interface
} eq 'virtio') {
1418 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1419 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1420 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1421 } elsif ($drive->{interface
} eq 'scsi') {
1423 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1424 my $unit = $drive->{index} % $maxdev;
1425 my $devicetype = 'hd';
1427 if (drive_is_cdrom
($drive)) {
1430 if ($drive->{file
} =~ m
|^/|) {
1431 $path = $drive->{file
};
1432 if (my $info = path_is_scsi
($path)) {
1433 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1434 $devicetype = 'block';
1435 } elsif ($info->{type
} == 1) { # tape
1436 $devicetype = 'generic';
1440 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1443 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1444 my $version = extract_version
($machine_type, kvm_user_version
());
1445 if ($path =~ m/^iscsi\:\/\
// &&
1446 !min_version
($version, 4, 1)) {
1447 $devicetype = 'generic';
1451 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1452 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1454 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1455 .",lun=$drive->{index}";
1457 $device .= ",drive=drive-$drive_id,id=$drive_id";
1459 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1460 $device .= ",rotation_rate=1";
1462 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1464 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1465 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1466 my $controller = int($drive->{index} / $maxdev);
1467 my $unit = $drive->{index} % $maxdev;
1468 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1470 $device = "ide-$devicetype";
1471 if ($drive->{interface
} eq 'ide') {
1472 $device .= ",bus=ide.$controller,unit=$unit";
1474 $device .= ",bus=ahci$controller.$unit";
1476 $device .= ",drive=drive-$drive_id,id=$drive_id";
1478 if ($devicetype eq 'hd') {
1479 if (my $model = $drive->{model
}) {
1480 $model = URI
::Escape
::uri_unescape
($model);
1481 $device .= ",model=$model";
1483 if ($drive->{ssd
}) {
1484 $device .= ",rotation_rate=1";
1487 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1488 } elsif ($drive->{interface
} eq 'usb') {
1490 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1492 die "unsupported interface type";
1495 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1497 if (my $serial = $drive->{serial
}) {
1498 $serial = URI
::Escape
::uri_unescape
($serial);
1499 $device .= ",serial=$serial";
1506 sub get_initiator_name
{
1509 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1510 while (defined(my $line = <$fh>)) {
1511 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1520 sub print_drive_commandline_full
{
1521 my ($storecfg, $vmid, $drive) = @_;
1524 my $volid = $drive->{file
};
1527 if (drive_is_cdrom
($drive)) {
1528 $path = get_iso_path
($storecfg, $vmid, $volid);
1530 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1532 $path = PVE
::Storage
::path
($storecfg, $volid);
1533 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1534 $format = qemu_img_format
($scfg, $volname);
1542 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1543 foreach my $o (@qemu_drive_options) {
1544 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1547 # snapshot only accepts on|off
1548 if (defined($drive->{snapshot
})) {
1549 my $v = $drive->{snapshot
} ?
'on' : 'off';
1550 $opts .= ",snapshot=$v";
1553 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1554 my ($dir, $qmpname) = @$type;
1555 if (my $v = $drive->{"mbps$dir"}) {
1556 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1558 if (my $v = $drive->{"mbps${dir}_max"}) {
1559 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1561 if (my $v = $drive->{"bps${dir}_max_length"}) {
1562 $opts .= ",throttling.bps$qmpname-max-length=$v";
1564 if (my $v = $drive->{"iops${dir}"}) {
1565 $opts .= ",throttling.iops$qmpname=$v";
1567 if (my $v = $drive->{"iops${dir}_max"}) {
1568 $opts .= ",throttling.iops$qmpname-max=$v";
1570 if (my $v = $drive->{"iops${dir}_max_length"}) {
1571 $opts .= ",throttling.iops$qmpname-max-length=$v";
1575 $opts .= ",format=$format" if $format && !$drive->{format
};
1577 my $cache_direct = 0;
1579 if (my $cache = $drive->{cache
}) {
1580 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1581 } elsif (!drive_is_cdrom
($drive)) {
1582 $opts .= ",cache=none";
1586 # aio native works only with O_DIRECT
1587 if (!$drive->{aio
}) {
1589 $opts .= ",aio=native";
1591 $opts .= ",aio=threads";
1595 if (!drive_is_cdrom
($drive)) {
1597 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1598 $detectzeroes = 'off';
1599 } elsif ($drive->{discard
}) {
1600 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1602 # This used to be our default with discard not being specified:
1603 $detectzeroes = 'on';
1605 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1608 my $pathinfo = $path ?
"file=$path," : '';
1610 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1613 sub print_netdevice_full
{
1614 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1616 my $device = $net->{model
};
1617 if ($net->{model
} eq 'virtio') {
1618 $device = 'virtio-net-pci';
1621 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1622 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1623 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1624 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1625 # and out of each queue plus one config interrupt and control vector queue
1626 my $vectors = $net->{queues
} * 2 + 2;
1627 $tmpstr .= ",vectors=$vectors,mq=on";
1629 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1631 if (my $mtu = $net->{mtu
}) {
1632 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1633 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1636 } elsif ($mtu < 576) {
1637 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1638 } elsif ($mtu > $bridge_mtu) {
1639 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1641 $tmpstr .= ",host_mtu=$mtu";
1643 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1647 if ($use_old_bios_files) {
1649 if ($device eq 'virtio-net-pci') {
1650 $romfile = 'pxe-virtio.rom';
1651 } elsif ($device eq 'e1000') {
1652 $romfile = 'pxe-e1000.rom';
1653 } elsif ($device eq 'ne2k') {
1654 $romfile = 'pxe-ne2k_pci.rom';
1655 } elsif ($device eq 'pcnet') {
1656 $romfile = 'pxe-pcnet.rom';
1657 } elsif ($device eq 'rtl8139') {
1658 $romfile = 'pxe-rtl8139.rom';
1660 $tmpstr .= ",romfile=$romfile" if $romfile;
1666 sub print_netdev_full
{
1667 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1670 if ($netid =~ m/^net(\d+)$/) {
1674 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1676 my $ifname = "tap${vmid}i$i";
1678 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1679 die "interface name '$ifname' is too long (max 15 character)\n"
1680 if length($ifname) >= 16;
1682 my $vhostparam = '';
1683 if (is_native
($arch)) {
1684 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1687 my $vmname = $conf->{name
} || "vm$vmid";
1690 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1692 if ($net->{bridge
}) {
1693 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1694 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1696 $netdev = "type=user,id=$netid,hostname=$vmname";
1699 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1705 'cirrus' => 'cirrus-vga',
1707 'vmware' => 'vmware-svga',
1708 'virtio' => 'virtio-vga',
1711 sub print_vga_device
{
1712 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1714 my $type = $vga_map->{$vga->{type
}};
1715 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1716 $type = 'virtio-gpu';
1718 my $vgamem_mb = $vga->{memory
};
1720 my $max_outputs = '';
1722 $type = $id ?
'qxl' : 'qxl-vga';
1724 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1725 # set max outputs so linux can have up to 4 qxl displays with one device
1726 if (min_version
($machine_version, 4, 1)) {
1727 $max_outputs = ",max_outputs=4";
1732 die "no devicetype for $vga->{type}\n" if !$type;
1736 if ($vga->{type
} eq 'virtio') {
1737 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1738 $memory = ",max_hostmem=$bytes";
1740 # from https://www.spice-space.org/multiple-monitors.html
1741 $memory = ",vgamem_mb=$vga->{memory}";
1742 my $ram = $vgamem_mb * 4;
1743 my $vram = $vgamem_mb * 2;
1744 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1746 $memory = ",vgamem_mb=$vga->{memory}";
1748 } elsif ($qxlnum && $id) {
1749 $memory = ",ram_size=67108864,vram_size=33554432";
1753 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1754 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1757 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1758 my $vgaid = "vga" . ($id // '');
1761 if ($q35 && $vgaid eq 'vga') {
1762 # the first display uses pcie.0 bus on q35 machines
1763 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1765 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1768 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1771 sub parse_number_sets
{
1774 foreach my $part (split(/;/, $set)) {
1775 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1776 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1777 push @$res, [ $1, $2 ];
1779 die "invalid range: $part\n";
1788 my $res = parse_property_string
($numa_fmt, $data);
1789 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1790 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1794 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1798 my $res = eval { parse_property_string
($net_fmt, $data) };
1803 if (!defined($res->{macaddr
})) {
1804 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1805 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1810 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1811 sub parse_ipconfig
{
1814 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1820 if ($res->{gw
} && !$res->{ip
}) {
1821 warn 'gateway specified without specifying an IP address';
1824 if ($res->{gw6
} && !$res->{ip6
}) {
1825 warn 'IPv6 gateway specified without specifying an IPv6 address';
1828 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1829 warn 'gateway specified together with DHCP';
1832 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1834 warn "IPv6 gateway specified together with $res->{ip6} address";
1838 if (!$res->{ip
} && !$res->{ip6
}) {
1839 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1848 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1851 sub add_random_macs
{
1852 my ($settings) = @_;
1854 foreach my $opt (keys %$settings) {
1855 next if $opt !~ m/^net(\d+)$/;
1856 my $net = parse_net
($settings->{$opt});
1858 $settings->{$opt} = print_net
($net);
1862 sub vm_is_volid_owner
{
1863 my ($storecfg, $vmid, $volid) = @_;
1865 if ($volid !~ m
|^/|) {
1867 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1868 if ($owner && ($owner == $vmid)) {
1876 sub vmconfig_register_unused_drive
{
1877 my ($storecfg, $vmid, $conf, $drive) = @_;
1879 if (drive_is_cloudinit
($drive)) {
1880 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1882 } elsif (!drive_is_cdrom
($drive)) {
1883 my $volid = $drive->{file
};
1884 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1885 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1890 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1894 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1895 format_description
=> 'UUID',
1896 description
=> "Set SMBIOS1 UUID.",
1901 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1902 format_description
=> 'Base64 encoded string',
1903 description
=> "Set SMBIOS1 version.",
1908 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1909 format_description
=> 'Base64 encoded string',
1910 description
=> "Set SMBIOS1 serial number.",
1915 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1916 format_description
=> 'Base64 encoded string',
1917 description
=> "Set SMBIOS1 manufacturer.",
1922 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1923 format_description
=> 'Base64 encoded string',
1924 description
=> "Set SMBIOS1 product ID.",
1929 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1930 format_description
=> 'Base64 encoded string',
1931 description
=> "Set SMBIOS1 SKU string.",
1936 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
1937 format_description
=> 'Base64 encoded string',
1938 description
=> "Set SMBIOS1 family string.",
1943 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
1951 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
1958 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
1961 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
1963 sub parse_watchdog
{
1968 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
1973 sub parse_guest_agent
{
1976 return {} if !defined($value->{agent
});
1978 my $res = eval { parse_property_string
($agent_fmt, $value->{agent
}) };
1981 # if the agent is disabled ignore the other potentially set properties
1982 return {} if !$res->{enabled
};
1989 return {} if !$value;
1990 my $res = eval { parse_property_string
($vga_fmt, $value) };
2000 my $res = eval { parse_property_string
($rng_fmt, $value) };
2005 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2006 sub verify_usb_device
{
2007 my ($value, $noerr) = @_;
2009 return $value if parse_usb_device
($value);
2013 die "unable to parse usb device\n";
2016 # add JSON properties for create and set function
2017 sub json_config_properties
{
2020 foreach my $opt (keys %$confdesc) {
2021 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2022 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2023 $prop->{$opt} = $confdesc->{$opt};
2029 # return copy of $confdesc_cloudinit to generate documentation
2030 sub cloudinit_config_properties
{
2032 return dclone
($confdesc_cloudinit);
2036 my ($key, $value) = @_;
2038 die "unknown setting '$key'\n" if !$confdesc->{$key};
2040 my $type = $confdesc->{$key}->{type
};
2042 if (!defined($value)) {
2043 die "got undefined value\n";
2046 if ($value =~ m/[\n\r]/) {
2047 die "property contains a line feed\n";
2050 if ($type eq 'boolean') {
2051 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2052 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2053 die "type check ('boolean') failed - got '$value'\n";
2054 } elsif ($type eq 'integer') {
2055 return int($1) if $value =~ m/^(\d+)$/;
2056 die "type check ('integer') failed - got '$value'\n";
2057 } elsif ($type eq 'number') {
2058 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2059 die "type check ('number') failed - got '$value'\n";
2060 } elsif ($type eq 'string') {
2061 if (my $fmt = $confdesc->{$key}->{format
}) {
2062 PVE
::JSONSchema
::check_format
($fmt, $value);
2065 $value =~ s/^\"(.*)\"$/$1/;
2068 die "internal error"
2073 my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2075 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2077 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2079 if ($conf->{template
}) {
2080 # check if any base image is still used by a linked clone
2081 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2082 my ($ds, $drive) = @_;
2083 return if drive_is_cdrom
($drive);
2085 my $volid = $drive->{file
};
2086 return if !$volid || $volid =~ m
|^/|;
2088 die "base volume '$volid' is still in use by linked cloned\n"
2089 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2094 # only remove disks owned by this VM
2095 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2096 my ($ds, $drive) = @_;
2097 return if drive_is_cdrom
($drive, 1);
2099 my $volid = $drive->{file
};
2100 return if !$volid || $volid =~ m
|^/|;
2102 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2103 return if !$path || !$owner || ($owner != $vmid);
2105 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2106 warn "Could not remove disk '$volid', check manually: $@" if $@;
2109 # also remove unused disk
2110 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2111 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2112 my ($volid, $sid, $volname, $d) = @_;
2113 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2117 if (defined $replacement_conf) {
2118 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2120 PVE
::QemuConfig-
>destroy_config($vmid);
2124 sub parse_vm_config
{
2125 my ($filename, $raw) = @_;
2127 return if !defined($raw);
2130 digest
=> Digest
::SHA
::sha1_hex
($raw),
2135 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2136 || die "got strange filename '$filename'";
2144 my @lines = split(/\n/, $raw);
2145 foreach my $line (@lines) {
2146 next if $line =~ m/^\s*$/;
2148 if ($line =~ m/^\[PENDING\]\s*$/i) {
2149 $section = 'pending';
2150 if (defined($descr)) {
2152 $conf->{description
} = $descr;
2155 $conf = $res->{$section} = {};
2158 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2160 if (defined($descr)) {
2162 $conf->{description
} = $descr;
2165 $conf = $res->{snapshots
}->{$section} = {};
2169 if ($line =~ m/^\#(.*)\s*$/) {
2170 $descr = '' if !defined($descr);
2171 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2175 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2176 $descr = '' if !defined($descr);
2177 $descr .= PVE
::Tools
::decode_text
($2);
2178 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2179 $conf->{snapstate
} = $1;
2180 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2183 $conf->{$key} = $value;
2184 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2186 if ($section eq 'pending') {
2187 $conf->{delete} = $value; # we parse this later
2189 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2191 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2194 eval { $value = check_type
($key, $value); };
2196 warn "vm $vmid - unable to parse value of '$key' - $@";
2198 $key = 'ide2' if $key eq 'cdrom';
2199 my $fmt = $confdesc->{$key}->{format
};
2200 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2201 my $v = parse_drive
($key, $value);
2202 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2203 $v->{file
} = $volid;
2204 $value = print_drive
($v);
2206 warn "vm $vmid - unable to parse value of '$key'\n";
2211 $conf->{$key} = $value;
2216 if (defined($descr)) {
2218 $conf->{description
} = $descr;
2220 delete $res->{snapstate
}; # just to be sure
2225 sub write_vm_config
{
2226 my ($filename, $conf) = @_;
2228 delete $conf->{snapstate
}; # just to be sure
2230 if ($conf->{cdrom
}) {
2231 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2232 $conf->{ide2
} = $conf->{cdrom
};
2233 delete $conf->{cdrom
};
2236 # we do not use 'smp' any longer
2237 if ($conf->{sockets
}) {
2238 delete $conf->{smp
};
2239 } elsif ($conf->{smp
}) {
2240 $conf->{sockets
} = $conf->{smp
};
2241 delete $conf->{cores
};
2242 delete $conf->{smp
};
2245 my $used_volids = {};
2247 my $cleanup_config = sub {
2248 my ($cref, $pending, $snapname) = @_;
2250 foreach my $key (keys %$cref) {
2251 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2252 $key eq 'snapstate' || $key eq 'pending';
2253 my $value = $cref->{$key};
2254 if ($key eq 'delete') {
2255 die "propertry 'delete' is only allowed in [PENDING]\n"
2257 # fixme: check syntax?
2260 eval { $value = check_type
($key, $value); };
2261 die "unable to parse value of '$key' - $@" if $@;
2263 $cref->{$key} = $value;
2265 if (!$snapname && is_valid_drivename
($key)) {
2266 my $drive = parse_drive
($key, $value);
2267 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2272 &$cleanup_config($conf);
2274 &$cleanup_config($conf->{pending
}, 1);
2276 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2277 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2278 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2281 # remove 'unusedX' settings if we re-add a volume
2282 foreach my $key (keys %$conf) {
2283 my $value = $conf->{$key};
2284 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2285 delete $conf->{$key};
2289 my $generate_raw_config = sub {
2290 my ($conf, $pending) = @_;
2294 # add description as comment to top of file
2295 if (defined(my $descr = $conf->{description
})) {
2297 foreach my $cl (split(/\n/, $descr)) {
2298 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2301 $raw .= "#\n" if $pending;
2305 foreach my $key (sort keys %$conf) {
2306 next if $key =~ /^(digest|description|pending|snapshots)$/;
2307 $raw .= "$key: $conf->{$key}\n";
2312 my $raw = &$generate_raw_config($conf);
2314 if (scalar(keys %{$conf->{pending
}})){
2315 $raw .= "\n[PENDING]\n";
2316 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2319 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2320 $raw .= "\n[$snapname]\n";
2321 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2331 # we use static defaults from our JSON schema configuration
2332 foreach my $key (keys %$confdesc) {
2333 if (defined(my $default = $confdesc->{$key}->{default})) {
2334 $res->{$key} = $default;
2342 my $vmlist = PVE
::Cluster
::get_vmlist
();
2344 return $res if !$vmlist || !$vmlist->{ids
};
2345 my $ids = $vmlist->{ids
};
2346 my $nodename = nodename
();
2348 foreach my $vmid (keys %$ids) {
2349 my $d = $ids->{$vmid};
2350 next if !$d->{node
} || $d->{node
} ne $nodename;
2351 next if !$d->{type
} || $d->{type
} ne 'qemu';
2352 $res->{$vmid}->{exists} = 1;
2357 # test if VM uses local resources (to prevent migration)
2358 sub check_local_resources
{
2359 my ($conf, $noerr) = @_;
2363 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2364 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2366 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2368 foreach my $k (keys %$conf) {
2369 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2370 # sockets are safe: they will recreated be on the target side post-migrate
2371 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2372 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2375 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2380 # check if used storages are available on all nodes (use by migrate)
2381 sub check_storage_availability
{
2382 my ($storecfg, $conf, $node) = @_;
2384 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2385 my ($ds, $drive) = @_;
2387 my $volid = $drive->{file
};
2390 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2393 # check if storage is available on both nodes
2394 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2395 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2399 # list nodes where all VM images are available (used by has_feature API)
2401 my ($conf, $storecfg) = @_;
2403 my $nodelist = PVE
::Cluster
::get_nodelist
();
2404 my $nodehash = { map { $_ => 1 } @$nodelist };
2405 my $nodename = nodename
();
2407 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2408 my ($ds, $drive) = @_;
2410 my $volid = $drive->{file
};
2413 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2415 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2416 if ($scfg->{disable
}) {
2418 } elsif (my $avail = $scfg->{nodes
}) {
2419 foreach my $node (keys %$nodehash) {
2420 delete $nodehash->{$node} if !$avail->{$node};
2422 } elsif (!$scfg->{shared
}) {
2423 foreach my $node (keys %$nodehash) {
2424 delete $nodehash->{$node} if $node ne $nodename
2433 sub check_local_storage_availability
{
2434 my ($conf, $storecfg) = @_;
2436 my $nodelist = PVE
::Cluster
::get_nodelist
();
2437 my $nodehash = { map { $_ => {} } @$nodelist };
2439 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2440 my ($ds, $drive) = @_;
2442 my $volid = $drive->{file
};
2445 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2447 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2449 if ($scfg->{disable
}) {
2450 foreach my $node (keys %$nodehash) {
2451 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2453 } elsif (my $avail = $scfg->{nodes
}) {
2454 foreach my $node (keys %$nodehash) {
2455 if (!$avail->{$node}) {
2456 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2463 foreach my $node (values %$nodehash) {
2464 if (my $unavail = $node->{unavailable_storages
}) {
2465 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2472 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2474 my ($vmid, $nocheck, $node) = @_;
2476 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2477 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2482 my $vzlist = config_list
();
2484 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2486 while (defined(my $de = $fd->read)) {
2487 next if $de !~ m/^(\d+)\.pid$/;
2489 next if !defined($vzlist->{$vmid});
2490 if (my $pid = check_running
($vmid)) {
2491 $vzlist->{$vmid}->{pid
} = $pid;
2498 our $vmstatus_return_properties = {
2499 vmid
=> get_standard_option
('pve-vmid'),
2501 description
=> "Qemu process status.",
2503 enum
=> ['stopped', 'running'],
2506 description
=> "Maximum memory in bytes.",
2509 renderer
=> 'bytes',
2512 description
=> "Root disk size in bytes.",
2515 renderer
=> 'bytes',
2518 description
=> "VM name.",
2523 description
=> "Qemu QMP agent status.",
2528 description
=> "PID of running qemu process.",
2533 description
=> "Uptime.",
2536 renderer
=> 'duration',
2539 description
=> "Maximum usable CPUs.",
2544 description
=> "The current config lock, if any.",
2549 description
=> "The current configured tags, if any",
2555 my $last_proc_pid_stat;
2557 # get VM status information
2558 # This must be fast and should not block ($full == false)
2559 # We only query KVM using QMP if $full == true (this can be slow)
2561 my ($opt_vmid, $full) = @_;
2565 my $storecfg = PVE
::Storage
::config
();
2567 my $list = vzlist
();
2568 my $defaults = load_defaults
();
2570 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2572 my $cpucount = $cpuinfo->{cpus
} || 1;
2574 foreach my $vmid (keys %$list) {
2575 next if $opt_vmid && ($vmid ne $opt_vmid);
2577 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2579 my $d = { vmid
=> $vmid };
2580 $d->{pid
} = $list->{$vmid}->{pid
};
2582 # fixme: better status?
2583 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2585 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2586 if (defined($size)) {
2587 $d->{disk
} = 0; # no info available
2588 $d->{maxdisk
} = $size;
2594 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2595 * ($conf->{cores
} || $defaults->{cores
});
2596 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2597 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2599 $d->{name
} = $conf->{name
} || "VM $vmid";
2600 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2601 : $defaults->{memory
}*(1024*1024);
2603 if ($conf->{balloon
}) {
2604 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2605 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2606 : $defaults->{shares
};
2617 $d->{diskwrite
} = 0;
2619 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
2621 $d->{serial
} = 1 if conf_has_serial
($conf);
2622 $d->{lock} = $conf->{lock} if $conf->{lock};
2623 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2628 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2629 foreach my $dev (keys %$netdev) {
2630 next if $dev !~ m/^tap([1-9]\d*)i/;
2632 my $d = $res->{$vmid};
2635 $d->{netout
} += $netdev->{$dev}->{receive
};
2636 $d->{netin
} += $netdev->{$dev}->{transmit
};
2639 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
2640 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
2645 my $ctime = gettimeofday
;
2647 foreach my $vmid (keys %$list) {
2649 my $d = $res->{$vmid};
2650 my $pid = $d->{pid
};
2653 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2654 next if !$pstat; # not running
2656 my $used = $pstat->{utime} + $pstat->{stime
};
2658 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2660 if ($pstat->{vsize
}) {
2661 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2664 my $old = $last_proc_pid_stat->{$pid};
2666 $last_proc_pid_stat->{$pid} = {
2674 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2676 if ($dtime > 1000) {
2677 my $dutime = $used - $old->{used
};
2679 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2680 $last_proc_pid_stat->{$pid} = {
2686 $d->{cpu
} = $old->{cpu
};
2690 return $res if !$full;
2692 my $qmpclient = PVE
::QMPClient-
>new();
2694 my $ballooncb = sub {
2695 my ($vmid, $resp) = @_;
2697 my $info = $resp->{'return'};
2698 return if !$info->{max_mem
};
2700 my $d = $res->{$vmid};
2702 # use memory assigned to VM
2703 $d->{maxmem
} = $info->{max_mem
};
2704 $d->{balloon
} = $info->{actual
};
2706 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2707 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2708 $d->{freemem
} = $info->{free_mem
};
2711 $d->{ballooninfo
} = $info;
2714 my $blockstatscb = sub {
2715 my ($vmid, $resp) = @_;
2716 my $data = $resp->{'return'} || [];
2717 my $totalrdbytes = 0;
2718 my $totalwrbytes = 0;
2720 for my $blockstat (@$data) {
2721 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2722 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2724 $blockstat->{device
} =~ s/drive-//;
2725 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2727 $res->{$vmid}->{diskread
} = $totalrdbytes;
2728 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2731 my $statuscb = sub {
2732 my ($vmid, $resp) = @_;
2734 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2735 # this fails if ballon driver is not loaded, so this must be
2736 # the last commnand (following command are aborted if this fails).
2737 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2739 my $status = 'unknown';
2740 if (!defined($status = $resp->{'return'}->{status
})) {
2741 warn "unable to get VM status\n";
2745 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2748 foreach my $vmid (keys %$list) {
2749 next if $opt_vmid && ($vmid ne $opt_vmid);
2750 next if !$res->{$vmid}->{pid
}; # not running
2751 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2754 $qmpclient->queue_execute(undef, 2);
2756 foreach my $vmid (keys %$list) {
2757 next if $opt_vmid && ($vmid ne $opt_vmid);
2758 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2764 sub conf_has_serial
{
2767 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2768 if ($conf->{"serial$i"}) {
2776 sub conf_has_audio
{
2777 my ($conf, $id) = @_;
2780 my $audio = $conf->{"audio$id"};
2781 return if !defined($audio);
2783 my $audioproperties = parse_property_string
($audio_fmt, $audio);
2784 my $audiodriver = $audioproperties->{driver
} // 'spice';
2787 dev
=> $audioproperties->{device
},
2788 dev_id
=> "audiodev$id",
2789 backend
=> $audiodriver,
2790 backend_id
=> "$audiodriver-backend${id}",
2795 my ($audio, $audiopciaddr, $machine_version) = @_;
2799 my $id = $audio->{dev_id
};
2801 if (min_version
($machine_version, 4, 2)) {
2802 $audiodev = ",audiodev=$audio->{backend_id}";
2805 if ($audio->{dev
} eq 'AC97') {
2806 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2807 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
2808 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2809 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2810 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2812 die "unkown audio device '$audio->{dev}', implement me!";
2815 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2820 sub vga_conf_has_spice
{
2823 my $vgaconf = parse_vga
($vga);
2824 my $vgatype = $vgaconf->{type
};
2825 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2832 return get_host_arch
() eq $arch;
2837 return $conf->{arch
} // get_host_arch
();
2840 my $default_machines = {
2845 sub get_vm_machine
{
2846 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2848 my $machine = $forcemachine || $conf->{machine
};
2850 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2852 $machine ||= $default_machines->{$arch};
2853 if ($add_pve_version) {
2854 $kvmversion //= kvm_user_version
();
2855 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
2856 $machine .= "+pve$pvever";
2860 if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2861 # for version-pinned machines that do not include a pve-version (e.g.
2862 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2863 $machine .= '+pve0';
2869 sub get_ovmf_files
($) {
2872 my $ovmf = $OVMF->{$arch}
2873 or die "no OVMF images known for architecture '$arch'\n";
2879 aarch64
=> '/usr/bin/qemu-system-aarch64',
2880 x86_64
=> '/usr/bin/qemu-system-x86_64',
2882 sub get_command_for_arch
($) {
2884 return '/usr/bin/kvm' if is_native
($arch);
2886 my $cmd = $Arch2Qemu->{$arch}
2887 or die "don't know how to emulate architecture '$arch'\n";
2891 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2892 # to use in a QEMU command line (-cpu element), first array_intersect the result
2893 # of query_supported_ with query_understood_. This is necessary because:
2895 # a) query_understood_ returns flags the host cannot use and
2896 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2897 # flags, but CPU settings - with most of them being flags. Those settings
2898 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
2900 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2901 # expensive. If you need the value returned from this, you can get it much
2902 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2903 # $accel being 'kvm' or 'tcg'.
2905 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2906 # changes, automatically populating pmxcfs.
2908 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2909 # since kvm and tcg machines support different flags
2911 sub query_supported_cpu_flags
{
2914 $arch //= get_host_arch
();
2915 my $default_machine = $default_machines->{$arch};
2919 # FIXME: Once this is merged, the code below should work for ARM as well:
2920 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2921 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2924 my $kvm_supported = defined(kvm_version
());
2925 my $qemu_cmd = get_command_for_arch
($arch);
2927 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
2929 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2930 my $query_supported_run_qemu = sub {
2936 '-machine', $default_machine,
2938 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2939 '-mon', 'chardev=qmp,mode=control',
2940 '-pidfile', $pidfile,
2945 push @$cmd, '-accel', 'tcg';
2948 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
2949 die "QEMU flag querying VM exited with code " . $rc if $rc;
2952 my $cmd_result = mon_cmd
(
2954 'query-cpu-model-expansion',
2956 model
=> { name
=> 'host' }
2959 my $props = $cmd_result->{model
}->{props
};
2960 foreach my $prop (keys %$props) {
2961 next if $props->{$prop} ne '1';
2962 # QEMU returns some flags multiple times, with '_', '.' or '-'
2963 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2964 # We only keep those with underscores, to match /proc/cpuinfo
2965 $prop =~ s/\.|-/_/g;
2966 $flags->{$prop} = 1;
2971 # force stop with 10 sec timeout and 'nocheck'
2972 # always stop, even if QMP failed
2973 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
2977 return [ sort keys %$flags ];
2980 # We need to query QEMU twice, since KVM and TCG have different supported flags
2981 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
2982 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
2983 warn "warning: failed querying supported tcg flags: $@\n" if $@;
2985 if ($kvm_supported) {
2986 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
2987 warn "warning: failed querying supported kvm flags: $@\n" if $@;
2994 # Understood CPU flags are written to a file at 'pve-qemu' compile time
2995 my $understood_cpu_flag_dir = "/usr/share/kvm";
2996 sub query_understood_cpu_flags
{
2997 my $arch = get_host_arch
();
2998 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3000 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3003 my $raw = file_get_contents
($filepath);
3004 $raw =~ s/^\s+|\s+$//g;
3005 my @flags = split(/\s+/, $raw);
3010 sub config_to_command
{
3011 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3014 my $globalFlags = [];
3015 my $machineFlags = [];
3020 my $ostype = $conf->{ostype
};
3021 my $winversion = windows_version
($ostype);
3022 my $kvm = $conf->{kvm
};
3023 my $nodename = nodename
();
3025 my $arch = get_vm_arch
($conf);
3026 my $kvm_binary = get_command_for_arch
($arch);
3027 my $kvmver = kvm_user_version
($kvm_binary);
3029 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3030 $kvmver //= "undefined";
3031 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3034 my $add_pve_version = min_version
($kvmver, 4, 1);
3036 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3037 my $machine_version = extract_version
($machine_type, $kvmver);
3038 $kvm //= 1 if is_native
($arch);
3040 $machine_version =~ m/(\d+)\.(\d+)/;
3041 my ($machine_major, $machine_minor) = ($1, $2);
3043 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3044 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3045 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3046 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3047 ." please upgrade node '$nodename'\n"
3048 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3049 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3050 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3051 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3052 ." node '$nodename'\n";
3055 # if a specific +pve version is required for a feature, use $version_guard
3056 # instead of min_version to allow machines to be run with the minimum
3058 my $required_pve_version = 0;
3059 my $version_guard = sub {
3060 my ($major, $minor, $pve) = @_;
3061 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3062 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3063 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3064 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3068 if ($kvm && !defined kvm_version
()) {
3069 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3070 ." or enable in BIOS.\n";
3073 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3074 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3075 my $use_old_bios_files = undef;
3076 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3078 my $cpuunits = defined($conf->{cpuunits
}) ?
3079 $conf->{cpuunits
} : $defaults->{cpuunits
};
3081 push @$cmd, $kvm_binary;
3083 push @$cmd, '-id', $vmid;
3085 my $vmname = $conf->{name
} || "vm$vmid";
3087 push @$cmd, '-name', $vmname;
3091 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3092 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3093 push @$cmd, '-mon', "chardev=qmp,mode=control";
3095 if (min_version
($machine_version, 2, 12)) {
3096 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3097 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3100 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3102 push @$cmd, '-daemonize';
3104 if ($conf->{smbios1
}) {
3105 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3106 if ($smbios_conf->{base64
}) {
3107 # Do not pass base64 flag to qemu
3108 delete $smbios_conf->{base64
};
3109 my $smbios_string = "";
3110 foreach my $key (keys %$smbios_conf) {
3112 if ($key eq "uuid") {
3113 $value = $smbios_conf->{uuid
}
3115 $value = decode_base64
($smbios_conf->{$key});
3117 # qemu accepts any binary data, only commas need escaping by double comma
3119 $smbios_string .= "," . $key . "=" . $value if $value;
3121 push @$cmd, '-smbios', "type=1" . $smbios_string;
3123 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3127 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3128 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3129 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3131 my ($path, $format);
3132 if (my $efidisk = $conf->{efidisk0
}) {
3133 my $d = parse_drive
('efidisk0', $efidisk);
3134 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3135 $format = $d->{format
};
3137 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3138 if (!defined($format)) {
3139 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3140 $format = qemu_img_format
($scfg, $volname);
3144 die "efidisk format must be specified\n"
3145 if !defined($format);
3148 warn "no efidisk configured! Using temporary efivars disk.\n";
3149 $path = "/tmp/$vmid-ovmf.fd";
3150 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3156 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3157 $size_str = ",size=" . (-s
$ovmf_vars);
3160 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3161 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3166 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3167 if (min_version
($machine_version, 4, 0)) {
3168 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3170 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3174 if ($conf->{vmgenid
}) {
3175 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3178 # add usb controllers
3179 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3180 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3181 push @$devices, @usbcontrollers if @usbcontrollers;
3182 my $vga = parse_vga
($conf->{vga
});
3184 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3185 $vga->{type
} = 'qxl' if $qxlnum;
3187 if (!$vga->{type
}) {
3188 if ($arch eq 'aarch64') {
3189 $vga->{type
} = 'virtio';
3190 } elsif (min_version
($machine_version, 2, 9)) {
3191 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3193 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3197 # enable absolute mouse coordinates (needed by vnc)
3199 if (defined($conf->{tablet
})) {
3200 $tablet = $conf->{tablet
};
3202 $tablet = $defaults->{tablet
};
3203 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3204 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3208 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3209 my $kbd = print_keyboarddevice_full
($conf, $arch);
3210 push @$devices, '-device', $kbd if defined($kbd);
3213 my $bootorder = device_bootorder
($conf);
3215 # host pci device passthrough
3216 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3217 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3220 my $usb_dev_features = {};
3221 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3223 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3224 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3225 push @$devices, @usbdevices if @usbdevices;
3228 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3229 if (my $path = $conf->{"serial$i"}) {
3230 if ($path eq 'socket') {
3231 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3232 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3233 # On aarch64, serial0 is the UART device. Qemu only allows
3234 # connecting UART devices via the '-serial' command line, as
3235 # the device has a fixed slot on the hardware...
3236 if ($arch eq 'aarch64' && $i == 0) {
3237 push @$devices, '-serial', "chardev:serial$i";
3239 push @$devices, '-device', "isa-serial,chardev=serial$i";
3242 die "no such serial device\n" if ! -c
$path;
3243 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3244 push @$devices, '-device', "isa-serial,chardev=serial$i";
3250 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3251 if (my $path = $conf->{"parallel$i"}) {
3252 die "no such parallel device\n" if ! -c
$path;
3253 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3254 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3255 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3259 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3260 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3261 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3262 push @$devices, @$audio_devs;
3266 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3267 $sockets = $conf->{sockets
} if $conf->{sockets
};
3269 my $cores = $conf->{cores
} || 1;
3271 my $maxcpus = $sockets * $cores;
3273 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3275 my $allowed_vcpus = $cpuinfo->{cpus
};
3277 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3278 if ($allowed_vcpus < $maxcpus);
3280 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3282 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3283 for (my $i = 2; $i <= $vcpus; $i++) {
3284 my $cpustr = print_cpu_device
($conf,$i);
3285 push @$cmd, '-device', $cpustr;
3290 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3292 push @$cmd, '-nodefaults';
3294 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3296 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3298 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3300 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3301 push @$devices, '-device', print_vga_device
(
3302 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3303 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3304 push @$cmd, '-vnc', "unix:$socket,password";
3306 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3307 push @$cmd, '-nographic';
3311 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3312 my $useLocaltime = $conf->{localtime};
3314 if ($winversion >= 5) { # windows
3315 $useLocaltime = 1 if !defined($conf->{localtime});
3317 # use time drift fix when acpi is enabled
3318 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3319 $tdf = 1 if !defined($conf->{tdf
});
3323 if ($winversion >= 6) {
3324 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3325 push @$cmd, '-no-hpet';
3328 push @$rtcFlags, 'driftfix=slew' if $tdf;
3330 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3331 push @$rtcFlags, "base=$conf->{startdate}";
3332 } elsif ($useLocaltime) {
3333 push @$rtcFlags, 'base=localtime';
3337 push @$cmd, '-cpu', $forcecpu;
3339 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3342 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3344 push @$cmd, '-S' if $conf->{freeze
};
3346 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3348 my $guest_agent = parse_guest_agent
($conf);
3350 if ($guest_agent->{enabled
}) {
3351 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3352 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3354 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3355 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3356 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3357 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3358 } elsif ($guest_agent->{type
} eq 'isa') {
3359 push @$devices, '-device', "isa-serial,chardev=qga0";
3363 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3364 if ($rng && $version_guard->(4, 1, 2)) {
3365 check_rng_source
($rng->{source
});
3367 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3368 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3369 my $limiter_str = "";
3371 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3374 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3375 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3376 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3384 for (my $i = 1; $i < $qxlnum; $i++){
3385 push @$devices, '-device', print_vga_device
(
3386 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3389 # assume other OS works like Linux
3390 my ($ram, $vram) = ("134217728", "67108864");
3391 if ($vga->{memory
}) {
3392 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3393 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3395 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3396 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3400 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3402 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3403 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3404 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3406 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3407 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3408 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3410 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3411 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3413 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3414 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3415 if ($spice_enhancement->{foldersharing
}) {
3416 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3417 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3420 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3421 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3422 if $spice_enhancement->{videostreaming
};
3424 push @$devices, '-spice', "$spice_opts";
3427 # enable balloon by default, unless explicitly disabled
3428 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3429 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3430 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3433 if ($conf->{watchdog
}) {
3434 my $wdopts = parse_watchdog
($conf->{watchdog
});
3435 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3436 my $watchdog = $wdopts->{model
} || 'i6300esb';
3437 push @$devices, '-device', "$watchdog$pciaddr";
3438 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3442 my $scsicontroller = {};
3443 my $ahcicontroller = {};
3444 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3446 # Add iscsi initiator name if available
3447 if (my $initiator = get_initiator_name
()) {
3448 push @$devices, '-iscsi', "initiator-name=$initiator";
3451 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3452 my ($ds, $drive) = @_;
3454 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3455 push @$vollist, $drive->{file
};
3458 # ignore efidisk here, already added in bios/fw handling code above
3459 return if $drive->{interface
} eq 'efidisk';
3461 $use_virtio = 1 if $ds =~ m/^virtio/;
3463 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3465 if ($drive->{interface
} eq 'virtio'){
3466 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3469 if ($drive->{interface
} eq 'scsi') {
3471 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3473 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3474 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3476 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3477 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3480 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3481 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3482 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3483 } elsif ($drive->{iothread
}) {
3484 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3488 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3489 $queues = ",num_queues=$drive->{queues}";
3492 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3493 if !$scsicontroller->{$controller};
3494 $scsicontroller->{$controller}=1;
3497 if ($drive->{interface
} eq 'sata') {
3498 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3499 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3500 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3501 if !$ahcicontroller->{$controller};
3502 $ahcicontroller->{$controller}=1;
3505 my $drive_cmd = print_drive_commandline_full
($storecfg, $vmid, $drive);
3506 $drive_cmd .= ',readonly' if PVE
::QemuConfig-
>is_template($conf);
3508 push @$devices, '-drive',$drive_cmd;
3509 push @$devices, '-device', print_drivedevice_full
(
3510 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3513 for (my $i = 0; $i < $MAX_NETS; $i++) {
3514 my $netname = "net$i";
3516 next if !$conf->{$netname};
3517 my $d = parse_net
($conf->{$netname});
3520 $use_virtio = 1 if $d->{model
} eq 'virtio';
3522 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
3524 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
3525 push @$devices, '-netdev', $netdevfull;
3527 my $netdevicefull = print_netdevice_full
(
3528 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3530 push @$devices, '-device', $netdevicefull;
3533 if ($conf->{ivshmem
}) {
3534 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3538 $bus = print_pcie_addr
("ivshmem");
3540 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3543 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3544 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3546 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3547 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3548 .",size=$ivshmem->{size}M";
3551 # pci.4 is nested in pci.1
3552 $bridges->{1} = 1 if $bridges->{4};
3556 if (min_version
($machine_version, 2, 3)) {
3561 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3565 for my $k (sort {$b cmp $a} keys %$bridges) {
3566 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3569 if ($k == 2 && $legacy_igd) {
3572 $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
3574 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3576 # add after -readconfig pve-q35.cfg
3577 splice @$devices, 2, 0, '-device', $devstr;
3579 unshift @$devices, '-device', $devstr if $k > 0;
3584 push @$machineFlags, 'accel=tcg';
3587 my $machine_type_min = $machine_type;
3588 if ($add_pve_version) {
3589 $machine_type_min =~ s/\+pve\d+$//;
3590 $machine_type_min .= "+pve$required_pve_version";
3592 push @$machineFlags, "type=${machine_type_min}";
3594 push @$cmd, @$devices;
3595 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3596 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3597 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3599 if (my $vmstate = $conf->{vmstate
}) {
3600 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
3601 push @$vollist, $vmstate;
3602 push @$cmd, '-loadstate', $statepath;
3603 print "activating and using '$vmstate' as vmstate\n";
3607 if ($conf->{args
}) {
3608 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3612 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3615 sub check_rng_source
{
3618 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3619 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3622 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3623 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
3624 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3625 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3626 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3627 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3635 my $res = mon_cmd
($vmid, 'query-spice');
3637 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3640 sub vm_devices_list
{
3643 my $res = mon_cmd
($vmid, 'query-pci');
3644 my $devices_to_check = [];
3646 foreach my $pcibus (@$res) {
3647 push @$devices_to_check, @{$pcibus->{devices
}},
3650 while (@$devices_to_check) {
3652 for my $d (@$devices_to_check) {
3653 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3654 next if !$d->{'pci_bridge'};
3656 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3657 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3659 $devices_to_check = $to_check;
3662 my $resblock = mon_cmd
($vmid, 'query-block');
3663 foreach my $block (@$resblock) {
3664 if($block->{device
} =~ m/^drive-(\S+)/){
3669 my $resmice = mon_cmd
($vmid, 'query-mice');
3670 foreach my $mice (@$resmice) {
3671 if ($mice->{name
} eq 'QEMU HID Tablet') {
3672 $devices->{tablet
} = 1;
3677 # for usb devices there is no query-usb
3678 # but we can iterate over the entries in
3679 # qom-list path=/machine/peripheral
3680 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3681 foreach my $per (@$resperipheral) {
3682 if ($per->{name
} =~ m/^usb\d+$/) {
3683 $devices->{$per->{name
}} = 1;
3691 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3693 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3695 my $devices_list = vm_devices_list
($vmid);
3696 return 1 if defined($devices_list->{$deviceid});
3698 # add PCI bridge if we need it for the device
3699 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3701 if ($deviceid eq 'tablet') {
3703 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
3705 } elsif ($deviceid eq 'keyboard') {
3707 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
3709 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3711 die "usb hotplug currently not reliable\n";
3712 # since we can't reliably hot unplug all added usb devices and usb
3713 # passthrough breaks live migration we disable usb hotplugging for now
3714 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3716 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3718 qemu_iothread_add
($vmid, $deviceid, $device);
3720 qemu_driveadd
($storecfg, $vmid, $device);
3721 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3723 qemu_deviceadd
($vmid, $devicefull);
3724 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3726 eval { qemu_drivedel
($vmid, $deviceid); };
3731 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3734 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3735 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3736 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3738 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3740 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3741 qemu_iothread_add
($vmid, $deviceid, $device);
3742 $devicefull .= ",iothread=iothread-$deviceid";
3745 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3746 $devicefull .= ",num_queues=$device->{queues}";
3749 qemu_deviceadd
($vmid, $devicefull);
3750 qemu_deviceaddverify
($vmid, $deviceid);
3752 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3754 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3755 qemu_driveadd
($storecfg, $vmid, $device);
3757 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3758 eval { qemu_deviceadd
($vmid, $devicefull); };
3760 eval { qemu_drivedel
($vmid, $deviceid); };
3765 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3767 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
3769 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
3770 my $use_old_bios_files = undef;
3771 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3773 my $netdevicefull = print_netdevice_full
(
3774 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3775 qemu_deviceadd
($vmid, $netdevicefull);
3777 qemu_deviceaddverify
($vmid, $deviceid);
3778 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
3781 eval { qemu_netdevdel
($vmid, $deviceid); };
3786 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3789 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
3790 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3792 qemu_deviceadd
($vmid, $devicefull);
3793 qemu_deviceaddverify
($vmid, $deviceid);
3796 die "can't hotplug device '$deviceid'\n";
3802 # fixme: this should raise exceptions on error!
3803 sub vm_deviceunplug
{
3804 my ($vmid, $conf, $deviceid) = @_;
3806 my $devices_list = vm_devices_list
($vmid);
3807 return 1 if !defined($devices_list->{$deviceid});
3809 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
3810 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3812 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3814 qemu_devicedel
($vmid, $deviceid);
3816 } elsif ($deviceid =~ m/^usb\d+$/) {
3818 die "usb hotplug currently not reliable\n";
3819 # when unplugging usb devices this way, there may be remaining usb
3820 # controllers/hubs so we disable it for now
3821 #qemu_devicedel($vmid, $deviceid);
3822 #qemu_devicedelverify($vmid, $deviceid);
3824 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3826 qemu_devicedel
($vmid, $deviceid);
3827 qemu_devicedelverify
($vmid, $deviceid);
3828 qemu_drivedel
($vmid, $deviceid);
3829 qemu_iothread_del
($conf, $vmid, $deviceid);
3831 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3833 qemu_devicedel
($vmid, $deviceid);
3834 qemu_devicedelverify
($vmid, $deviceid);
3835 qemu_iothread_del
($conf, $vmid, $deviceid);
3837 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3839 qemu_devicedel
($vmid, $deviceid);
3840 qemu_drivedel
($vmid, $deviceid);
3841 qemu_deletescsihw
($conf, $vmid, $deviceid);
3843 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3845 qemu_devicedel
($vmid, $deviceid);
3846 qemu_devicedelverify
($vmid, $deviceid);
3847 qemu_netdevdel
($vmid, $deviceid);
3850 die "can't unplug device '$deviceid'\n";
3856 sub qemu_deviceadd
{
3857 my ($vmid, $devicefull) = @_;
3859 $devicefull = "driver=".$devicefull;
3860 my %options = split(/[=,]/, $devicefull);
3862 mon_cmd
($vmid, "device_add" , %options);
3865 sub qemu_devicedel
{
3866 my ($vmid, $deviceid) = @_;
3868 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
3871 sub qemu_iothread_add
{
3872 my($vmid, $deviceid, $device) = @_;
3874 if ($device->{iothread
}) {
3875 my $iothreads = vm_iothreads_list
($vmid);
3876 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3880 sub qemu_iothread_del
{
3881 my($conf, $vmid, $deviceid) = @_;
3883 my $confid = $deviceid;
3884 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3885 $confid = 'scsi' . $1;
3887 my $device = parse_drive
($confid, $conf->{$confid});
3888 if ($device->{iothread
}) {
3889 my $iothreads = vm_iothreads_list
($vmid);
3890 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3894 sub qemu_objectadd
{
3895 my($vmid, $objectid, $qomtype) = @_;
3897 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
3902 sub qemu_objectdel
{
3903 my($vmid, $objectid) = @_;
3905 mon_cmd
($vmid, "object-del", id
=> $objectid);
3911 my ($storecfg, $vmid, $device) = @_;
3913 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device);
3914 $drive =~ s/\\/\\\\/g;
3915 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
3917 # If the command succeeds qemu prints: "OK
"
3918 return 1 if $ret =~ m/OK/s;
3920 die "adding drive failed
: $ret\n";
3924 my($vmid, $deviceid) = @_;
3926 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
3929 return 1 if $ret eq "";
3931 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3932 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3934 die "deleting drive
$deviceid failed
: $ret\n";
3937 sub qemu_deviceaddverify {
3938 my ($vmid, $deviceid) = @_;
3940 for (my $i = 0; $i <= 5; $i++) {
3941 my $devices_list = vm_devices_list($vmid);
3942 return 1 if defined($devices_list->{$deviceid});
3946 die "error on hotplug device
'$deviceid'\n";
3950 sub qemu_devicedelverify {
3951 my ($vmid, $deviceid) = @_;
3953 # need to verify that the device is correctly removed as device_del
3954 # is async and empty return is not reliable
3956 for (my $i = 0; $i <= 5; $i++) {
3957 my $devices_list = vm_devices_list($vmid);
3958 return 1 if !defined($devices_list->{$deviceid});
3962 die "error on hot-unplugging device
'$deviceid'\n";
3965 sub qemu_findorcreatescsihw {
3966 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3968 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3970 my $scsihwid="$controller_prefix$controller";
3971 my $devices_list = vm_devices_list($vmid);
3973 if(!defined($devices_list->{$scsihwid})) {
3974 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3980 sub qemu_deletescsihw {
3981 my ($conf, $vmid, $opt) = @_;
3983 my $device = parse_drive($opt, $conf->{$opt});
3985 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3986 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
3990 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3992 my $devices_list = vm_devices_list($vmid);
3993 foreach my $opt (keys %{$devices_list}) {
3994 if (is_valid_drivename($opt)) {
3995 my $drive = parse_drive($opt, $conf->{$opt});
3996 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4002 my $scsihwid="scsihw
$controller";
4004 vm_deviceunplug($vmid, $conf, $scsihwid);
4009 sub qemu_add_pci_bridge {
4010 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4016 print_pci_addr($device, $bridges, $arch, $machine_type);
4018 while (my ($k, $v) = each %$bridges) {
4021 return 1 if !defined($bridgeid) || $bridgeid < 1;
4023 my $bridge = "pci
.$bridgeid";
4024 my $devices_list = vm_devices_list($vmid);
4026 if (!defined($devices_list->{$bridge})) {
4027 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4033 sub qemu_set_link_status {
4034 my ($vmid, $device, $up) = @_;
4036 mon_cmd($vmid, "set_link
", name => $device,
4037 up => $up ? JSON::true : JSON::false);
4040 sub qemu_netdevadd {
4041 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4043 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4044 my %options = split(/[=,]/, $netdev);
4046 if (defined(my $vhost = $options{vhost})) {
4047 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4050 if (defined(my $queues = $options{queues})) {
4051 $options{queues} = $queues + 0;
4054 mon_cmd($vmid, "netdev_add
", %options);
4058 sub qemu_netdevdel {
4059 my ($vmid, $deviceid) = @_;
4061 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4064 sub qemu_usb_hotplug {
4065 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4069 # remove the old one first
4070 vm_deviceunplug($vmid, $conf, $deviceid);
4072 # check if xhci controller is necessary and available
4073 if ($device->{usb3}) {
4075 my $devicelist = vm_devices_list($vmid);
4077 if (!$devicelist->{xhci}) {
4078 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4079 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4082 my $d = parse_usb_device($device->{host});
4083 $d->{usb3} = $device->{usb3};
4086 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4089 sub qemu_cpu_hotplug {
4090 my ($vmid, $conf, $vcpus) = @_;
4092 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4095 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4096 $sockets = $conf->{sockets} if $conf->{sockets};
4097 my $cores = $conf->{cores} || 1;
4098 my $maxcpus = $sockets * $cores;
4100 $vcpus = $maxcpus if !$vcpus;
4102 die "you can
't add more vcpus than maxcpus\n"
4103 if $vcpus > $maxcpus;
4105 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4107 if ($vcpus < $currentvcpus) {
4109 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4111 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4112 qemu_devicedel($vmid, "cpu$i");
4114 my $currentrunningvcpus = undef;
4116 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4117 last if scalar(@{$currentrunningvcpus}) == $i-1;
4118 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4122 #update conf after each succesfull cpu unplug
4123 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4124 PVE::QemuConfig->write_config($vmid, $conf);
4127 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4133 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4134 die "vcpus in running vm does not match its configuration\n"
4135 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4137 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4139 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4140 my $cpustr = print_cpu_device($conf, $i);
4141 qemu_deviceadd($vmid, $cpustr);
4144 my $currentrunningvcpus = undef;
4146 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4147 last if scalar(@{$currentrunningvcpus}) == $i;
4148 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4152 #update conf after each succesfull cpu hotplug
4153 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4154 PVE::QemuConfig->write_config($vmid, $conf);
4158 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4159 mon_cmd($vmid, "cpu-add", id => int($i));
4164 sub qemu_block_set_io_throttle {
4165 my ($vmid, $deviceid,
4166 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4167 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4168 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4169 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4171 return if !check_running($vmid) ;
4173 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4175 bps_rd => int($bps_rd),
4176 bps_wr => int($bps_wr),
4178 iops_rd => int($iops_rd),
4179 iops_wr => int($iops_wr),
4180 bps_max => int($bps_max),
4181 bps_rd_max => int($bps_rd_max),
4182 bps_wr_max => int($bps_wr_max),
4183 iops_max => int($iops_max),
4184 iops_rd_max => int($iops_rd_max),
4185 iops_wr_max => int($iops_wr_max),
4186 bps_max_length => int($bps_max_length),
4187 bps_rd_max_length => int($bps_rd_max_length),
4188 bps_wr_max_length => int($bps_wr_max_length),
4189 iops_max_length => int($iops_max_length),
4190 iops_rd_max_length => int($iops_rd_max_length),
4191 iops_wr_max_length => int($iops_wr_max_length),
4196 sub qemu_block_resize {
4197 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4199 my $running = check_running($vmid);
4201 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4203 return if !$running;
4205 my $padding = (1024 - $size % 1024) % 1024;
4206 $size = $size + $padding;
4208 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4212 sub qemu_volume_snapshot {
4213 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4215 my $running = check_running($vmid);
4217 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4218 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4220 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4224 sub qemu_volume_snapshot_delete {
4225 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4227 my $running = check_running($vmid);
4232 my $conf = PVE::QemuConfig->load_config($vmid);
4233 PVE::QemuConfig->foreach_volume($conf, sub {
4234 my ($ds, $drive) = @_;
4235 $running = 1 if $drive->{file} eq $volid;
4239 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4240 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4242 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4246 sub set_migration_caps {
4252 "auto-converge" => 1,
4254 "x-rdma-pin-all" => 0,
4259 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4261 for my $supported_capability (@$supported_capabilities) {
4263 capability => $supported_capability->{capability},
4264 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4268 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4272 my ($conf, $func, @param) = @_;
4276 my $test_volid = sub {
4277 my ($key, $drive, $snapname) = @_;
4279 my $volid = $drive->{file};
4282 $volhash->{$volid}->{cdrom} //= 1;
4283 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4285 my $replicate = $drive->{replicate} // 1;
4286 $volhash->{$volid}->{replicate} //= 0;
4287 $volhash->{$volid}->{replicate} = 1 if $replicate;
4289 $volhash->{$volid}->{shared} //= 0;
4290 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4292 $volhash->{$volid}->{referenced_in_config} //= 0;
4293 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4295 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4296 if defined($snapname);
4298 my $size = $drive->{size};
4299 $volhash->{$volid}->{size} //= $size if $size;
4301 $volhash->{$volid}->{is_vmstate} //= 0;
4302 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4304 $volhash->{$volid}->{is_unused} //= 0;
4305 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4308 my $include_opts = {
4309 extra_keys => ['vmstate
'],
4310 include_unused => 1,
4313 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4314 foreach my $snapname (keys %{$conf->{snapshots}}) {
4315 my $snap = $conf->{snapshots}->{$snapname};
4316 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4319 foreach my $volid (keys %$volhash) {
4320 &$func($volid, $volhash->{$volid}, @param);
4324 my $fast_plug_option = {
4332 'vmstatestorage
' => 1,
4337 # hotplug changes in [PENDING]
4338 # $selection hash can be used to only apply specified options, for
4339 # example: { cores => 1 } (only apply changed 'cores
')
4340 # $errors ref is used to return error messages
4341 sub vmconfig_hotplug_pending {
4342 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4344 my $defaults = load_defaults();
4345 my $arch = get_vm_arch($conf);
4346 my $machine_type = get_vm_machine($conf, undef, $arch);
4348 # commit values which do not have any impact on running VM first
4349 # Note: those option cannot raise errors, we we do not care about
4350 # $selection and always apply them.
4352 my $add_error = sub {
4353 my ($opt, $msg) = @_;
4354 $errors->{$opt} = "hotplug problem - $msg";
4358 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4359 if ($fast_plug_option->{$opt}) {
4360 $conf->{$opt} = $conf->{pending}->{$opt};
4361 delete $conf->{pending}->{$opt};
4367 PVE::QemuConfig->write_config($vmid, $conf);
4370 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4372 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4373 foreach my $opt (sort keys %$pending_delete_hash) {
4374 next if $selection && !$selection->{$opt};
4375 my $force = $pending_delete_hash->{$opt}->{force};
4377 if ($opt eq 'hotplug
') {
4378 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4379 } elsif ($opt eq 'tablet
') {
4380 die "skip\n" if !$hotplug_features->{usb};
4381 if ($defaults->{tablet}) {
4382 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4383 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4384 if $arch eq 'aarch64
';
4386 vm_deviceunplug($vmid, $conf, 'tablet
');
4387 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4389 } elsif ($opt =~ m/^usb\d+/) {
4391 # since we cannot reliably hot unplug usb devices we are disabling it
4392 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4393 #vm_deviceunplug($vmid, $conf, $opt);
4394 } elsif ($opt eq 'vcpus
') {
4395 die "skip\n" if !$hotplug_features->{cpu};
4396 qemu_cpu_hotplug($vmid, $conf, undef);
4397 } elsif ($opt eq 'balloon
') {
4398 # enable balloon device is not hotpluggable
4399 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4400 # here we reset the ballooning value to memory
4401 my $balloon = $conf->{memory} || $defaults->{memory};
4402 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4403 } elsif ($fast_plug_option->{$opt}) {
4405 } elsif ($opt =~ m/^net(\d+)$/) {
4406 die "skip\n" if !$hotplug_features->{network};
4407 vm_deviceunplug($vmid, $conf, $opt);
4408 } elsif (is_valid_drivename($opt)) {
4409 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4410 vm_deviceunplug($vmid, $conf, $opt);
4411 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4412 } elsif ($opt =~ m/^memory$/) {
4413 die "skip\n" if !$hotplug_features->{memory};
4414 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4415 } elsif ($opt eq 'cpuunits
') {
4416 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4417 } elsif ($opt eq 'cpulimit
') {
4418 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4424 &$add_error($opt, $err) if $err ne "skip\n";
4426 delete $conf->{$opt};
4427 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4431 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4432 $apply_pending_cloudinit = sub {
4433 return if $apply_pending_cloudinit_done; # once is enough
4434 $apply_pending_cloudinit_done = 1; # once is enough
4436 my ($key, $value) = @_;
4438 my @cloudinit_opts = keys %$confdesc_cloudinit;
4439 foreach my $opt (keys %{$conf->{pending}}) {
4440 next if !grep { $_ eq $opt } @cloudinit_opts;
4441 $conf->{$opt} = delete $conf->{pending}->{$opt};
4444 my $new_conf = { %$conf };
4445 $new_conf->{$key} = $value;
4446 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4449 foreach my $opt (keys %{$conf->{pending}}) {
4450 next if $selection && !$selection->{$opt};
4451 my $value = $conf->{pending}->{$opt};
4453 if ($opt eq 'hotplug
') {
4454 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4455 } elsif ($opt eq 'tablet
') {
4456 die "skip\n" if !$hotplug_features->{usb};
4458 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4459 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4460 if $arch eq 'aarch64
';
4461 } elsif ($value == 0) {
4462 vm_deviceunplug($vmid, $conf, 'tablet
');
4463 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4465 } elsif ($opt =~ m/^usb\d+$/) {
4467 # since we cannot reliably hot unplug usb devices we disable it for now
4468 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4469 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4470 #die "skip\n" if !$d;
4471 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4472 } elsif ($opt eq 'vcpus
') {
4473 die "skip\n" if !$hotplug_features->{cpu};
4474 qemu_cpu_hotplug($vmid, $conf, $value);
4475 } elsif ($opt eq 'balloon
') {
4476 # enable/disable balloning device is not hotpluggable
4477 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4478 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4479 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4481 # allow manual ballooning if shares is set to zero
4482 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4483 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4484 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4486 } elsif ($opt =~ m/^net(\d+)$/) {
4487 # some changes can be done without hotplug
4488 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4489 $vmid, $opt, $value, $arch, $machine_type);
4490 } elsif (is_valid_drivename($opt)) {
4491 die "skip\n" if $opt eq 'efidisk0
';
4492 # some changes can be done without hotplug
4493 my $drive = parse_drive($opt, $value);
4494 if (drive_is_cloudinit($drive)) {
4495 &$apply_pending_cloudinit($opt, $value);
4497 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4498 $vmid, $opt, $value, $arch, $machine_type);
4499 } elsif ($opt =~ m/^memory$/) { #dimms
4500 die "skip\n" if !$hotplug_features->{memory};
4501 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4502 } elsif ($opt eq 'cpuunits
') {
4503 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4504 } elsif ($opt eq 'cpulimit
') {
4505 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4506 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4508 die "skip\n"; # skip non-hot-pluggable options
4512 &$add_error($opt, $err) if $err ne "skip\n";
4514 $conf->{$opt} = $value;
4515 delete $conf->{pending}->{$opt};
4519 PVE::QemuConfig->write_config($vmid, $conf);
4522 sub try_deallocate_drive {
4523 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4525 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4526 my $volid = $drive->{file};
4527 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4528 my $sid = PVE::Storage::parse_volume_id($volid);
4529 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4531 # check if the disk is really unused
4532 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4533 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4534 PVE::Storage::vdisk_free($storecfg, $volid);
4537 # If vm is not owner of this disk remove from config
4545 sub vmconfig_delete_or_detach_drive {
4546 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4548 my $drive = parse_drive($opt, $conf->{$opt});
4550 my $rpcenv = PVE::RPCEnvironment::get();
4551 my $authuser = $rpcenv->get_user();
4554 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4555 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4557 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4563 sub vmconfig_apply_pending {
4564 my ($vmid, $conf, $storecfg, $errors) = @_;
4566 my $add_apply_error = sub {
4567 my ($opt, $msg) = @_;
4568 my $err_msg = "unable to apply pending change $opt : $msg";
4569 $errors->{$opt} = $err_msg;
4575 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4576 foreach my $opt (sort keys %$pending_delete_hash) {
4577 my $force = $pending_delete_hash->{$opt}->{force};
4579 if ($opt =~ m/^unused/) {
4580 die "internal error";
4581 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4582 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4586 $add_apply_error->($opt, $err);
4588 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4589 delete $conf->{$opt};
4593 PVE::QemuConfig->cleanup_pending($conf);
4595 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4596 next if $opt eq 'delete'; # just to be sure
4598 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4599 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4603 $add_apply_error->($opt, $err);
4605 $conf->{$opt} = delete $conf->{pending}->{$opt};
4609 # write all changes at once to avoid unnecessary i/o
4610 PVE::QemuConfig->write_config($vmid, $conf);
4613 sub vmconfig_update_net {
4614 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4616 my $newnet = parse_net($value);
4618 if ($conf->{$opt}) {
4619 my $oldnet = parse_net($conf->{$opt});
4621 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4622 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4623 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4624 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4626 # for non online change, we try to hot-unplug
4627 die "skip\n" if !$hotplug;
4628 vm_deviceunplug($vmid, $conf, $opt);
4631 die "internal error" if $opt !~ m/net(\d+)/;
4632 my $iface = "tap${vmid}i$1";
4634 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4635 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4636 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4637 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4638 PVE::Network::tap_unplug($iface);
4641 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4643 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4645 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4646 # Rate can be applied on its own but any change above needs to
4647 # include the rate in tap_plug since OVS resets everything.
4648 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4651 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4652 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4660 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4666 sub vmconfig_update_disk {
4667 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4669 my $drive = parse_drive($opt, $value);
4671 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4672 my $media = $drive->{media} || 'disk
';
4673 my $oldmedia = $old_drive->{media} || 'disk
';
4674 die "unable to change media type\n" if $media ne $oldmedia;
4676 if (!drive_is_cdrom($old_drive)) {
4678 if ($drive->{file} ne $old_drive->{file}) {
4680 die "skip\n" if !$hotplug;
4682 # unplug and register as unused
4683 vm_deviceunplug($vmid, $conf, $opt);
4684 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4687 # update existing disk
4689 # skip non hotpluggable value
4690 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4691 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4692 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4693 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4694 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4699 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4700 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4701 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4702 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4703 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4704 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4705 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4706 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4707 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4708 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4709 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4710 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4711 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4712 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4713 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4714 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4715 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4716 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4718 qemu_block_set_io_throttle(
4720 ($drive->{mbps} || 0)*1024*1024,
4721 ($drive->{mbps_rd} || 0)*1024*1024,
4722 ($drive->{mbps_wr} || 0)*1024*1024,
4723 $drive->{iops} || 0,
4724 $drive->{iops_rd} || 0,
4725 $drive->{iops_wr} || 0,
4726 ($drive->{mbps_max} || 0)*1024*1024,
4727 ($drive->{mbps_rd_max} || 0)*1024*1024,
4728 ($drive->{mbps_wr_max} || 0)*1024*1024,
4729 $drive->{iops_max} || 0,
4730 $drive->{iops_rd_max} || 0,
4731 $drive->{iops_wr_max} || 0,
4732 $drive->{bps_max_length} || 1,
4733 $drive->{bps_rd_max_length} || 1,
4734 $drive->{bps_wr_max_length} || 1,
4735 $drive->{iops_max_length} || 1,
4736 $drive->{iops_rd_max_length} || 1,
4737 $drive->{iops_wr_max_length} || 1,
4747 if ($drive->{file} eq 'none
') {
4748 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4749 if (drive_is_cloudinit($old_drive)) {
4750 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4753 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4755 # force eject if locked
4756 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4759 mon_cmd($vmid, "blockdev-change-medium",
4760 id => "$opt", filename => "$path");
4768 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4770 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4771 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4774 # called in locked context by incoming migration
4775 sub vm_migrate_get_nbd_disks {
4776 my ($storecfg, $conf, $replicated_volumes) = @_;
4778 my $local_volumes = {};
4779 PVE::QemuConfig->foreach_volume($conf, sub {
4780 my ($ds, $drive) = @_;
4782 return if drive_is_cdrom($drive);
4784 my $volid = $drive->{file};
4788 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4790 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4791 return if $scfg->{shared};
4793 # replicated disks re-use existing state via bitmap
4794 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4795 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4797 return $local_volumes;
4800 # called in locked context by incoming migration
4801 sub vm_migrate_alloc_nbd_disks {
4802 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4807 foreach my $opt (sort keys %$source_volumes) {
4808 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4810 if ($use_existing) {
4811 $nbd->{$opt}->{drivestr} = print_drive($drive);
4812 $nbd->{$opt}->{volid} = $volid;
4813 $nbd->{$opt}->{replicated} = 1;
4817 # If a remote storage is specified and the format of the original
4818 # volume is not available there, fall back to the default format.
4819 # Otherwise use the same format as the original.
4820 if (!$storagemap->{identity}) {
4821 $storeid = map_storage($storagemap, $storeid);
4822 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4823 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4824 my $fileFormat = qemu_img_format($scfg, $volname);
4825 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4827 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4828 $format = qemu_img_format($scfg, $volname);
4831 my $size = $drive->{size} / 1024;
4832 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4833 my $newdrive = $drive;
4834 $newdrive->{format} = $format;
4835 $newdrive->{file} = $newvolid;
4836 my $drivestr = print_drive($newdrive);
4837 $nbd->{$opt}->{drivestr} = $drivestr;
4838 $nbd->{$opt}->{volid} = $newvolid;
4844 # see vm_start_nolock for parameters, additionally:
4846 # storagemap = parsed storage map for allocating NBD disks
4848 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4850 return PVE::QemuConfig->lock_config($vmid, sub {
4851 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4853 die "you can't start a vm
if it
's a template\n"
4854 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4856 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended
');
4858 PVE::QemuConfig->check_lock($conf)
4859 if !($params->{skiplock} || $has_suspended_lock);
4861 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4863 die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4865 if (my $storagemap = $migrate_opts->{storagemap}) {
4866 my $replicated = $migrate_opts->{replicated_volumes};
4867 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4868 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4870 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4871 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4875 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4881 # statefile => 'tcp
', 'unix
' for migration or path/volid for RAM state
4882 # skiplock => 0/1, skip checking for config lock
4883 # skiptemplate => 0/1, skip checking whether VM is template
4884 # forcemachine => to force Qemu machine (rollback/migration)
4885 # forcecpu => a QEMU '-cpu
' argument string to override get_cpu_options
4886 # timeout => in seconds
4887 # paused => start VM in paused state (backup)
4888 # resume => resume from hibernation
4890 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4891 # migratedfrom => source node
4892 # spice_ticket => used for spice migration, passed via tunnel/stdin
4893 # network => CIDR of migration network
4894 # type => secure/insecure - tunnel over encrypted connection or plain-text
4895 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
4896 # replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4897 sub vm_start_nolock {
4898 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4900 my $statefile = $params->{statefile};
4901 my $resume = $params->{resume};
4903 my $migratedfrom = $migrate_opts->{migratedfrom};
4904 my $migration_type = $migrate_opts->{type};
4908 # clean up leftover reboot request files
4909 eval { clear_reboot_request($vmid); };
4912 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4913 vmconfig_apply_pending($vmid, $conf, $storecfg);
4914 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4917 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4919 my $defaults = load_defaults();
4921 # set environment variable useful inside network script
4922 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4924 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
4926 my $forcemachine = $params->{forcemachine};
4927 my $forcecpu = $params->{forcecpu};
4929 # enforce machine and CPU type on suspended vm to ensure HW compatibility
4930 $forcemachine = $conf->{runningmachine};
4931 $forcecpu = $conf->{runningcpu};
4932 print "Resuming suspended VM\n";
4935 my ($cmd, $vollist, $spice_port) =
4936 config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4939 my $get_migration_ip = sub {
4940 my ($nodename) = @_;
4942 return $migration_ip if defined($migration_ip);
4944 my $cidr = $migrate_opts->{network};
4946 if (!defined($cidr)) {
4947 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
4948 $cidr = $dc_conf->{migration}->{network};
4951 if (defined($cidr)) {
4952 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4954 die "could not get IP: no address configured on local " .
4955 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4957 die "could not get IP: multiple addresses configured on local " .
4958 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4960 $migration_ip = @$ips[0];
4963 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4964 if !defined($migration_ip);
4966 return $migration_ip;
4971 if ($statefile eq 'tcp
') {
4972 my $localip = "localhost";
4973 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
4974 my $nodename = nodename();
4976 if (!defined($migration_type)) {
4977 if (defined($datacenterconf->{migration}->{type})) {
4978 $migration_type = $datacenterconf->{migration}->{type};
4980 $migration_type = 'secure
';
4984 if ($migration_type eq 'insecure
') {
4985 $localip = $get_migration_ip->($nodename);
4986 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4989 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4990 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4991 $migrate_uri = "tcp:${localip}:${migrate_port}";
4992 push @$cmd, '-incoming
', $migrate_uri;
4995 } elsif ($statefile eq 'unix
') {
4996 # should be default for secure migrations as a ssh TCP forward
4997 # tunnel is not deterministic reliable ready and fails regurarly
4998 # to set up in time, so use UNIX socket forwards
4999 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5000 unlink $socket_addr;
5002 $migrate_uri = "unix:$socket_addr";
5004 push @$cmd, '-incoming
', $migrate_uri;
5007 } elsif (-e $statefile) {
5008 push @$cmd, '-loadstate
', $statefile;
5010 my $statepath = PVE::Storage::path($storecfg, $statefile);
5011 push @$vollist, $statefile;
5012 push @$cmd, '-loadstate
', $statepath;
5014 } elsif ($params->{paused}) {
5019 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
5020 my $d = parse_hostpci($conf->{"hostpci$i"});
5022 my $pcidevices = $d->{pciid};
5023 foreach my $pcidevice (@$pcidevices) {
5024 my $pciid = $pcidevice->{id};
5026 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5027 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5028 die "no pci device info for device '$pciid'\n" if !$info;
5031 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5032 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5034 die "can't unbind
/bind PCI group to VFIO
'$pciid'\n"
5035 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5036 die "can
't reset PCI device '$pciid'\n"
5037 if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5042 PVE::Storage::activate_volumes($storecfg, $vollist);
5045 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5046 outfunc => sub {}, errfunc => sub {});
5048 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5049 # timeout should be more than enough here...
5050 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5052 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5053 : $defaults->{cpuunits};
5055 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5057 timeout => $statefile ? undef : $start_timeout,
5062 # when migrating, prefix QEMU output so other side can pick up any
5063 # errors that might occur and show the user
5064 if ($migratedfrom) {
5065 $run_params{quiet} = 1;
5066 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5070 Slice => 'qemu
.slice
',
5072 CPUShares => $cpuunits
5075 if (my $cpulimit = $conf->{cpulimit}) {
5076 $properties{CPUQuota} = int($cpulimit * 100);
5078 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5080 my $run_qemu = sub {
5081 PVE::Tools::run_fork sub {
5082 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5084 my $exitcode = run_command($cmd, %run_params);
5085 die "QEMU exited with code $exitcode\n" if $exitcode;
5089 if ($conf->{hugepages}) {
5092 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5093 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5095 PVE::QemuServer::Memory::hugepages_mount();
5096 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5098 eval { $run_qemu->() };
5100 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5101 if !$conf->{keephugepages};
5105 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5106 if !$conf->{keephugepages};
5108 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5111 eval { $run_qemu->() };
5115 # deactivate volumes if start fails
5116 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5117 die "start failed: $err";
5120 print "migration listens on $migrate_uri\n" if $migrate_uri;
5121 $res->{migrate_uri} = $migrate_uri;
5123 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5124 eval { mon_cmd($vmid, "cont"); };
5128 #start nbd server for storage migration
5129 if (my $nbd = $migrate_opts->{nbd}) {
5130 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5132 my $migrate_storage_uri;
5133 # nbd_protocol_version > 0 for unix socket support
5134 if ($nbd_protocol_version > 0 && $migration_type eq 'secure
') {
5135 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5136 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix
', data => { path => $socket_path } } );
5137 $migrate_storage_uri = "nbd:unix:$socket_path";
5139 my $nodename = nodename();
5140 my $localip = $get_migration_ip->($nodename);
5141 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5142 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5144 mon_cmd($vmid, "nbd-server-start", addr => {
5147 host => "${localip}",
5148 port => "${storage_migrate_port}",
5151 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5152 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5155 $res->{migrate_storage_uri} = $migrate_storage_uri;
5157 foreach my $opt (sort keys %$nbd) {
5158 my $drivestr = $nbd->{$opt}->{drivestr};
5159 my $volid = $nbd->{$opt}->{volid};
5160 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5161 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5162 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5163 print "re-using replicated volume: $opt - $volid\n"
5164 if $nbd->{$opt}->{replicated};
5166 $res->{drives}->{$opt} = $nbd->{$opt};
5167 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5171 if ($migratedfrom) {
5173 set_migration_caps($vmid);
5178 print "spice listens on port $spice_port\n";
5179 $res->{spice_port} = $spice_port;
5180 if ($migrate_opts->{spice_ticket}) {
5181 mon_cmd($vmid, "set_password", protocol => 'spice
', password =>
5182 $migrate_opts->{spice_ticket});
5183 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5188 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5189 if !$statefile && $conf->{balloon};
5191 foreach my $opt (keys %$conf) {
5192 next if $opt !~ m/^net\d+$/;
5193 my $nicconf = parse_net($conf->{$opt});
5194 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5198 mon_cmd($vmid, 'qom-set
',
5199 path => "machine/peripheral/balloon0",
5200 property => "guest-stats-polling-interval",
5201 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5204 print "Resumed VM, removing state\n";
5205 if (my $vmstate = $conf->{vmstate}) {
5206 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5207 PVE::Storage::vdisk_free($storecfg, $vmstate);
5209 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5210 PVE
::QemuConfig-
>write_config($vmid, $conf);
5213 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5218 sub vm_commandline
{
5219 my ($storecfg, $vmid, $snapname) = @_;
5221 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5226 my $snapshot = $conf->{snapshots
}->{$snapname};
5227 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5229 # check for machine or CPU overrides in snapshot
5230 $forcemachine = $snapshot->{runningmachine
};
5231 $forcecpu = $snapshot->{runningcpu
};
5233 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5238 my $defaults = load_defaults
();
5240 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults,
5241 $forcemachine, $forcecpu);
5243 return PVE
::Tools
::cmd2string
($cmd);
5247 my ($vmid, $skiplock) = @_;
5249 PVE
::QemuConfig-
>lock_config($vmid, sub {
5251 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5253 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5255 mon_cmd
($vmid, "system_reset");
5259 sub get_vm_volumes
{
5263 foreach_volid
($conf, sub {
5264 my ($volid, $attr) = @_;
5266 return if $volid =~ m
|^/|;
5268 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5271 push @$vollist, $volid;
5277 sub vm_stop_cleanup
{
5278 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5283 my $vollist = get_vm_volumes
($conf);
5284 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5287 foreach my $ext (qw(mon qmp pid vnc qga)) {
5288 unlink "/var/run/qemu-server/${vmid}.$ext";
5291 if ($conf->{ivshmem
}) {
5292 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5293 # just delete it for now, VMs which have this already open do not
5294 # are affected, but new VMs will get a separated one. If this
5295 # becomes an issue we either add some sort of ref-counting or just
5296 # add a "don't delete on stop" flag to the ivshmem format.
5297 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5300 foreach my $key (keys %$conf) {
5301 next if $key !~ m/^hostpci(\d+)$/;
5302 my $hostpciindex = $1;
5303 my $d = parse_hostpci
($conf->{$key});
5304 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5306 foreach my $pci (@{$d->{pciid
}}) {
5307 my $pciid = $pci->{id
};
5308 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5312 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5314 warn $@ if $@; # avoid errors - just warn
5317 # call only in locked context
5319 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5321 my $pid = check_running
($vmid, $nocheck);
5326 $conf = PVE
::QemuConfig-
>load_config($vmid);
5327 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5328 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5329 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5330 $timeout = $opts->{down
} if $opts->{down
};
5332 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5337 if (defined($conf) && parse_guest_agent
($conf)->{enabled
}) {
5338 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5340 mon_cmd
($vmid, "system_powerdown");
5343 mon_cmd
($vmid, "quit");
5349 $timeout = 60 if !defined($timeout);
5352 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5357 if ($count >= $timeout) {
5359 warn "VM still running - terminating now with SIGTERM\n";
5362 die "VM quit/powerdown failed - got timeout\n";
5365 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5369 if (!check_running
($vmid, $nocheck)) {
5370 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5374 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5377 die "VM quit/powerdown failed\n";
5385 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5390 if ($count >= $timeout) {
5391 warn "VM still running - terminating now with SIGKILL\n";
5396 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5399 # Note: use $nocheck to skip tests if VM configuration file exists.
5400 # We need that when migration VMs to other nodes (files already moved)
5401 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5403 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5405 $force = 1 if !defined($force) && !$shutdown;
5408 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5409 kill 15, $pid if $pid;
5410 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5411 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5415 PVE
::QemuConfig-
>lock_config($vmid, sub {
5416 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5421 my ($vmid, $timeout) = @_;
5423 PVE
::QemuConfig-
>lock_config($vmid, sub {
5426 # only reboot if running, as qmeventd starts it again on a stop event
5427 return if !check_running
($vmid);
5429 create_reboot_request
($vmid);
5431 my $storecfg = PVE
::Storage
::config
();
5432 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5436 # avoid that the next normal shutdown will be confused for a reboot
5437 clear_reboot_request
($vmid);
5443 # note: if using the statestorage parameter, the caller has to check privileges
5445 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5452 PVE
::QemuConfig-
>lock_config($vmid, sub {
5454 $conf = PVE
::QemuConfig-
>load_config($vmid);
5456 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5457 PVE
::QemuConfig-
>check_lock($conf)
5458 if !($skiplock || $is_backing_up);
5460 die "cannot suspend to disk during backup\n"
5461 if $is_backing_up && $includestate;
5463 if ($includestate) {
5464 $conf->{lock} = 'suspending';
5465 my $date = strftime
("%Y-%m-%d", localtime(time()));
5466 $storecfg = PVE
::Storage
::config
();
5467 if (!$statestorage) {
5468 $statestorage = find_vmstate_storage
($conf, $storecfg);
5469 # check permissions for the storage
5470 my $rpcenv = PVE
::RPCEnvironment
::get
();
5471 if ($rpcenv->{type
} ne 'cli') {
5472 my $authuser = $rpcenv->get_user();
5473 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5478 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
5479 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5480 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5481 PVE
::QemuConfig-
>write_config($vmid, $conf);
5483 mon_cmd
($vmid, "stop");
5487 if ($includestate) {
5489 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5492 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5494 my $state = mon_cmd
($vmid, "query-savevm");
5495 if (!$state->{status
}) {
5496 die "savevm not active\n";
5497 } elsif ($state->{status
} eq 'active') {
5500 } elsif ($state->{status
} eq 'completed') {
5501 print "State saved, quitting\n";
5503 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5504 die "query-savevm failed with error '$state->{error}'\n"
5506 die "query-savevm returned status '$state->{status}'\n";
5512 PVE
::QemuConfig-
>lock_config($vmid, sub {
5513 $conf = PVE
::QemuConfig-
>load_config($vmid);
5515 # cleanup, but leave suspending lock, to indicate something went wrong
5517 mon_cmd
($vmid, "savevm-end");
5518 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5519 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5520 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5521 PVE
::QemuConfig-
>write_config($vmid, $conf);
5527 die "lock changed unexpectedly\n"
5528 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5530 mon_cmd
($vmid, "quit");
5531 $conf->{lock} = 'suspended';
5532 PVE
::QemuConfig-
>write_config($vmid, $conf);
5538 my ($vmid, $skiplock, $nocheck) = @_;
5540 PVE
::QemuConfig-
>lock_config($vmid, sub {
5541 my $res = mon_cmd
($vmid, 'query-status');
5542 my $resume_cmd = 'cont';
5544 if ($res->{status
} && $res->{status
} eq 'suspended') {
5545 $resume_cmd = 'system_wakeup';
5550 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5552 PVE
::QemuConfig-
>check_lock($conf)
5553 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5556 mon_cmd
($vmid, $resume_cmd);
5561 my ($vmid, $skiplock, $key) = @_;
5563 PVE
::QemuConfig-
>lock_config($vmid, sub {
5565 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5567 # there is no qmp command, so we use the human monitor command
5568 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5569 die $res if $res ne '';
5573 # vzdump restore implementaion
5575 sub tar_archive_read_firstfile
{
5576 my $archive = shift;
5578 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5580 # try to detect archive type first
5581 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5582 die "unable to open file '$archive'\n";
5583 my $firstfile = <$fh>;
5587 die "ERROR: archive contaions no data\n" if !$firstfile;
5593 sub tar_restore_cleanup
{
5594 my ($storecfg, $statfile) = @_;
5596 print STDERR
"starting cleanup\n";
5598 if (my $fd = IO
::File-
>new($statfile, "r")) {
5599 while (defined(my $line = <$fd>)) {
5600 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5603 if ($volid =~ m
|^/|) {
5604 unlink $volid || die 'unlink failed\n';
5606 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5608 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5610 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5612 print STDERR
"unable to parse line in statfile - $line";
5619 sub restore_file_archive
{
5620 my ($archive, $vmid, $user, $opts) = @_;
5622 return restore_vma_archive
($archive, $vmid, $user, $opts)
5625 my $info = PVE
::Storage
::archive_info
($archive);
5626 my $format = $opts->{format
} // $info->{format
};
5627 my $comp = $info->{compression
};
5629 # try to detect archive format
5630 if ($format eq 'tar') {
5631 return restore_tar_archive
($archive, $vmid, $user, $opts);
5633 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5637 # hepler to remove disks that will not be used after restore
5638 my $restore_cleanup_oldconf = sub {
5639 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5641 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
5642 my ($ds, $drive) = @_;
5644 return if drive_is_cdrom
($drive, 1);
5646 my $volid = $drive->{file
};
5647 return if !$volid || $volid =~ m
|^/|;
5649 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
5650 return if !$path || !$owner || ($owner != $vmid);
5652 # Note: only delete disk we want to restore
5653 # other volumes will become unused
5654 if ($virtdev_hash->{$ds}) {
5655 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
5662 # delete vmstate files, after the restore we have no snapshots anymore
5663 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5664 my $snap = $oldconf->{snapshots
}->{$snapname};
5665 if ($snap->{vmstate
}) {
5666 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
5674 # Helper to parse vzdump backup device hints
5676 # $rpcenv: Environment, used to ckeck storage permissions
5677 # $user: User ID, to check storage permissions
5678 # $storecfg: Storage configuration
5679 # $fh: the file handle for reading the configuration
5680 # $devinfo: should contain device sizes for all backu-up'ed devices
5681 # $options: backup options (pool, default storage)
5683 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5684 my $parse_backup_hints = sub {
5685 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5687 my $virtdev_hash = {};
5689 while (defined(my $line = <$fh>)) {
5690 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5691 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5692 die "archive does not contain data for drive '$virtdev'\n"
5693 if !$devinfo->{$devname};
5695 if (defined($options->{storage
})) {
5696 $storeid = $options->{storage
} || 'local';
5697 } elsif (!$storeid) {
5700 $format = 'raw' if !$format;
5701 $devinfo->{$devname}->{devname
} = $devname;
5702 $devinfo->{$devname}->{virtdev
} = $virtdev;
5703 $devinfo->{$devname}->{format
} = $format;
5704 $devinfo->{$devname}->{storeid
} = $storeid;
5706 # check permission on storage
5707 my $pool = $options->{pool
}; # todo: do we need that?
5708 if ($user ne 'root@pam') {
5709 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5712 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5713 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5715 my $drive = parse_drive
($virtdev, $2);
5716 if (drive_is_cloudinit
($drive)) {
5717 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
5718 $storeid = $options->{storage
} if defined ($options->{storage
});
5719 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5720 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
5722 $virtdev_hash->{$virtdev} = {
5724 storeid
=> $storeid,
5725 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
5732 return $virtdev_hash;
5735 # Helper to allocate and activate all volumes required for a restore
5737 # $storecfg: Storage configuration
5738 # $virtdev_hash: as returned by parse_backup_hints()
5740 # Returns: { $virtdev => $volid }
5741 my $restore_allocate_devices = sub {
5742 my ($storecfg, $virtdev_hash, $vmid) = @_;
5745 foreach my $virtdev (sort keys %$virtdev_hash) {
5746 my $d = $virtdev_hash->{$virtdev};
5747 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5748 my $storeid = $d->{storeid
};
5749 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5751 # test if requested format is supported
5752 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
5753 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
5754 $d->{format
} = $defFormat if !$supported;
5757 if ($d->{is_cloudinit
}) {
5758 $name = "vm-$vmid-cloudinit";
5759 $name .= ".$d->{format}" if $d->{format
} ne 'raw';
5762 my $volid = PVE
::Storage
::vdisk_alloc
(
5763 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
5765 print STDERR
"new volume ID is '$volid'\n";
5766 $d->{volid
} = $volid;
5768 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
5770 $map->{$virtdev} = $volid;
5776 my $restore_update_config_line = sub {
5777 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5779 return if $line =~ m/^\#qmdump\#/;
5780 return if $line =~ m/^\#vzdump\#/;
5781 return if $line =~ m/^lock:/;
5782 return if $line =~ m/^unused\d+:/;
5783 return if $line =~ m/^parent:/;
5785 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5786 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5787 # try to convert old 1.X settings
5788 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5789 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5790 my ($model, $macaddr) = split(/\=/, $devconfig);
5791 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5794 bridge
=> "vmbr$ind",
5795 macaddr
=> $macaddr,
5797 my $netstr = print_net
($net);
5799 print $outfd "net$cookie->{netcount}: $netstr\n";
5800 $cookie->{netcount
}++;
5802 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5803 my ($id, $netstr) = ($1, $2);
5804 my $net = parse_net
($netstr);
5805 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5806 $netstr = print_net
($net);
5807 print $outfd "$id: $netstr\n";
5808 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5811 my $di = parse_drive
($virtdev, $value);
5812 if (defined($di->{backup
}) && !$di->{backup
}) {
5813 print $outfd "#$line";
5814 } elsif ($map->{$virtdev}) {
5815 delete $di->{format
}; # format can change on restore
5816 $di->{file
} = $map->{$virtdev};
5817 $value = print_drive
($di);
5818 print $outfd "$virtdev: $value\n";
5822 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5824 if ($vmgenid ne '0') {
5825 # always generate a new vmgenid if there was a valid one setup
5826 $vmgenid = generate_uuid
();
5828 print $outfd "vmgenid: $vmgenid\n";
5829 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5830 my ($uuid, $uuid_str);
5831 UUID
::generate
($uuid);
5832 UUID
::unparse
($uuid, $uuid_str);
5833 my $smbios1 = parse_smbios1
($2);
5834 $smbios1->{uuid
} = $uuid_str;
5835 print $outfd $1.print_smbios1
($smbios1)."\n";
5841 my $restore_deactivate_volumes = sub {
5842 my ($storecfg, $devinfo) = @_;
5845 foreach my $devname (keys %$devinfo) {
5846 my $volid = $devinfo->{$devname}->{volid
};
5847 push @$vollist, $volid if $volid;
5850 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5853 my $restore_destroy_volumes = sub {
5854 my ($storecfg, $devinfo) = @_;
5856 foreach my $devname (keys %$devinfo) {
5857 my $volid = $devinfo->{$devname}->{volid
};
5860 if ($volid =~ m
|^/|) {
5861 unlink $volid || die 'unlink failed\n';
5863 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5865 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5867 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5872 my ($cfg, $vmid) = @_;
5874 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5876 my $volid_hash = {};
5877 foreach my $storeid (keys %$info) {
5878 foreach my $item (@{$info->{$storeid}}) {
5879 next if !($item->{volid
} && $item->{size
});
5880 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5881 $volid_hash->{$item->{volid
}} = $item;
5888 sub update_disk_config
{
5889 my ($vmid, $conf, $volid_hash) = @_;
5892 my $prefix = "VM $vmid";
5894 # used and unused disks
5895 my $referenced = {};
5897 # Note: it is allowed to define multiple storages with same path (alias), so
5898 # we need to check both 'volid' and real 'path' (two different volid can point
5899 # to the same path).
5901 my $referencedpath = {};
5904 PVE
::QemuConfig-
>foreach_volume($conf, sub {
5905 my ($opt, $drive) = @_;
5907 my $volid = $drive->{file
};
5909 my $volume = $volid_hash->{$volid};
5911 # mark volid as "in-use" for next step
5912 $referenced->{$volid} = 1;
5913 if ($volume && (my $path = $volume->{path
})) {
5914 $referencedpath->{$path} = 1;
5917 return if drive_is_cdrom
($drive);
5920 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
5921 if (defined($updated)) {
5923 $conf->{$opt} = print_drive
($updated);
5924 print "$prefix ($opt): $msg\n";
5928 # remove 'unusedX' entry if volume is used
5929 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
5930 my ($opt, $drive) = @_;
5932 my $volid = $drive->{file
};
5936 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
5937 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5938 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5940 delete $conf->{$opt};
5943 $referenced->{$volid} = 1;
5944 $referencedpath->{$path} = 1 if $path;
5947 foreach my $volid (sort keys %$volid_hash) {
5948 next if $volid =~ m/vm-$vmid-state-/;
5949 next if $referenced->{$volid};
5950 my $path = $volid_hash->{$volid}->{path
};
5951 next if !$path; # just to be sure
5952 next if $referencedpath->{$path};
5954 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
5955 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5956 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5963 my ($vmid, $nolock, $dryrun) = @_;
5965 my $cfg = PVE
::Storage
::config
();
5967 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5968 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5969 foreach my $stor (keys %{$cfg->{ids
}}) {
5970 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
5973 print "rescan volumes...\n";
5974 my $volid_hash = scan_volids
($cfg, $vmid);
5976 my $updatefn = sub {
5979 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5981 PVE
::QemuConfig-
>check_lock($conf);
5984 foreach my $volid (keys %$volid_hash) {
5985 my $info = $volid_hash->{$volid};
5986 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
5989 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
5991 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
5994 if (defined($vmid)) {
5998 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6001 my $vmlist = config_list
();
6002 foreach my $vmid (keys %$vmlist) {
6006 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6012 sub restore_proxmox_backup_archive
{
6013 my ($archive, $vmid, $user, $options) = @_;
6015 my $storecfg = PVE
::Storage
::config
();
6017 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6018 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6020 my $server = $scfg->{server
};
6021 my $datastore = $scfg->{datastore
};
6022 my $username = $scfg->{username
} // 'root@pam';
6023 my $fingerprint = $scfg->{fingerprint
};
6024 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6026 my $repo = "$username\@$server:$datastore";
6028 # This is only used for `pbs-restore`!
6029 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6030 local $ENV{PBS_PASSWORD
} = $password;
6031 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6033 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6034 PVE
::Storage
::parse_volname
($storecfg, $archive);
6036 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6038 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6040 my $tmpdir = "/var/tmp/vzdumptmp$$";
6044 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6045 my $tmpfn = "$conffile.$$.tmp";
6046 # disable interrupts (always do cleanups)
6050 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6052 # Note: $oldconf is undef if VM does not exists
6053 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6054 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6056 my $rpcenv = PVE
::RPCEnvironment
::get
();
6065 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6067 my $cfgfn = "$tmpdir/qemu-server.conf";
6068 my $firewall_config_fn = "$tmpdir/fw.conf";
6069 my $index_fn = "$tmpdir/index.json";
6071 my $cmd = "restore";
6073 my $param = [$pbs_backup_name, "index.json", $index_fn];
6074 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6075 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6076 $index = decode_json
($index);
6078 # print Dumper($index);
6079 foreach my $info (@{$index->{files
}}) {
6080 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6082 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6083 $devinfo->{$devname}->{size
} = $1;
6085 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6090 my $is_qemu_server_backup = scalar(
6091 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6093 if (!$is_qemu_server_backup) {
6094 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6096 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6098 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6099 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6101 if ($has_firewall_config) {
6102 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6103 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6105 my $pve_firewall_dir = '/etc/pve/firewall';
6106 mkdir $pve_firewall_dir; # make sure the dir exists
6107 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6110 my $fh = IO
::File-
>new($cfgfn, "r") ||
6111 die "unable to read qemu-server.conf - $!\n";
6113 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6115 # fixme: rate limit?
6117 # create empty/temp config
6118 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6120 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6123 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6125 foreach my $virtdev (sort keys %$virtdev_hash) {
6126 my $d = $virtdev_hash->{$virtdev};
6127 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6129 my $volid = $d->{volid
};
6131 my $path = PVE
::Storage
::path
($storecfg, $volid);
6133 # This is the ONLY user of the PBS_ env vars set on top of this function!
6134 my $pbs_restore_cmd = [
6135 '/usr/bin/pbs-restore',
6136 '--repository', $repo,
6138 "$d->{devname}.img.fidx",
6143 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6144 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6146 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6147 push @$pbs_restore_cmd, '--skip-zero';
6150 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6151 print "restore proxmox backup image: $dbg_cmdstring\n";
6152 run_command
($pbs_restore_cmd);
6155 $fh->seek(0, 0) || die "seek failed - $!\n";
6157 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6159 my $cookie = { netcount
=> 0 };
6160 while (defined(my $line = <$fh>)) {
6161 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique
});
6169 $restore_deactivate_volumes->($storecfg, $devinfo);
6175 $restore_destroy_volumes->($storecfg, $devinfo);
6179 rename($tmpfn, $conffile) ||
6180 die "unable to commit configuration file '$conffile'\n";
6182 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6184 eval { rescan
($vmid, 1); };
6188 sub restore_vma_archive
{
6189 my ($archive, $vmid, $user, $opts, $comp) = @_;
6191 my $readfrom = $archive;
6193 my $cfg = PVE
::Storage
::config
();
6195 my $bwlimit = $opts->{bwlimit
};
6197 my $dbg_cmdstring = '';
6198 my $add_pipe = sub {
6200 push @$commands, $cmd;
6201 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6202 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6207 if ($archive eq '-') {
6210 # If we use a backup from a PVE defined storage we also consider that
6211 # storage's rate limit:
6212 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6213 if (defined($volid)) {
6214 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6215 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6217 print STDERR
"applying read rate limit: $readlimit\n";
6218 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6219 $add_pipe->($cstream);
6225 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6226 my $cmd = $info->{decompressor
};
6227 push @$cmd, $readfrom;
6231 my $tmpdir = "/var/tmp/vzdumptmp$$";
6234 # disable interrupts (always do cleanups)
6238 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6240 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6241 POSIX
::mkfifo
($mapfifo, 0600);
6243 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6245 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6252 my $rpcenv = PVE
::RPCEnvironment
::get
();
6254 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6255 my $tmpfn = "$conffile.$$.tmp";
6257 # Note: $oldconf is undef if VM does not exist
6258 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6259 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6263 my $print_devmap = sub {
6264 my $cfgfn = "$tmpdir/qemu-server.conf";
6266 # we can read the config - that is already extracted
6267 my $fh = IO
::File-
>new($cfgfn, "r") ||
6268 die "unable to read qemu-server.conf - $!\n";
6270 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6272 my $pve_firewall_dir = '/etc/pve/firewall';
6273 mkdir $pve_firewall_dir; # make sure the dir exists
6274 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6277 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6279 foreach my $key (keys %storage_limits) {
6280 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6282 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6283 $storage_limits{$key} = $limit * 1024;
6286 foreach my $devname (keys %$devinfo) {
6287 die "found no device mapping information for device '$devname'\n"
6288 if !$devinfo->{$devname}->{virtdev
};
6291 # create empty/temp config
6293 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6294 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6298 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6300 # print restore information to $fifofh
6301 foreach my $virtdev (sort keys %$virtdev_hash) {
6302 my $d = $virtdev_hash->{$virtdev};
6303 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6305 my $storeid = $d->{storeid
};
6306 my $volid = $d->{volid
};
6309 if (my $limit = $storage_limits{$storeid}) {
6310 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6313 my $write_zeros = 1;
6314 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6318 my $path = PVE
::Storage
::path
($cfg, $volid);
6320 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6322 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6325 $fh->seek(0, 0) || die "seek failed - $!\n";
6327 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6329 my $cookie = { netcount
=> 0 };
6330 while (defined(my $line = <$fh>)) {
6331 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6344 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6345 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6347 $oldtimeout = alarm($timeout);
6354 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6355 my ($dev_id, $size, $devname) = ($1, $2, $3);
6356 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6357 } elsif ($line =~ m/^CTIME: /) {
6358 # we correctly received the vma config, so we can disable
6359 # the timeout now for disk allocation (set to 10 minutes, so
6360 # that we always timeout if something goes wrong)
6363 print $fifofh "done\n";
6364 my $tmp = $oldtimeout || 0;
6365 $oldtimeout = undef;
6372 print "restore vma archive: $dbg_cmdstring\n";
6373 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6377 alarm($oldtimeout) if $oldtimeout;
6379 $restore_deactivate_volumes->($cfg, $devinfo);
6381 close($fifofh) if $fifofh;
6387 $restore_destroy_volumes->($cfg, $devinfo);
6391 rename($tmpfn, $conffile) ||
6392 die "unable to commit configuration file '$conffile'\n";
6394 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6396 eval { rescan
($vmid, 1); };
6400 sub restore_tar_archive
{
6401 my ($archive, $vmid, $user, $opts) = @_;
6403 if ($archive ne '-') {
6404 my $firstfile = tar_archive_read_firstfile
($archive);
6405 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6406 if $firstfile ne 'qemu-server.conf';
6409 my $storecfg = PVE
::Storage
::config
();
6411 # avoid zombie disks when restoring over an existing VM -> cleanup first
6412 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6413 # skiplock=1 because qmrestore has set the 'create' lock itself already
6414 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6415 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6417 my $tocmd = "/usr/lib/qemu-server/qmextract";
6419 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6420 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6421 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6422 $tocmd .= ' --info' if $opts->{info
};
6424 # tar option "xf" does not autodetect compression when read from STDIN,
6425 # so we pipe to zcat
6426 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6427 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6429 my $tmpdir = "/var/tmp/vzdumptmp$$";
6432 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6433 local $ENV{VZDUMP_VMID
} = $vmid;
6434 local $ENV{VZDUMP_USER
} = $user;
6436 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6437 my $tmpfn = "$conffile.$$.tmp";
6439 # disable interrupts (always do cleanups)
6443 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6451 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6453 if ($archive eq '-') {
6454 print "extracting archive from STDIN\n";
6455 run_command
($cmd, input
=> "<&STDIN");
6457 print "extracting archive '$archive'\n";
6461 return if $opts->{info
};
6465 my $statfile = "$tmpdir/qmrestore.stat";
6466 if (my $fd = IO
::File-
>new($statfile, "r")) {
6467 while (defined (my $line = <$fd>)) {
6468 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6469 $map->{$1} = $2 if $1;
6471 print STDERR
"unable to parse line in statfile - $line\n";
6477 my $confsrc = "$tmpdir/qemu-server.conf";
6479 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6481 my $outfd = IO
::File-
>new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6483 my $cookie = { netcount
=> 0 };
6484 while (defined (my $line = <$srcfd>)) {
6485 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6493 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6499 rename $tmpfn, $conffile ||
6500 die "unable to commit configuration file '$conffile'\n";
6502 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6504 eval { rescan
($vmid, 1); };
6508 sub foreach_storage_used_by_vm
{
6509 my ($conf, $func) = @_;
6513 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6514 my ($ds, $drive) = @_;
6515 return if drive_is_cdrom
($drive);
6517 my $volid = $drive->{file
};
6519 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6520 $sidhash->{$sid} = $sid if $sid;
6523 foreach my $sid (sort keys %$sidhash) {
6528 my $qemu_snap_storage = {
6531 sub do_snapshots_with_qemu
{
6532 my ($storecfg, $volid) = @_;
6534 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6535 my $scfg = $storecfg->{ids
}->{$storage_name};
6536 die "could not find storage '$storage_name'\n" if !defined($scfg);
6538 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6542 if ($volid =~ m/\.(qcow2|qed)$/){
6549 sub qga_check_running
{
6550 my ($vmid, $nowarn) = @_;
6552 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6554 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6560 sub template_create
{
6561 my ($vmid, $conf, $disk) = @_;
6563 my $storecfg = PVE
::Storage
::config
();
6565 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6566 my ($ds, $drive) = @_;
6568 return if drive_is_cdrom
($drive);
6569 return if $disk && $ds ne $disk;
6571 my $volid = $drive->{file
};
6572 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6574 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6575 $drive->{file
} = $voliddst;
6576 $conf->{$ds} = print_drive
($drive);
6577 PVE
::QemuConfig-
>write_config($vmid, $conf);
6581 sub convert_iscsi_path
{
6584 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6589 my $initiator_name = get_initiator_name
();
6591 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6592 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6595 die "cannot convert iscsi path '$path', unkown format\n";
6598 sub qemu_img_convert
{
6599 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6601 my $storecfg = PVE
::Storage
::config
();
6602 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6603 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6605 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6609 my $src_is_iscsi = 0;
6613 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6614 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6615 $src_format = qemu_img_format
($src_scfg, $src_volname);
6616 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6617 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6618 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6619 } elsif (-f
$src_volid) {
6620 $src_path = $src_volid;
6621 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6626 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6628 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6629 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6630 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6631 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6634 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6635 push @$cmd, '-l', "snapshot.name=$snapname"
6636 if $snapname && $src_format && $src_format eq "qcow2";
6637 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6638 push @$cmd, '-T', $cachemode if defined($cachemode);
6640 if ($src_is_iscsi) {
6641 push @$cmd, '--image-opts';
6642 $src_path = convert_iscsi_path
($src_path);
6643 } elsif ($src_format) {
6644 push @$cmd, '-f', $src_format;
6647 if ($dst_is_iscsi) {
6648 push @$cmd, '--target-image-opts';
6649 $dst_path = convert_iscsi_path
($dst_path);
6651 push @$cmd, '-O', $dst_format;
6654 push @$cmd, $src_path;
6656 if (!$dst_is_iscsi && $is_zero_initialized) {
6657 push @$cmd, "zeroinit:$dst_path";
6659 push @$cmd, $dst_path;
6664 if($line =~ m/\((\S+)\/100\
%\)/){
6666 my $transferred = int($size * $percent / 100);
6667 my $remaining = $size - $transferred;
6669 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6674 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6676 die "copy failed: $err" if $err;
6679 sub qemu_img_format
{
6680 my ($scfg, $volname) = @_;
6682 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6689 sub qemu_drive_mirror
{
6690 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6692 $jobs = {} if !$jobs;
6696 $jobs->{"drive-$drive"} = {};
6698 if ($dst_volid =~ /^nbd:/) {
6699 $qemu_target = $dst_volid;
6702 my $storecfg = PVE
::Storage
::config
();
6703 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6705 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6707 $format = qemu_img_format
($dst_scfg, $dst_volname);
6709 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6711 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6714 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6715 $opts->{format
} = $format if $format;
6717 if (defined($src_bitmap)) {
6718 $opts->{sync
} = 'incremental';
6719 $opts->{bitmap
} = $src_bitmap;
6720 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6723 if (defined($bwlimit)) {
6724 $opts->{speed
} = $bwlimit * 1024;
6725 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6727 print "drive mirror is starting for drive-$drive\n";
6730 # if a job already runs for this device we get an error, catch it for cleanup
6731 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
6733 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6735 die "mirroring error: $err\n";
6738 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
6741 # $completion can be either
6742 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6743 # 'cancel': wait until all jobs are ready, block-job-cancel them
6744 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6745 sub qemu_drive_mirror_monitor
{
6746 my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6748 $completion //= 'complete';
6751 my $err_complete = 0;
6754 die "storage migration timed out\n" if $err_complete > 300;
6756 my $stats = mon_cmd
($vmid, "query-block-jobs");
6758 my $running_mirror_jobs = {};
6759 foreach my $stat (@$stats) {
6760 next if $stat->{type
} ne 'mirror';
6761 $running_mirror_jobs->{$stat->{device
}} = $stat;
6764 my $readycounter = 0;
6766 foreach my $job (keys %$jobs) {
6768 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6769 print "$job : finished\n";
6770 delete $jobs->{$job};
6774 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6776 my $busy = $running_mirror_jobs->{$job}->{busy
};
6777 my $ready = $running_mirror_jobs->{$job}->{ready
};
6778 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6779 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6780 my $remaining = $total - $transferred;
6781 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6783 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6786 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6789 last if scalar(keys %$jobs) == 0;
6791 if ($readycounter == scalar(keys %$jobs)) {
6792 print "all mirroring jobs are ready \n";
6793 last if $completion eq 'skip'; #do the complete later
6795 if ($vmiddst && $vmiddst != $vmid) {
6796 my $agent_running = $qga && qga_check_running
($vmid);
6797 if ($agent_running) {
6798 print "freeze filesystem\n";
6799 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6801 print "suspend vm\n";
6802 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6805 # if we clone a disk for a new target vm, we don't switch the disk
6806 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6808 if ($agent_running) {
6809 print "unfreeze filesystem\n";
6810 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6812 print "resume vm\n";
6813 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6819 foreach my $job (keys %$jobs) {
6820 # try to switch the disk if source and destination are on the same guest
6821 print "$job: Completing block job...\n";
6824 if ($completion eq 'complete') {
6825 $op = 'block-job-complete';
6826 } elsif ($completion eq 'cancel') {
6827 $op = 'block-job-cancel';
6829 die "invalid completion value: $completion\n";
6831 eval { mon_cmd
($vmid, $op, device
=> $job) };
6832 if ($@ =~ m/cannot be completed/) {
6833 print "$job: Block job cannot be completed, try again.\n";
6836 print "$job: Completed successfully.\n";
6837 $jobs->{$job}->{complete
} = 1;
6848 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6849 die "mirroring error: $err";
6854 sub qemu_blockjobs_cancel
{
6855 my ($vmid, $jobs) = @_;
6857 foreach my $job (keys %$jobs) {
6858 print "$job: Cancelling block job\n";
6859 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6860 $jobs->{$job}->{cancel
} = 1;
6864 my $stats = mon_cmd
($vmid, "query-block-jobs");
6866 my $running_jobs = {};
6867 foreach my $stat (@$stats) {
6868 $running_jobs->{$stat->{device
}} = $stat;
6871 foreach my $job (keys %$jobs) {
6873 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6874 print "$job: Done.\n";
6875 delete $jobs->{$job};
6879 last if scalar(keys %$jobs) == 0;
6886 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6887 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6892 print "create linked clone of drive $drivename ($drive->{file})\n";
6893 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6894 push @$newvollist, $newvolid;
6897 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6898 $storeid = $storage if $storage;
6900 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6902 print "create full clone of drive $drivename ($drive->{file})\n";
6905 if (drive_is_cloudinit
($drive)) {
6906 $name = "vm-$newvmid-cloudinit";
6907 $name .= ".$dst_format" if $dst_format ne 'raw';
6909 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
6910 } elsif ($drivename eq 'efidisk0') {
6911 $size = get_efivars_size
($conf);
6913 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
6916 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $dst_format, $name, $size);
6917 push @$newvollist, $newvolid;
6919 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
6921 if (drive_is_cloudinit
($drive)) {
6925 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6926 if (!$running || $snapname) {
6927 # TODO: handle bwlimits
6928 if ($drivename eq 'efidisk0') {
6929 # the relevant data on the efidisk may be smaller than the source
6930 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6931 # that is given by the OVMF_VARS.fd
6932 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
});
6933 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
6934 run_command
(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
6935 "if=$src_path", "of=$dst_path"]);
6937 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
6941 my $kvmver = get_running_qemu_version
($vmid);
6942 if (!min_version
($kvmver, 2, 7)) {
6943 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6944 if $drive->{iothread
};
6947 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
6948 $completion, $qga, $bwlimit);
6953 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
6956 $disk->{format
} = undef;
6957 $disk->{file
} = $newvolid;
6958 $disk->{size
} = $size;
6963 sub get_running_qemu_version
{
6965 my $res = mon_cmd
($vmid, "query-version");
6966 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6969 sub qemu_use_old_bios_files
{
6970 my ($machine_type) = @_;
6972 return if !$machine_type;
6974 my $use_old_bios_files = undef;
6976 if ($machine_type =~ m/^(\S+)\.pxe$/) {
6978 $use_old_bios_files = 1;
6980 my $version = extract_version
($machine_type, kvm_user_version
());
6981 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6982 # load new efi bios files on migration. So this hack is required to allow
6983 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6984 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6985 $use_old_bios_files = !min_version
($version, 2, 4);
6988 return ($use_old_bios_files, $machine_type);
6991 sub get_efivars_size
{
6993 my $arch = get_vm_arch
($conf);
6994 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
6995 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
6996 return -s
$ovmf_vars;
6999 sub update_efidisk_size
{
7002 return if !defined($conf->{efidisk0
});
7004 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
7005 $disk->{size
} = get_efivars_size
($conf);
7006 $conf->{efidisk0
} = print_drive
($disk);
7011 sub create_efidisk
($$$$$) {
7012 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7014 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7015 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7017 my $vars_size_b = -s
$ovmf_vars;
7018 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7019 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7020 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7022 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7023 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7025 return ($volid, $size/1024);
7028 sub vm_iothreads_list
{
7031 my $res = mon_cmd
($vmid, 'query-iothreads');
7034 foreach my $iothread (@$res) {
7035 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7042 my ($conf, $drive) = @_;
7046 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7048 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7054 my $controller = int($drive->{index} / $maxdev);
7055 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
7059 return ($maxdev, $controller, $controller_prefix);
7062 sub windows_version
{
7065 return 0 if !$ostype;
7069 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7071 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7073 } elsif ($ostype =~ m/^win(\d+)$/) {
7080 sub resolve_dst_disk_format
{
7081 my ($storecfg, $storeid, $src_volname, $format) = @_;
7082 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7085 # if no target format is specified, use the source disk format as hint
7087 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7088 $format = qemu_img_format
($scfg, $src_volname);
7094 # test if requested format is supported - else use default
7095 my $supported = grep { $_ eq $format } @$validFormats;
7096 $format = $defFormat if !$supported;
7100 # NOTE: if this logic changes, please update docs & possibly gui logic
7101 sub find_vmstate_storage
{
7102 my ($conf, $storecfg) = @_;
7104 # first, return storage from conf if set
7105 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7107 my ($target, $shared, $local);
7109 foreach_storage_used_by_vm
($conf, sub {
7111 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7112 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7113 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7116 # second, use shared storage where VM has at least one disk
7117 # third, use local storage where VM has at least one disk
7118 # fall back to local storage
7119 $target = $shared // $local // 'local';
7125 my ($uuid, $uuid_str);
7126 UUID
::generate
($uuid);
7127 UUID
::unparse
($uuid, $uuid_str);
7131 sub generate_smbios1_uuid
{
7132 return "uuid=".generate_uuid
();
7138 mon_cmd
($vmid, 'nbd-server-stop');
7141 sub create_reboot_request
{
7143 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7144 or die "failed to create reboot trigger file: $!\n";
7148 sub clear_reboot_request
{
7150 my $path = "/run/qemu-server/$vmid.reboot";
7153 $res = unlink($path);
7154 die "could not remove reboot request for $vmid: $!"
7155 if !$res && $! != POSIX
::ENOENT
;
7160 sub bootorder_from_legacy
{
7161 my ($conf, $bootcfg) = @_;
7163 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
7164 my $bootindex_hash = {};
7166 foreach my $o (split(//, $boot)) {
7167 $bootindex_hash->{$o} = $i*100;
7173 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7174 my ($ds, $drive) = @_;
7176 if (drive_is_cdrom
($drive, 1)) {
7177 if ($bootindex_hash->{d
}) {
7178 $bootorder->{$ds} = $bootindex_hash->{d
};
7179 $bootindex_hash->{d
} += 1;
7181 } elsif ($bootindex_hash->{c
}) {
7182 $bootorder->{$ds} = $bootindex_hash->{c
}
7183 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
7184 $bootindex_hash->{c
} += 1;
7188 if ($bootindex_hash->{n
}) {
7189 for (my $i = 0; $i < $MAX_NETS; $i++) {
7190 my $netname = "net$i";
7191 next if !$conf->{$netname};
7192 $bootorder->{$netname} = $bootindex_hash->{n
};
7193 $bootindex_hash->{n
} += 1;
7200 # Generate default device list for 'boot: order=' property. Matches legacy
7201 # default boot order, but with explicit device names. This is important, since
7202 # the fallback for when neither 'order' nor the old format is specified relies
7203 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7204 sub get_default_bootdevices
{
7210 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
7211 push @ret, $first if $first;
7214 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
7215 push @ret, $first if $first;
7218 for (my $i = 0; $i < $MAX_NETS; $i++) {
7219 my $netname = "net$i";
7220 next if !$conf->{$netname};
7221 push @ret, $netname;
7228 sub device_bootorder
{
7231 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
7233 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
7236 if (!defined($boot) || $boot->{legacy
}) {
7237 $bootorder = bootorder_from_legacy
($conf, $boot);
7238 } elsif ($boot->{order
}) {
7239 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7240 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
7241 $bootorder->{$dev} = $i++;
7248 # bash completion helper
7250 sub complete_backup_archives
{
7251 my ($cmdname, $pname, $cvalue) = @_;
7253 my $cfg = PVE
::Storage
::config
();
7257 if ($cvalue =~ m/^([^:]+):/) {
7261 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7264 foreach my $id (keys %$data) {
7265 foreach my $item (@{$data->{$id}}) {
7266 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7267 push @$res, $item->{volid
} if defined($item->{volid
});
7274 my $complete_vmid_full = sub {
7277 my $idlist = vmstatus
();
7281 foreach my $id (keys %$idlist) {
7282 my $d = $idlist->{$id};
7283 if (defined($running)) {
7284 next if $d->{template
};
7285 next if $running && $d->{status
} ne 'running';
7286 next if !$running && $d->{status
} eq 'running';
7295 return &$complete_vmid_full();
7298 sub complete_vmid_stopped
{
7299 return &$complete_vmid_full(0);
7302 sub complete_vmid_running
{
7303 return &$complete_vmid_full(1);
7306 sub complete_storage
{
7308 my $cfg = PVE
::Storage
::config
();
7309 my $ids = $cfg->{ids
};
7312 foreach my $sid (keys %$ids) {
7313 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7314 next if !$ids->{$sid}->{content
}->{images
};
7321 sub complete_migration_storage
{
7322 my ($cmd, $param, $current_value, $all_args) = @_;
7324 my $targetnode = @$all_args[1];
7326 my $cfg = PVE
::Storage
::config
();
7327 my $ids = $cfg->{ids
};
7330 foreach my $sid (keys %$ids) {
7331 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7332 next if !$ids->{$sid}->{content
}->{images
};