1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
30 use PVE
::DataCenterConfig
;
31 use PVE
::Exception
qw(raise raise_param_exc);
32 use PVE
::GuestHelpers
;
34 use PVE
::JSONSchema
qw(get_standard_option);
36 use PVE
::RPCEnvironment
;
40 use PVE
::Tools
qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
44 use PVE
::QemuServer
::Helpers
qw(min_version);
45 use PVE
::QemuServer
::Cloudinit
;
46 use PVE
::QemuServer
::Machine
;
47 use PVE
::QemuServer
::Memory
;
48 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
49 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port);
50 use PVE
::QemuServer
::USB
qw(parse_usb_device);
52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
55 "$EDK2_FW_BASE/OVMF_CODE.fd",
56 "$EDK2_FW_BASE/OVMF_VARS.fd"
59 "$EDK2_FW_BASE/AAVMF_CODE.fd",
60 "$EDK2_FW_BASE/AAVMF_VARS.fd"
64 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
68 # Note about locking: we use flock on the config file protect
69 # against concurent actions.
70 # Aditionaly, we have a 'lock' setting in the config file. This
71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
72 # allowed when such lock is set. But you can ignore this kind of
73 # lock with the --skiplock flag.
75 cfs_register_file
('/qemu-server/',
79 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
80 description
=> "Some command save/restore state from this location.",
86 PVE
::JSONSchema
::register_standard_option
('pve-qm-image-format', {
88 enum
=> [qw(raw cow qcow qed qcow2 vmdk cloop)],
89 description
=> "The drive's backing file's data format.",
93 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
94 description
=> "Specifies the Qemu machine type.",
96 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
101 #no warnings 'redefine';
104 my ($controller, $vmid, $option, $value) = @_;
106 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
107 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
111 my $nodename = PVE
::INotify
::nodename
();
113 my $cpu_vendor_list = {
115 486 => 'GenuineIntel',
116 pentium
=> 'GenuineIntel',
117 pentium2
=> 'GenuineIntel',
118 pentium3
=> 'GenuineIntel',
119 coreduo
=> 'GenuineIntel',
120 core2duo
=> 'GenuineIntel',
121 Conroe
=> 'GenuineIntel',
122 Penryn
=> 'GenuineIntel',
123 Nehalem
=> 'GenuineIntel',
124 'Nehalem-IBRS' => 'GenuineIntel',
125 Westmere
=> 'GenuineIntel',
126 'Westmere-IBRS' => 'GenuineIntel',
127 SandyBridge
=> 'GenuineIntel',
128 'SandyBridge-IBRS' => 'GenuineIntel',
129 IvyBridge
=> 'GenuineIntel',
130 'IvyBridge-IBRS' => 'GenuineIntel',
131 Haswell
=> 'GenuineIntel',
132 'Haswell-IBRS' => 'GenuineIntel',
133 'Haswell-noTSX' => 'GenuineIntel',
134 'Haswell-noTSX-IBRS' => 'GenuineIntel',
135 Broadwell
=> 'GenuineIntel',
136 'Broadwell-IBRS' => 'GenuineIntel',
137 'Broadwell-noTSX' => 'GenuineIntel',
138 'Broadwell-noTSX-IBRS' => 'GenuineIntel',
139 'Skylake-Client' => 'GenuineIntel',
140 'Skylake-Client-IBRS' => 'GenuineIntel',
141 'Skylake-Server' => 'GenuineIntel',
142 'Skylake-Server-IBRS' => 'GenuineIntel',
143 'Cascadelake-Server' => 'GenuineIntel',
144 KnightsMill
=> 'GenuineIntel',
148 athlon
=> 'AuthenticAMD',
149 phenom
=> 'AuthenticAMD',
150 Opteron_G1
=> 'AuthenticAMD',
151 Opteron_G2
=> 'AuthenticAMD',
152 Opteron_G3
=> 'AuthenticAMD',
153 Opteron_G4
=> 'AuthenticAMD',
154 Opteron_G5
=> 'AuthenticAMD',
155 EPYC
=> 'AuthenticAMD',
156 'EPYC-IBPB' => 'AuthenticAMD',
158 # generic types, use vendor from host node
167 my @supported_cpu_flags = (
181 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
185 description
=> "Emulated CPU type.",
187 enum
=> [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
192 description
=> "Do not identify as a KVM virtual machine.",
199 pattern
=> qr/[a-zA-Z0-9]{1,12}/,
200 format_description
=> 'vendor-id',
201 description
=> 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
205 description
=> "List of additional CPU flags separated by ';'."
206 . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
207 . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
208 format_description
=> '+FLAG[;-FLAG...]',
210 pattern
=> qr/$cpu_flag(;$cpu_flag)*/,
219 enum
=> [qw(i6300esb ib700)],
220 description
=> "Watchdog type to emulate.",
221 default => 'i6300esb',
226 enum
=> [qw(reset shutdown poweroff pause debug none)],
227 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
231 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
235 description
=> "Enable/disable Qemu GuestAgent.",
240 fstrim_cloned_disks
=> {
241 description
=> "Run fstrim after cloning/moving a disk.",
247 description
=> "Select the agent type",
251 enum
=> [qw(virtio isa)],
257 description
=> "Select the VGA type.",
262 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
265 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
277 description
=> "The size of the file in MB.",
281 pattern
=> '[a-zA-Z0-9\-]+',
283 format_description
=> 'string',
284 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
291 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
292 description
=> "Configure an audio device."
299 description
=> "Driver backend for the audio device."
303 my $spice_enhancements_fmt = {
308 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
312 enum
=> ['off', 'all', 'filter'],
315 description
=> "Enable video streaming. Uses compression for detected video streams."
323 description
=> "Specifies whether a VM will be started during system bootup.",
329 description
=> "Automatic restart after crash (currently ignored).",
334 type
=> 'string', format
=> 'pve-hotplug-features',
335 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
336 default => 'network,disk,usb',
341 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
347 description
=> "Lock/unlock the VM.",
348 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
353 description
=> "Limit of CPU usage.",
354 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
362 description
=> "CPU weight for a VM.",
363 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
371 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
378 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
384 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
392 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
393 "It should not be necessary to set it.",
394 enum
=> PVE
::Tools
::kvmkeymaplist
(),
399 type
=> 'string', format
=> 'dns-name',
400 description
=> "Set a name for the VM. Only used on the configuration web interface.",
405 description
=> "SCSI controller model",
406 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
412 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
417 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
418 description
=> "Specify guest operating system.",
419 verbose_description
=> <<EODESC,
420 Specify guest operating system. This is used to enable special
421 optimization/features for specific operating systems:
424 other;; unspecified OS
425 wxp;; Microsoft Windows XP
426 w2k;; Microsoft Windows 2000
427 w2k3;; Microsoft Windows 2003
428 w2k8;; Microsoft Windows 2008
429 wvista;; Microsoft Windows Vista
430 win7;; Microsoft Windows 7
431 win8;; Microsoft Windows 8/2012/2012r2
432 win10;; Microsoft Windows 10/2016
433 l24;; Linux 2.4 Kernel
434 l26;; Linux 2.6 - 5.X Kernel
435 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
441 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
442 pattern
=> '[acdn]{1,4}',
447 type
=> 'string', format
=> 'pve-qm-bootdisk',
448 description
=> "Enable booting from specified disk.",
449 pattern
=> '(ide|sata|scsi|virtio)\d+',
454 description
=> "The number of CPUs. Please use option -sockets instead.",
461 description
=> "The number of CPU sockets.",
468 description
=> "The number of cores per socket.",
475 description
=> "Enable/disable NUMA.",
481 description
=> "Enable/disable hugepages memory.",
482 enum
=> [qw(any 2 1024)],
487 description
=> "Number of hotplugged vcpus.",
494 description
=> "Enable/disable ACPI.",
499 description
=> "Enable/disable Qemu GuestAgent and its properties.",
501 format
=> $agent_fmt,
506 description
=> "Enable/disable KVM hardware virtualization.",
512 description
=> "Enable/disable time drift fix.",
518 description
=> "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
523 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
527 type
=> 'string', format
=> $vga_fmt,
528 description
=> "Configure the VGA hardware.",
529 verbose_description
=> "Configure the VGA Hardware. If you want to use ".
530 "high resolution modes (>= 1280x1024x16) you may need to increase " .
531 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
532 "is 'std' for all OS types besides some Windows versions (XP and " .
533 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
534 "display server. For win* OS you can select how many independent " .
535 "displays you want, Linux guests can add displays them self.\n".
536 "You can also run without any graphic card, using a serial device as terminal.",
540 type
=> 'string', format
=> 'pve-qm-watchdog',
541 description
=> "Create a virtual hardware watchdog device.",
542 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled" .
543 " (by a guest action), the watchdog must be periodically polled " .
544 "by an agent inside the guest or else the watchdog will reset " .
545 "the guest (or execute the respective action specified)",
550 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
551 description
=> "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
552 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
555 startup
=> get_standard_option
('pve-startup-order'),
559 description
=> "Enable/disable Template.",
565 description
=> "Arbitrary arguments passed to kvm.",
566 verbose_description
=> <<EODESCR,
567 Arbitrary arguments passed to kvm, for example:
569 args: -no-reboot -no-hpet
571 NOTE: this option is for experts only.
578 description
=> "Enable/disable the USB tablet device.",
579 verbose_description
=> "Enable/disable the USB tablet device. This device is " .
580 "usually needed to allow absolute mouse positioning with VNC. " .
581 "Else the mouse runs out of sync with normal VNC clients. " .
582 "If you're running lots of console-only guests on one host, " .
583 "you may consider disabling this to save some context switches. " .
584 "This is turned off by default if you use spice (-vga=qxl).",
589 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
593 migrate_downtime
=> {
596 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
602 type
=> 'string', format
=> 'pve-qm-ide',
603 typetext
=> '<volume>',
604 description
=> "This is an alias for option -ide2",
608 description
=> "Emulated CPU type.",
612 parent
=> get_standard_option
('pve-snapshot-name', {
614 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
618 description
=> "Timestamp for snapshots.",
624 type
=> 'string', format
=> 'pve-volume-id',
625 description
=> "Reference to a volume which stores the VM state. This is used internally for snapshots.",
627 vmstatestorage
=> get_standard_option
('pve-storage-id', {
628 description
=> "Default storage for VM state volumes/files.",
631 runningmachine
=> get_standard_option
('pve-qemu-machine', {
632 description
=> "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
634 machine
=> get_standard_option
('pve-qemu-machine'),
636 description
=> "Virtual processor architecture. Defaults to the host.",
639 enum
=> [qw(x86_64 aarch64)],
642 description
=> "Specify SMBIOS type 1 fields.",
643 type
=> 'string', format
=> 'pve-qm-smbios1',
650 description
=> "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
656 enum
=> [ qw(seabios ovmf) ],
657 description
=> "Select BIOS implementation.",
658 default => 'seabios',
662 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
663 format_description
=> 'UUID',
664 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
665 verbose_description
=> "The VM generation ID (vmgenid) device exposes a".
666 " 128-bit integer value identifier to the guest OS. This allows to".
667 " notify the guest operating system when the virtual machine is".
668 " executed with a different configuration (e.g. snapshot execution".
669 " or creation from a template). The guest operating system notices".
670 " the change, and is then able to react as appropriate by marking".
671 " its copies of distributed databases as dirty, re-initializing its".
672 " random number generator, etc.\n".
673 "Note that auto-creation only works when done throug API/CLI create".
674 " or update methods, but not when manually editing the config file.",
675 default => "1 (autogenerated)",
680 format
=> 'pve-volume-id',
682 description
=> "Script that will be executed during various steps in the vms lifetime.",
686 format
=> $ivshmem_fmt,
687 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
692 format
=> $audio_fmt,
693 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
696 spice_enhancements
=> {
698 format
=> $spice_enhancements_fmt,
699 description
=> "Configure additional enhancements for SPICE.",
708 description
=> 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
709 format
=> 'pve-volume-id',
710 format_description
=> 'volume',
715 description
=> 'Specify a custom file containing all network data passed to the VM via cloud-init.',
716 format
=> 'pve-volume-id',
717 format_description
=> 'volume',
722 description
=> 'Specify a custom file containing all user data passed to the VM via cloud-init.',
723 format
=> 'pve-volume-id',
724 format_description
=> 'volume',
727 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
729 my $confdesc_cloudinit = {
733 description
=> 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
734 enum
=> ['configdrive2', 'nocloud'],
739 description
=> "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
744 description
=> 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
749 description
=> 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
750 format
=> 'pve-qm-cicustom',
755 description
=> "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
759 type
=> 'string', format
=> 'address-list',
760 description
=> "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
765 format
=> 'urlencoded',
766 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
770 # what about other qemu settings ?
772 #machine => 'string',
785 ##soundhw => 'string',
787 while (my ($k, $v) = each %$confdesc) {
788 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
791 my $MAX_IDE_DISKS = 4;
792 my $MAX_SCSI_DISKS = 14;
793 my $MAX_VIRTIO_DISKS = 16;
794 my $MAX_SATA_DISKS = 6;
795 my $MAX_USB_DEVICES = 5;
797 my $MAX_UNUSED_DISKS = 256;
798 my $MAX_HOSTPCI_DEVICES = 16;
799 my $MAX_SERIAL_PORTS = 4;
800 my $MAX_PARALLEL_PORTS = 3;
806 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
807 description
=> "CPUs accessing this NUMA node.",
808 format_description
=> "id[-id];...",
812 description
=> "Amount of memory this NUMA node provides.",
817 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
818 description
=> "Host NUMA nodes to use.",
819 format_description
=> "id[-id];...",
824 enum
=> [qw(preferred bind interleave)],
825 description
=> "NUMA allocation policy.",
829 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
832 type
=> 'string', format
=> $numa_fmt,
833 description
=> "NUMA topology.",
835 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
837 for (my $i = 0; $i < $MAX_NUMA; $i++) {
838 $confdesc->{"numa$i"} = $numadesc;
841 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
842 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
843 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
844 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
846 my $net_fmt_bridge_descr = <<__EOD__;
847 Bridge to attach the network device to. The Proxmox VE standard bridge
850 If you do not specify a bridge, we create a kvm user (NATed) network
851 device, which provides DHCP and DNS services. The following addresses
858 The DHCP server assign addresses to the guest starting from 10.0.2.15.
862 macaddr
=> get_standard_option
('mac-addr', {
863 description
=> "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
867 description
=> "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
868 enum
=> $nic_model_list,
871 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
874 description
=> $net_fmt_bridge_descr,
875 format_description
=> 'bridge',
880 minimum
=> 0, maximum
=> 16,
881 description
=> 'Number of packet queues to be used on the device.',
887 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
892 minimum
=> 1, maximum
=> 4094,
893 description
=> 'VLAN tag to apply to packets on this interface.',
898 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
899 description
=> 'VLAN trunks to pass through this interface.',
900 format_description
=> 'vlanid[;vlanid...]',
905 description
=> 'Whether this interface should be protected by the firewall.',
910 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
917 type
=> 'string', format
=> $net_fmt,
918 description
=> "Specify network devices.",
921 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
926 format
=> 'pve-ipv4-config',
927 format_description
=> 'IPv4Format/CIDR',
928 description
=> 'IPv4 address in CIDR format.',
935 format_description
=> 'GatewayIPv4',
936 description
=> 'Default gateway for IPv4 traffic.',
942 format
=> 'pve-ipv6-config',
943 format_description
=> 'IPv6Format/CIDR',
944 description
=> 'IPv6 address in CIDR format.',
951 format_description
=> 'GatewayIPv6',
952 description
=> 'Default gateway for IPv6 traffic.',
957 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
960 type
=> 'string', format
=> 'pve-qm-ipconfig',
961 description
=> <<'EODESCR',
962 cloud-init: Specify IP addresses and gateways for the corresponding interface.
964 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
966 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
972 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
974 for (my $i = 0; $i < $MAX_NETS; $i++) {
975 $confdesc->{"net$i"} = $netdesc;
976 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
979 foreach my $key (keys %$confdesc_cloudinit) {
980 $confdesc->{$key} = $confdesc_cloudinit->{$key};
983 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
984 sub verify_volume_id_or_qm_path
{
985 my ($volid, $noerr) = @_;
987 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
991 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
992 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
994 return undef if $noerr;
1002 my %drivedesc_base = (
1003 volume
=> { alias
=> 'file' },
1006 format
=> 'pve-volume-id-or-qm-path',
1008 format_description
=> 'volume',
1009 description
=> "The drive's backing volume.",
1013 enum
=> [qw(cdrom disk)],
1014 description
=> "The drive's media type.",
1020 description
=> "Force the drive's physical geometry to have a specific cylinder count.",
1025 description
=> "Force the drive's physical geometry to have a specific head count.",
1030 description
=> "Force the drive's physical geometry to have a specific sector count.",
1035 enum
=> [qw(none lba auto)],
1036 description
=> "Force disk geometry bios translation mode.",
1041 description
=> "Controls qemu's snapshot mode feature."
1042 . " If activated, changes made to the disk are temporary and will"
1043 . " be discarded when the VM is shutdown.",
1048 enum
=> [qw(none writethrough writeback unsafe directsync)],
1049 description
=> "The drive's cache mode",
1052 format
=> get_standard_option
('pve-qm-image-format'),
1055 format
=> 'disk-size',
1056 format_description
=> 'DiskSize',
1057 description
=> "Disk size. This is purely informational and has no effect.",
1062 description
=> "Whether the drive should be included when making backups.",
1067 description
=> 'Whether the drive should considered for replication jobs.',
1073 enum
=> [qw(ignore report stop)],
1074 description
=> 'Read error action.',
1079 enum
=> [qw(enospc ignore report stop)],
1080 description
=> 'Write error action.',
1085 enum
=> [qw(native threads)],
1086 description
=> 'AIO type to use.',
1091 enum
=> [qw(ignore on)],
1092 description
=> 'Controls whether to pass discard/trim requests to the underlying storage.',
1097 description
=> 'Controls whether to detect and try to optimize writes of zeroes.',
1102 format
=> 'urlencoded',
1103 format_description
=> 'serial',
1104 maxLength
=> 20*3, # *3 since it's %xx url enoded
1105 description
=> "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1110 description
=> 'Mark this locally-managed volume as available on all nodes',
1111 verbose_description
=> "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1117 my %iothread_fmt = ( iothread
=> {
1119 description
=> "Whether to use iothreads for this drive",
1126 format
=> 'urlencoded',
1127 format_description
=> 'model',
1128 maxLength
=> 40*3, # *3 since it's %xx url enoded
1129 description
=> "The drive's reported model name, url-encoded, up to 40 bytes long.",
1137 description
=> "Number of queues.",
1143 my %scsiblock_fmt = (
1146 description
=> "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1155 description
=> "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1163 pattern
=> qr/^(0x)[0-9a-fA-F]{16}/,
1164 format_description
=> 'wwn',
1165 description
=> "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1170 my $add_throttle_desc = sub {
1171 my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1174 format_description
=> $unit,
1175 description
=> "Maximum $what in $longunit.",
1178 $d->{minimum
} = $minimum if defined($minimum);
1179 $drivedesc_base{$key} = $d;
1181 # throughput: (leaky bucket)
1182 $add_throttle_desc->('bps', 'integer', 'r/w speed', 'bps', 'bytes per second');
1183 $add_throttle_desc->('bps_rd', 'integer', 'read speed', 'bps', 'bytes per second');
1184 $add_throttle_desc->('bps_wr', 'integer', 'write speed', 'bps', 'bytes per second');
1185 $add_throttle_desc->('mbps', 'number', 'r/w speed', 'mbps', 'megabytes per second');
1186 $add_throttle_desc->('mbps_rd', 'number', 'read speed', 'mbps', 'megabytes per second');
1187 $add_throttle_desc->('mbps_wr', 'number', 'write speed', 'mbps', 'megabytes per second');
1188 $add_throttle_desc->('iops', 'integer', 'r/w I/O', 'iops', 'operations per second');
1189 $add_throttle_desc->('iops_rd', 'integer', 'read I/O', 'iops', 'operations per second');
1190 $add_throttle_desc->('iops_wr', 'integer', 'write I/O', 'iops', 'operations per second');
1192 # pools: (pool of IO before throttling starts taking effect)
1193 $add_throttle_desc->('mbps_max', 'number', 'unthrottled r/w pool', 'mbps', 'megabytes per second');
1194 $add_throttle_desc->('mbps_rd_max', 'number', 'unthrottled read pool', 'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_wr_max', 'number', 'unthrottled write pool', 'mbps', 'megabytes per second');
1196 $add_throttle_desc->('iops_max', 'integer', 'unthrottled r/w I/O pool', 'iops', 'operations per second');
1197 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool', 'iops', 'operations per second');
1198 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1201 $add_throttle_desc->('bps_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1202 $add_throttle_desc->('bps_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1203 $add_throttle_desc->('bps_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1204 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1206 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209 $drivedesc_base{'bps_rd_length'} = { alias
=> 'bps_rd_max_length' };
1210 $drivedesc_base{'bps_wr_length'} = { alias
=> 'bps_wr_max_length' };
1211 $drivedesc_base{'iops_rd_length'} = { alias
=> 'iops_rd_max_length' };
1212 $drivedesc_base{'iops_wr_length'} = { alias
=> 'iops_wr_max_length' };
1220 PVE
::JSONSchema
::register_format
("pve-qm-ide", $ide_fmt);
1224 type
=> 'string', format
=> $ide_fmt,
1225 description
=> "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1227 PVE
::JSONSchema
::register_standard_option
("pve-qm-ide", $idedesc);
1239 type
=> 'string', format
=> $scsi_fmt,
1240 description
=> "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1242 PVE
::JSONSchema
::register_standard_option
("pve-qm-scsi", $scsidesc);
1251 type
=> 'string', format
=> $sata_fmt,
1252 description
=> "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1254 PVE
::JSONSchema
::register_standard_option
("pve-qm-sata", $satadesc);
1262 type
=> 'string', format
=> $virtio_fmt,
1263 description
=> "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1265 PVE
::JSONSchema
::register_standard_option
("pve-qm-virtio", $virtiodesc);
1267 my $alldrive_fmt = {
1278 volume
=> { alias
=> 'file' },
1281 format
=> 'pve-volume-id-or-qm-path',
1283 format_description
=> 'volume',
1284 description
=> "The drive's backing volume.",
1286 format
=> get_standard_option
('pve-qm-image-format'),
1289 format
=> 'disk-size',
1290 format_description
=> 'DiskSize',
1291 description
=> "Disk size. This is purely informational and has no effect.",
1296 my $efidisk_desc = {
1298 type
=> 'string', format
=> $efidisk_fmt,
1299 description
=> "Configure a Disk for storing EFI vars",
1302 PVE
::JSONSchema
::register_standard_option
("pve-qm-efidisk", $efidisk_desc);
1307 type
=> 'string', format
=> 'pve-qm-usb-device',
1308 format_description
=> 'HOSTUSBDEVICE|spice',
1309 description
=> <<EODESCR,
1310 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1312 'bus-port(.port)*' (decimal numbers) or
1313 'vendor_id:product_id' (hexadeciaml numbers) or
1316 You can use the 'lsusb -t' command to list existing usb devices.
1318 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1320 The value 'spice' can be used to add a usb redirection devices for spice.
1326 description
=> "Specifies whether if given host option is a USB3 device or port.",
1333 type
=> 'string', format
=> $usb_fmt,
1334 description
=> "Configure an USB device (n is 0 to 4).",
1336 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1338 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1343 pattern
=> qr/$PCIRE(;$PCIRE)*/,
1344 format_description
=> 'HOSTPCIID[;HOSTPCIID2...]',
1345 description
=> <<EODESCR,
1346 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1347 of PCI virtual functions of the host. HOSTPCIID syntax is:
1349 'bus:dev.func' (hexadecimal numbers)
1351 You can us the 'lspci' command to list existing PCI devices.
1356 description
=> "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1362 pattern
=> '[^,;]+',
1363 format_description
=> 'string',
1364 description
=> "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1369 description
=> "Choose the PCI-express bus (needs the 'q35' machine model).",
1375 description
=> "Enable vfio-vga device support.",
1381 format_description
=> 'string',
1382 pattern
=> '[^/\.:]+',
1384 description
=> <<EODESCR
1385 The type of mediated device to use.
1386 An instance of this type will be created on startup of the VM and
1387 will be cleaned up when the VM stops.
1391 PVE
::JSONSchema
::register_format
('pve-qm-hostpci', $hostpci_fmt);
1395 type
=> 'string', format
=> 'pve-qm-hostpci',
1396 description
=> "Map host PCI devices into guest.",
1397 verbose_description
=> <<EODESCR,
1398 Map host PCI devices into guest.
1400 NOTE: This option allows direct access to host hardware. So it is no longer
1401 possible to migrate such machines - use with special care.
1403 CAUTION: Experimental! User reported problems with this option.
1406 PVE
::JSONSchema
::register_standard_option
("pve-qm-hostpci", $hostpcidesc);
1411 pattern
=> '(/dev/.+|socket)',
1412 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1413 verbose_description
=> <<EODESCR,
1414 Create a serial device inside the VM (n is 0 to 3), and pass through a
1415 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1416 host side (use 'qm terminal' to open a terminal connection).
1418 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1420 CAUTION: Experimental! User reported problems with this option.
1427 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1428 description
=> "Map host parallel devices (n is 0 to 2).",
1429 verbose_description
=> <<EODESCR,
1430 Map host parallel devices (n is 0 to 2).
1432 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1434 CAUTION: Experimental! User reported problems with this option.
1438 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1439 $confdesc->{"parallel$i"} = $paralleldesc;
1442 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1443 $confdesc->{"serial$i"} = $serialdesc;
1446 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
1447 $confdesc->{"hostpci$i"} = $hostpcidesc;
1450 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) {
1451 $drivename_hash->{"ide$i"} = 1;
1452 $confdesc->{"ide$i"} = $idedesc;
1455 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) {
1456 $drivename_hash->{"sata$i"} = 1;
1457 $confdesc->{"sata$i"} = $satadesc;
1460 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) {
1461 $drivename_hash->{"scsi$i"} = 1;
1462 $confdesc->{"scsi$i"} = $scsidesc ;
1465 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) {
1466 $drivename_hash->{"virtio$i"} = 1;
1467 $confdesc->{"virtio$i"} = $virtiodesc;
1470 $drivename_hash->{efidisk0
} = 1;
1471 $confdesc->{efidisk0
} = $efidisk_desc;
1473 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1474 $confdesc->{"usb$i"} = $usbdesc;
1479 type
=> 'string', format
=> 'pve-volume-id',
1480 description
=> "Reference to unused volumes. This is used internally, and should not be modified manually.",
1483 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1484 $confdesc->{"unused$i"} = $unuseddesc;
1487 my $kvm_api_version = 0;
1490 return $kvm_api_version if $kvm_api_version;
1492 open my $fh, '<', '/dev/kvm'
1495 # 0xae00 => KVM_GET_API_VERSION
1496 $kvm_api_version = ioctl($fh, 0xae00, 0);
1498 return $kvm_api_version;
1501 my $kvm_user_version = {};
1504 sub kvm_user_version
{
1507 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1508 my $st = stat($binary);
1510 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1511 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1512 $cachedmtime == $st->mtime;
1514 $kvm_user_version->{$binary} = 'unknown';
1515 $kvm_mtime->{$binary} = $st->mtime;
1519 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1520 $kvm_user_version->{$binary} = $2;
1524 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1527 return $kvm_user_version->{$binary};
1531 sub kernel_has_vhost_net
{
1532 return -c
'/dev/vhost-net';
1535 sub valid_drive_names
{
1536 # order is important - used to autoselect boot disk
1537 return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1538 (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1539 (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1540 (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1544 sub is_valid_drivename
{
1547 return defined($drivename_hash->{$dev});
1552 return defined($confdesc->{$key});
1556 sub get_cdrom_path
{
1558 return $cdrom_path if $cdrom_path;
1560 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1561 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1562 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1566 my ($storecfg, $vmid, $cdrom) = @_;
1568 if ($cdrom eq 'cdrom') {
1569 return get_cdrom_path
();
1570 } elsif ($cdrom eq 'none') {
1572 } elsif ($cdrom =~ m
|^/|) {
1575 return PVE
::Storage
::path
($storecfg, $cdrom);
1579 # try to convert old style file names to volume IDs
1580 sub filename_to_volume_id
{
1581 my ($vmid, $file, $media) = @_;
1583 if (!($file eq 'none' || $file eq 'cdrom' ||
1584 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1586 return undef if $file =~ m
|/|;
1588 if ($media && $media eq 'cdrom') {
1589 $file = "local:iso/$file";
1591 $file = "local:$vmid/$file";
1598 sub verify_media_type
{
1599 my ($opt, $vtype, $media) = @_;
1604 if ($media eq 'disk') {
1606 } elsif ($media eq 'cdrom') {
1609 die "internal error";
1612 return if ($vtype eq $etype);
1614 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1617 sub cleanup_drive_path
{
1618 my ($opt, $storecfg, $drive) = @_;
1620 # try to convert filesystem paths to volume IDs
1622 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1623 ($drive->{file
} !~ m
|^/dev/.+|) &&
1624 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1625 ($drive->{file
} !~ m/^\d+$/)) {
1626 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1627 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1628 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1629 verify_media_type
($opt, $vtype, $drive->{media
});
1630 $drive->{file
} = $volid;
1633 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1636 sub parse_hotplug_features
{
1641 return $res if $data eq '0';
1643 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1645 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1646 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1649 die "invalid hotplug feature '$feature'\n";
1655 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1656 sub pve_verify_hotplug_features
{
1657 my ($value, $noerr) = @_;
1659 return $value if parse_hotplug_features
($value);
1661 return undef if $noerr;
1663 die "unable to parse hotplug option\n";
1666 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1667 # [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1668 # [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1669 # [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1670 # [,iothread=on][,serial=serial][,model=model]
1673 my ($key, $data) = @_;
1675 my ($interface, $index);
1677 if ($key =~ m/^([^\d]+)(\d+)$/) {
1684 my $desc = $key =~ /^unused\d+$/ ?
$alldrive_fmt
1685 : $confdesc->{$key}->{format
};
1687 warn "invalid drive key: $key\n";
1690 my $res = eval { PVE
::JSONSchema
::parse_property_string
($desc, $data) };
1691 return undef if !$res;
1692 $res->{interface
} = $interface;
1693 $res->{index} = $index;
1696 foreach my $opt (qw(bps bps_rd bps_wr)) {
1697 if (my $bps = defined(delete $res->{$opt})) {
1698 if (defined($res->{"m$opt"})) {
1699 warn "both $opt and m$opt specified\n";
1703 $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1707 # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1708 for my $requirement (
1709 [mbps_max
=> 'mbps'],
1710 [mbps_rd_max
=> 'mbps_rd'],
1711 [mbps_wr_max
=> 'mbps_wr'],
1712 [miops_max
=> 'miops'],
1713 [miops_rd_max
=> 'miops_rd'],
1714 [miops_wr_max
=> 'miops_wr'],
1715 [bps_max_length
=> 'mbps_max'],
1716 [bps_rd_max_length
=> 'mbps_rd_max'],
1717 [bps_wr_max_length
=> 'mbps_wr_max'],
1718 [iops_max_length
=> 'iops_max'],
1719 [iops_rd_max_length
=> 'iops_rd_max'],
1720 [iops_wr_max_length
=> 'iops_wr_max']) {
1721 my ($option, $requires) = @$requirement;
1722 if ($res->{$option} && !$res->{$requires}) {
1723 warn "$option requires $requires\n";
1728 return undef if $error;
1730 return undef if $res->{mbps_rd
} && $res->{mbps
};
1731 return undef if $res->{mbps_wr
} && $res->{mbps
};
1732 return undef if $res->{iops_rd
} && $res->{iops
};
1733 return undef if $res->{iops_wr
} && $res->{iops
};
1735 if ($res->{media
} && ($res->{media
} eq 'cdrom')) {
1736 return undef if $res->{snapshot
} || $res->{trans
} || $res->{format
};
1737 return undef if $res->{heads
} || $res->{secs
} || $res->{cyls
};
1738 return undef if $res->{interface
} eq 'virtio';
1741 if (my $size = $res->{size
}) {
1742 return undef if !defined($res->{size
} = PVE
::JSONSchema
::parse_size
($size));
1749 my ($vmid, $drive) = @_;
1750 my $data = { %$drive };
1751 delete $data->{$_} for qw(index interface);
1752 return PVE
::JSONSchema
::print_property_string
($data, $alldrive_fmt);
1756 my($fh, $noerr) = @_;
1759 my $SG_GET_VERSION_NUM = 0x2282;
1761 my $versionbuf = "\x00" x
8;
1762 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1764 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1767 my $version = unpack("I", $versionbuf);
1768 if ($version < 30000) {
1769 die "scsi generic interface too old\n" if !$noerr;
1773 my $buf = "\x00" x
36;
1774 my $sensebuf = "\x00" x
8;
1775 my $cmd = pack("C x3 C x1", 0x12, 36);
1777 # see /usr/include/scsi/sg.h
1778 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1780 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1781 length($sensebuf), 0, length($buf), $buf,
1782 $cmd, $sensebuf, 6000);
1784 $ret = ioctl($fh, $SG_IO, $packet);
1786 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1790 my @res = unpack($sg_io_hdr_t, $packet);
1791 if ($res[17] || $res[18]) {
1792 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1797 (my $byte0, my $byte1, $res->{vendor
},
1798 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1800 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1801 $res->{type
} = $byte0 & 31;
1809 my $fh = IO
::File-
>new("+<$path") || return undef;
1810 my $res = scsi_inquiry
($fh, 1);
1816 sub print_tabletdevice_full
{
1817 my ($conf, $arch) = @_;
1819 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1821 # we use uhci for old VMs because tablet driver was buggy in older qemu
1823 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1829 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1832 sub print_keyboarddevice_full
{
1833 my ($conf, $arch, $machine) = @_;
1835 return undef if $arch ne 'aarch64';
1837 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1840 sub print_drivedevice_full
{
1841 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1846 if ($drive->{interface
} eq 'virtio') {
1847 my $pciaddr = print_pci_addr
("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1848 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1849 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread
};
1850 } elsif ($drive->{interface
} eq 'scsi') {
1852 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1853 my $unit = $drive->{index} % $maxdev;
1854 my $devicetype = 'hd';
1856 if (drive_is_cdrom
($drive)) {
1859 if ($drive->{file
} =~ m
|^/|) {
1860 $path = $drive->{file
};
1861 if (my $info = path_is_scsi
($path)) {
1862 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1863 $devicetype = 'block';
1864 } elsif ($info->{type
} == 1) { # tape
1865 $devicetype = 'generic';
1869 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1872 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1873 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type) // kvm_user_version
();
1874 if ($path =~ m/^iscsi\:\/\
// &&
1875 !min_version
($version, 4, 1)) {
1876 $devicetype = 'generic';
1880 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1881 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1883 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1886 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1887 $device .= ",rotation_rate=1";
1889 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1891 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1892 my $maxdev = ($drive->{interface
} eq 'sata') ?
$MAX_SATA_DISKS : 2;
1893 my $controller = int($drive->{index} / $maxdev);
1894 my $unit = $drive->{index} % $maxdev;
1895 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1897 $device = "ide-$devicetype";
1898 if ($drive->{interface
} eq 'ide') {
1899 $device .= ",bus=ide.$controller,unit=$unit";
1901 $device .= ",bus=ahci$controller.$unit";
1903 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1905 if ($devicetype eq 'hd') {
1906 if (my $model = $drive->{model
}) {
1907 $model = URI
::Escape
::uri_unescape
($model);
1908 $device .= ",model=$model";
1910 if ($drive->{ssd
}) {
1911 $device .= ",rotation_rate=1";
1914 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1915 } elsif ($drive->{interface
} eq 'usb') {
1917 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1919 die "unsupported interface type";
1922 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1924 if (my $serial = $drive->{serial
}) {
1925 $serial = URI
::Escape
::uri_unescape
($serial);
1926 $device .= ",serial=$serial";
1933 sub get_initiator_name
{
1936 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return undef;
1937 while (defined(my $line = <$fh>)) {
1938 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1947 sub print_drive_full
{
1948 my ($storecfg, $vmid, $drive) = @_;
1951 my $volid = $drive->{file
};
1954 if (drive_is_cdrom
($drive)) {
1955 $path = get_iso_path
($storecfg, $vmid, $volid);
1957 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1959 $path = PVE
::Storage
::path
($storecfg, $volid);
1960 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1961 $format = qemu_img_format
($scfg, $volname);
1969 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1970 foreach my $o (@qemu_drive_options) {
1971 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1974 # snapshot only accepts on|off
1975 if (defined($drive->{snapshot
})) {
1976 my $v = $drive->{snapshot
} ?
'on' : 'off';
1977 $opts .= ",snapshot=$v";
1980 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1981 my ($dir, $qmpname) = @$type;
1982 if (my $v = $drive->{"mbps$dir"}) {
1983 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1985 if (my $v = $drive->{"mbps${dir}_max"}) {
1986 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1988 if (my $v = $drive->{"bps${dir}_max_length"}) {
1989 $opts .= ",throttling.bps$qmpname-max-length=$v";
1991 if (my $v = $drive->{"iops${dir}"}) {
1992 $opts .= ",throttling.iops$qmpname=$v";
1994 if (my $v = $drive->{"iops${dir}_max"}) {
1995 $opts .= ",throttling.iops$qmpname-max=$v";
1997 if (my $v = $drive->{"iops${dir}_max_length"}) {
1998 $opts .= ",throttling.iops$qmpname-max-length=$v";
2002 $opts .= ",format=$format" if $format && !$drive->{format
};
2004 my $cache_direct = 0;
2006 if (my $cache = $drive->{cache
}) {
2007 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2008 } elsif (!drive_is_cdrom
($drive)) {
2009 $opts .= ",cache=none";
2013 # aio native works only with O_DIRECT
2014 if (!$drive->{aio
}) {
2016 $opts .= ",aio=native";
2018 $opts .= ",aio=threads";
2022 if (!drive_is_cdrom
($drive)) {
2024 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
2025 $detectzeroes = 'off';
2026 } elsif ($drive->{discard
}) {
2027 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
2029 # This used to be our default with discard not being specified:
2030 $detectzeroes = 'on';
2032 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2035 my $pathinfo = $path ?
"file=$path," : '';
2037 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2040 sub print_netdevice_full
{
2041 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2043 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
2045 my $device = $net->{model
};
2046 if ($net->{model
} eq 'virtio') {
2047 $device = 'virtio-net-pci';
2050 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
2051 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2052 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
2053 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2054 my $vectors = $net->{queues
} * 2 + 2;
2055 $tmpstr .= ",vectors=$vectors,mq=on";
2057 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
2059 if ($use_old_bios_files) {
2061 if ($device eq 'virtio-net-pci') {
2062 $romfile = 'pxe-virtio.rom';
2063 } elsif ($device eq 'e1000') {
2064 $romfile = 'pxe-e1000.rom';
2065 } elsif ($device eq 'ne2k') {
2066 $romfile = 'pxe-ne2k_pci.rom';
2067 } elsif ($device eq 'pcnet') {
2068 $romfile = 'pxe-pcnet.rom';
2069 } elsif ($device eq 'rtl8139') {
2070 $romfile = 'pxe-rtl8139.rom';
2072 $tmpstr .= ",romfile=$romfile" if $romfile;
2078 sub print_netdev_full
{
2079 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2082 if ($netid =~ m/^net(\d+)$/) {
2086 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
2088 my $ifname = "tap${vmid}i$i";
2090 # kvm uses TUNSETIFF ioctl, and that limits ifname length
2091 die "interface name '$ifname' is too long (max 15 character)\n"
2092 if length($ifname) >= 16;
2094 my $vhostparam = '';
2095 if (is_native
($arch)) {
2096 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
2099 my $vmname = $conf->{name
} || "vm$vmid";
2102 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
2104 if ($net->{bridge
}) {
2105 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2107 $netdev = "type=user,id=$netid,hostname=$vmname";
2110 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
2116 sub print_cpu_device
{
2117 my ($conf, $id) = @_;
2119 my $kvm = $conf->{kvm
} // 1;
2120 my $cpu = $kvm ?
"kvm64" : "qemu64";
2121 if (my $cputype = $conf->{cpu
}) {
2122 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
2123 or die "Cannot parse cpu description: $cputype\n";
2124 $cpu = $cpuconf->{cputype
};
2127 my $cores = $conf->{cores
} || 1;
2129 my $current_core = ($id - 1) % $cores;
2130 my $current_socket = int(($id - 1 - $current_core)/$cores);
2132 return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2136 'cirrus' => 'cirrus-vga',
2138 'vmware' => 'vmware-svga',
2139 'virtio' => 'virtio-vga',
2142 sub print_vga_device
{
2143 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2145 my $type = $vga_map->{$vga->{type
}};
2146 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2147 $type = 'virtio-gpu';
2149 my $vgamem_mb = $vga->{memory
};
2151 my $max_outputs = '';
2153 $type = $id ?
'qxl' : 'qxl-vga';
2155 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
2156 # set max outputs so linux can have up to 4 qxl displays with one device
2157 if (min_version
($machine_version, 4, 1)) {
2158 $max_outputs = ",max_outputs=4";
2163 die "no devicetype for $vga->{type}\n" if !$type;
2167 if ($vga->{type
} eq 'virtio') {
2168 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
2169 $memory = ",max_hostmem=$bytes";
2171 # from https://www.spice-space.org/multiple-monitors.html
2172 $memory = ",vgamem_mb=$vga->{memory}";
2173 my $ram = $vgamem_mb * 4;
2174 my $vram = $vgamem_mb * 2;
2175 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2177 $memory = ",vgamem_mb=$vga->{memory}";
2179 } elsif ($qxlnum && $id) {
2180 $memory = ",ram_size=67108864,vram_size=33554432";
2183 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2184 my $vgaid = "vga" . ($id // '');
2187 if ($q35 && $vgaid eq 'vga') {
2188 # the first display uses pcie.0 bus on q35 machines
2189 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
2191 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
2194 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2197 sub drive_is_cloudinit
{
2199 return $drive->{file
} =~ m
@[:/]vm-\d
+-cloudinit
(?
:\
.$QEMU_FORMAT_RE)?
$@;
2202 sub drive_is_cdrom
{
2203 my ($drive, $exclude_cloudinit) = @_;
2205 return 0 if $exclude_cloudinit && drive_is_cloudinit
($drive);
2207 return $drive && $drive->{media
} && ($drive->{media
} eq 'cdrom');
2211 sub parse_number_sets
{
2214 foreach my $part (split(/;/, $set)) {
2215 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2216 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2217 push @$res, [ $1, $2 ];
2219 die "invalid range: $part\n";
2228 my $res = PVE
::JSONSchema
::parse_property_string
($numa_fmt, $data);
2229 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
2230 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
2237 return undef if !$value;
2239 my $res = PVE
::JSONSchema
::parse_property_string
($hostpci_fmt, $value);
2241 my @idlist = split(/;/, $res->{host
});
2242 delete $res->{host
};
2243 foreach my $id (@idlist) {
2244 if ($id =~ m/\./) { # full id 00:00.1
2245 push @{$res->{pciid
}}, {
2248 } else { # partial id 00:00
2249 $res->{pciid
} = PVE
::SysFSTools
::lspci
($id);
2255 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2259 my $res = eval { PVE
::JSONSchema
::parse_property_string
($net_fmt, $data) };
2264 if (!defined($res->{macaddr
})) {
2265 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
2266 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
2271 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2272 sub parse_ipconfig
{
2275 my $res = eval { PVE
::JSONSchema
::parse_property_string
($ipconfig_fmt, $data) };
2281 if ($res->{gw
} && !$res->{ip
}) {
2282 warn 'gateway specified without specifying an IP address';
2285 if ($res->{gw6
} && !$res->{ip6
}) {
2286 warn 'IPv6 gateway specified without specifying an IPv6 address';
2289 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
2290 warn 'gateway specified together with DHCP';
2293 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
2295 warn "IPv6 gateway specified together with $res->{ip6} address";
2299 if (!$res->{ip
} && !$res->{ip6
}) {
2300 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
2309 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
2312 sub add_random_macs
{
2313 my ($settings) = @_;
2315 foreach my $opt (keys %$settings) {
2316 next if $opt !~ m/^net(\d+)$/;
2317 my $net = parse_net
($settings->{$opt});
2319 $settings->{$opt} = print_net
($net);
2323 sub vm_is_volid_owner
{
2324 my ($storecfg, $vmid, $volid) = @_;
2326 if ($volid !~ m
|^/|) {
2328 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
2329 if ($owner && ($owner == $vmid)) {
2337 sub vmconfig_register_unused_drive
{
2338 my ($storecfg, $vmid, $conf, $drive) = @_;
2340 if (drive_is_cloudinit
($drive)) {
2341 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2343 } elsif (!drive_is_cdrom
($drive)) {
2344 my $volid = $drive->{file
};
2345 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2346 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2351 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2355 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2356 format_description
=> 'UUID',
2357 description
=> "Set SMBIOS1 UUID.",
2362 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2363 format_description
=> 'Base64 encoded string',
2364 description
=> "Set SMBIOS1 version.",
2369 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2370 format_description
=> 'Base64 encoded string',
2371 description
=> "Set SMBIOS1 serial number.",
2376 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2377 format_description
=> 'Base64 encoded string',
2378 description
=> "Set SMBIOS1 manufacturer.",
2383 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2384 format_description
=> 'Base64 encoded string',
2385 description
=> "Set SMBIOS1 product ID.",
2390 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2391 format_description
=> 'Base64 encoded string',
2392 description
=> "Set SMBIOS1 SKU string.",
2397 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2398 format_description
=> 'Base64 encoded string',
2399 description
=> "Set SMBIOS1 family string.",
2404 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2412 my $res = eval { PVE
::JSONSchema
::parse_property_string
($smbios1_fmt, $data) };
2419 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2422 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2424 PVE
::JSONSchema
::register_format
('pve-qm-bootdisk', \
&verify_bootdisk
);
2425 sub verify_bootdisk
{
2426 my ($value, $noerr) = @_;
2428 return $value if is_valid_drivename
($value);
2430 return undef if $noerr;
2432 die "invalid boot disk '$value'\n";
2435 sub parse_watchdog
{
2438 return undef if !$value;
2440 my $res = eval { PVE
::JSONSchema
::parse_property_string
($watchdog_fmt, $value) };
2445 sub parse_guest_agent
{
2448 return {} if !defined($value->{agent
});
2450 my $res = eval { PVE
::JSONSchema
::parse_property_string
($agent_fmt, $value->{agent
}) };
2453 # if the agent is disabled ignore the other potentially set properties
2454 return {} if !$res->{enabled
};
2461 return {} if !$value;
2462 my $res = eval { PVE
::JSONSchema
::parse_property_string
($vga_fmt, $value) };
2467 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2468 sub verify_usb_device
{
2469 my ($value, $noerr) = @_;
2471 return $value if parse_usb_device
($value);
2473 return undef if $noerr;
2475 die "unable to parse usb device\n";
2478 # add JSON properties for create and set function
2479 sub json_config_properties
{
2482 foreach my $opt (keys %$confdesc) {
2483 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2484 $prop->{$opt} = $confdesc->{$opt};
2490 # return copy of $confdesc_cloudinit to generate documentation
2491 sub cloudinit_config_properties
{
2493 return dclone
($confdesc_cloudinit);
2497 my ($key, $value) = @_;
2499 die "unknown setting '$key'\n" if !$confdesc->{$key};
2501 my $type = $confdesc->{$key}->{type
};
2503 if (!defined($value)) {
2504 die "got undefined value\n";
2507 if ($value =~ m/[\n\r]/) {
2508 die "property contains a line feed\n";
2511 if ($type eq 'boolean') {
2512 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2513 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2514 die "type check ('boolean') failed - got '$value'\n";
2515 } elsif ($type eq 'integer') {
2516 return int($1) if $value =~ m/^(\d+)$/;
2517 die "type check ('integer') failed - got '$value'\n";
2518 } elsif ($type eq 'number') {
2519 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2520 die "type check ('number') failed - got '$value'\n";
2521 } elsif ($type eq 'string') {
2522 if (my $fmt = $confdesc->{$key}->{format
}) {
2523 PVE
::JSONSchema
::check_format
($fmt, $value);
2526 $value =~ s/^\"(.*)\"$/$1/;
2529 die "internal error"
2534 my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2536 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2538 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2540 if ($conf->{template
}) {
2541 # check if any base image is still used by a linked clone
2542 foreach_drive
($conf, sub {
2543 my ($ds, $drive) = @_;
2544 return if drive_is_cdrom
($drive);
2546 my $volid = $drive->{file
};
2547 return if !$volid || $volid =~ m
|^/|;
2549 die "base volume '$volid' is still in use by linked cloned\n"
2550 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2555 # only remove disks owned by this VM
2556 foreach_drive
($conf, sub {
2557 my ($ds, $drive) = @_;
2558 return if drive_is_cdrom
($drive, 1);
2560 my $volid = $drive->{file
};
2561 return if !$volid || $volid =~ m
|^/|;
2563 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2564 return if !$path || !$owner || ($owner != $vmid);
2566 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2567 warn "Could not remove disk '$volid', check manually: $@" if $@;
2570 # also remove unused disk
2571 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2572 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2573 my ($volid, $sid, $volname, $d) = @_;
2574 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2578 if (defined $replacement_conf) {
2579 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2581 PVE
::QemuConfig-
>destroy_config($vmid);
2585 sub parse_vm_config
{
2586 my ($filename, $raw) = @_;
2588 return undef if !defined($raw);
2591 digest
=> Digest
::SHA
::sha1_hex
($raw),
2596 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2597 || die "got strange filename '$filename'";
2605 my @lines = split(/\n/, $raw);
2606 foreach my $line (@lines) {
2607 next if $line =~ m/^\s*$/;
2609 if ($line =~ m/^\[PENDING\]\s*$/i) {
2610 $section = 'pending';
2611 if (defined($descr)) {
2613 $conf->{description
} = $descr;
2616 $conf = $res->{$section} = {};
2619 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2621 if (defined($descr)) {
2623 $conf->{description
} = $descr;
2626 $conf = $res->{snapshots
}->{$section} = {};
2630 if ($line =~ m/^\#(.*)\s*$/) {
2631 $descr = '' if !defined($descr);
2632 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2636 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2637 $descr = '' if !defined($descr);
2638 $descr .= PVE
::Tools
::decode_text
($2);
2639 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2640 $conf->{snapstate
} = $1;
2641 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2644 $conf->{$key} = $value;
2645 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2647 if ($section eq 'pending') {
2648 $conf->{delete} = $value; # we parse this later
2650 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2652 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2655 eval { $value = check_type
($key, $value); };
2657 warn "vm $vmid - unable to parse value of '$key' - $@";
2659 $key = 'ide2' if $key eq 'cdrom';
2660 my $fmt = $confdesc->{$key}->{format
};
2661 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2662 my $v = parse_drive
($key, $value);
2663 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2664 $v->{file
} = $volid;
2665 $value = print_drive
($vmid, $v);
2667 warn "vm $vmid - unable to parse value of '$key'\n";
2672 $conf->{$key} = $value;
2677 if (defined($descr)) {
2679 $conf->{description
} = $descr;
2681 delete $res->{snapstate
}; # just to be sure
2686 sub write_vm_config
{
2687 my ($filename, $conf) = @_;
2689 delete $conf->{snapstate
}; # just to be sure
2691 if ($conf->{cdrom
}) {
2692 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2693 $conf->{ide2
} = $conf->{cdrom
};
2694 delete $conf->{cdrom
};
2697 # we do not use 'smp' any longer
2698 if ($conf->{sockets
}) {
2699 delete $conf->{smp
};
2700 } elsif ($conf->{smp
}) {
2701 $conf->{sockets
} = $conf->{smp
};
2702 delete $conf->{cores
};
2703 delete $conf->{smp
};
2706 my $used_volids = {};
2708 my $cleanup_config = sub {
2709 my ($cref, $pending, $snapname) = @_;
2711 foreach my $key (keys %$cref) {
2712 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2713 $key eq 'snapstate' || $key eq 'pending';
2714 my $value = $cref->{$key};
2715 if ($key eq 'delete') {
2716 die "propertry 'delete' is only allowed in [PENDING]\n"
2718 # fixme: check syntax?
2721 eval { $value = check_type
($key, $value); };
2722 die "unable to parse value of '$key' - $@" if $@;
2724 $cref->{$key} = $value;
2726 if (!$snapname && is_valid_drivename
($key)) {
2727 my $drive = parse_drive
($key, $value);
2728 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2733 &$cleanup_config($conf);
2735 &$cleanup_config($conf->{pending
}, 1);
2737 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2738 die "internal error" if $snapname eq 'pending';
2739 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2742 # remove 'unusedX' settings if we re-add a volume
2743 foreach my $key (keys %$conf) {
2744 my $value = $conf->{$key};
2745 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2746 delete $conf->{$key};
2750 my $generate_raw_config = sub {
2751 my ($conf, $pending) = @_;
2755 # add description as comment to top of file
2756 if (defined(my $descr = $conf->{description
})) {
2758 foreach my $cl (split(/\n/, $descr)) {
2759 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2762 $raw .= "#\n" if $pending;
2766 foreach my $key (sort keys %$conf) {
2767 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2768 $raw .= "$key: $conf->{$key}\n";
2773 my $raw = &$generate_raw_config($conf);
2775 if (scalar(keys %{$conf->{pending
}})){
2776 $raw .= "\n[PENDING]\n";
2777 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2780 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2781 $raw .= "\n[$snapname]\n";
2782 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2792 # we use static defaults from our JSON schema configuration
2793 foreach my $key (keys %$confdesc) {
2794 if (defined(my $default = $confdesc->{$key}->{default})) {
2795 $res->{$key} = $default;
2803 my $vmlist = PVE
::Cluster
::get_vmlist
();
2805 return $res if !$vmlist || !$vmlist->{ids
};
2806 my $ids = $vmlist->{ids
};
2808 foreach my $vmid (keys %$ids) {
2809 my $d = $ids->{$vmid};
2810 next if !$d->{node
} || $d->{node
} ne $nodename;
2811 next if !$d->{type
} || $d->{type
} ne 'qemu';
2812 $res->{$vmid}->{exists} = 1;
2817 # test if VM uses local resources (to prevent migration)
2818 sub check_local_resources
{
2819 my ($conf, $noerr) = @_;
2823 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2824 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2826 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2828 foreach my $k (keys %$conf) {
2829 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2830 # sockets are safe: they will recreated be on the target side post-migrate
2831 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2832 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2835 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2840 # check if used storages are available on all nodes (use by migrate)
2841 sub check_storage_availability
{
2842 my ($storecfg, $conf, $node) = @_;
2844 foreach_drive
($conf, sub {
2845 my ($ds, $drive) = @_;
2847 my $volid = $drive->{file
};
2850 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2853 # check if storage is available on both nodes
2854 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2855 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2859 # list nodes where all VM images are available (used by has_feature API)
2861 my ($conf, $storecfg) = @_;
2863 my $nodelist = PVE
::Cluster
::get_nodelist
();
2864 my $nodehash = { map { $_ => 1 } @$nodelist };
2865 my $nodename = PVE
::INotify
::nodename
();
2867 foreach_drive
($conf, sub {
2868 my ($ds, $drive) = @_;
2870 my $volid = $drive->{file
};
2873 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2875 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2876 if ($scfg->{disable
}) {
2878 } elsif (my $avail = $scfg->{nodes
}) {
2879 foreach my $node (keys %$nodehash) {
2880 delete $nodehash->{$node} if !$avail->{$node};
2882 } elsif (!$scfg->{shared
}) {
2883 foreach my $node (keys %$nodehash) {
2884 delete $nodehash->{$node} if $node ne $nodename
2893 sub check_local_storage_availability
{
2894 my ($conf, $storecfg) = @_;
2896 my $nodelist = PVE
::Cluster
::get_nodelist
();
2897 my $nodehash = { map { $_ => {} } @$nodelist };
2899 foreach_drive
($conf, sub {
2900 my ($ds, $drive) = @_;
2902 my $volid = $drive->{file
};
2905 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2907 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2909 if ($scfg->{disable
}) {
2910 foreach my $node (keys %$nodehash) {
2911 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2913 } elsif (my $avail = $scfg->{nodes
}) {
2914 foreach my $node (keys %$nodehash) {
2915 if (!$avail->{$node}) {
2916 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2923 foreach my $node (values %$nodehash) {
2924 if (my $unavail = $node->{unavailable_storages
}) {
2925 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2932 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2934 my ($vmid, $nocheck, $node) = @_;
2936 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2937 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2942 my $vzlist = config_list
();
2944 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2946 while (defined(my $de = $fd->read)) {
2947 next if $de !~ m/^(\d+)\.pid$/;
2949 next if !defined($vzlist->{$vmid});
2950 if (my $pid = check_running
($vmid)) {
2951 $vzlist->{$vmid}->{pid
} = $pid;
2959 my ($storecfg, $conf) = @_;
2961 my $bootdisk = $conf->{bootdisk
};
2962 return undef if !$bootdisk;
2963 return undef if !is_valid_drivename
($bootdisk);
2965 return undef if !$conf->{$bootdisk};
2967 my $drive = parse_drive
($bootdisk, $conf->{$bootdisk});
2968 return undef if !defined($drive);
2970 return undef if drive_is_cdrom
($drive);
2972 my $volid = $drive->{file
};
2973 return undef if !$volid;
2975 return $drive->{size
};
2978 our $vmstatus_return_properties = {
2979 vmid
=> get_standard_option
('pve-vmid'),
2981 description
=> "Qemu process status.",
2983 enum
=> ['stopped', 'running'],
2986 description
=> "Maximum memory in bytes.",
2989 renderer
=> 'bytes',
2992 description
=> "Root disk size in bytes.",
2995 renderer
=> 'bytes',
2998 description
=> "VM name.",
3003 description
=> "Qemu QMP agent status.",
3008 description
=> "PID of running qemu process.",
3013 description
=> "Uptime.",
3016 renderer
=> 'duration',
3019 description
=> "Maximum usable CPUs.",
3024 description
=> "The current config lock, if any.",
3030 my $last_proc_pid_stat;
3032 # get VM status information
3033 # This must be fast and should not block ($full == false)
3034 # We only query KVM using QMP if $full == true (this can be slow)
3036 my ($opt_vmid, $full) = @_;
3040 my $storecfg = PVE
::Storage
::config
();
3042 my $list = vzlist
();
3043 my $defaults = load_defaults
();
3045 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
3047 my $cpucount = $cpuinfo->{cpus
} || 1;
3049 foreach my $vmid (keys %$list) {
3050 next if $opt_vmid && ($vmid ne $opt_vmid);
3052 my $conf = PVE
::QemuConfig-
>load_config($vmid);
3054 my $d = { vmid
=> $vmid };
3055 $d->{pid
} = $list->{$vmid}->{pid
};
3057 # fixme: better status?
3058 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
3060 my $size = disksize
($storecfg, $conf);
3061 if (defined($size)) {
3062 $d->{disk
} = 0; # no info available
3063 $d->{maxdisk
} = $size;
3069 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
3070 * ($conf->{cores
} || $defaults->{cores
});
3071 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
3072 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
3074 $d->{name
} = $conf->{name
} || "VM $vmid";
3075 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
3076 : $defaults->{memory
}*(1024*1024);
3078 if ($conf->{balloon
}) {
3079 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
3080 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
3081 : $defaults->{shares
};
3092 $d->{diskwrite
} = 0;
3094 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
3096 $d->{serial
} = 1 if conf_has_serial
($conf);
3097 $d->{lock} = $conf->{lock} if $conf->{lock};
3102 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
3103 foreach my $dev (keys %$netdev) {
3104 next if $dev !~ m/^tap([1-9]\d*)i/;
3106 my $d = $res->{$vmid};
3109 $d->{netout
} += $netdev->{$dev}->{receive
};
3110 $d->{netin
} += $netdev->{$dev}->{transmit
};
3113 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
3114 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
3119 my $ctime = gettimeofday
;
3121 foreach my $vmid (keys %$list) {
3123 my $d = $res->{$vmid};
3124 my $pid = $d->{pid
};
3127 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
3128 next if !$pstat; # not running
3130 my $used = $pstat->{utime} + $pstat->{stime
};
3132 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
3134 if ($pstat->{vsize
}) {
3135 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
3138 my $old = $last_proc_pid_stat->{$pid};
3140 $last_proc_pid_stat->{$pid} = {
3148 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
3150 if ($dtime > 1000) {
3151 my $dutime = $used - $old->{used
};
3153 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
3154 $last_proc_pid_stat->{$pid} = {
3160 $d->{cpu
} = $old->{cpu
};
3164 return $res if !$full;
3166 my $qmpclient = PVE
::QMPClient-
>new();
3168 my $ballooncb = sub {
3169 my ($vmid, $resp) = @_;
3171 my $info = $resp->{'return'};
3172 return if !$info->{max_mem
};
3174 my $d = $res->{$vmid};
3176 # use memory assigned to VM
3177 $d->{maxmem
} = $info->{max_mem
};
3178 $d->{balloon
} = $info->{actual
};
3180 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
3181 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
3182 $d->{freemem
} = $info->{free_mem
};
3185 $d->{ballooninfo
} = $info;
3188 my $blockstatscb = sub {
3189 my ($vmid, $resp) = @_;
3190 my $data = $resp->{'return'} || [];
3191 my $totalrdbytes = 0;
3192 my $totalwrbytes = 0;
3194 for my $blockstat (@$data) {
3195 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
3196 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
3198 $blockstat->{device
} =~ s/drive-//;
3199 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
3201 $res->{$vmid}->{diskread
} = $totalrdbytes;
3202 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
3205 my $statuscb = sub {
3206 my ($vmid, $resp) = @_;
3208 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3209 # this fails if ballon driver is not loaded, so this must be
3210 # the last commnand (following command are aborted if this fails).
3211 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3213 my $status = 'unknown';
3214 if (!defined($status = $resp->{'return'}->{status
})) {
3215 warn "unable to get VM status\n";
3219 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3222 foreach my $vmid (keys %$list) {
3223 next if $opt_vmid && ($vmid ne $opt_vmid);
3224 next if !$res->{$vmid}->{pid
}; # not running
3225 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3228 $qmpclient->queue_execute(undef, 2);
3230 foreach my $vmid (keys %$list) {
3231 next if $opt_vmid && ($vmid ne $opt_vmid);
3232 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3239 my ($conf, $func, @param) = @_;
3241 foreach my $ds (valid_drive_names
()) {
3242 next if !defined($conf->{$ds});
3244 my $drive = parse_drive
($ds, $conf->{$ds});
3247 &$func($ds, $drive, @param);
3252 my ($conf, $func, @param) = @_;
3256 my $test_volid = sub {
3257 my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3261 $volhash->{$volid}->{cdrom
} //= 1;
3262 $volhash->{$volid}->{cdrom
} = 0 if !$is_cdrom;
3264 $volhash->{$volid}->{replicate
} //= 0;
3265 $volhash->{$volid}->{replicate
} = 1 if $replicate;
3267 $volhash->{$volid}->{shared
} //= 0;
3268 $volhash->{$volid}->{shared
} = 1 if $shared;
3270 $volhash->{$volid}->{referenced_in_config
} //= 0;
3271 $volhash->{$volid}->{referenced_in_config
} = 1 if !defined($snapname);
3273 $volhash->{$volid}->{referenced_in_snapshot
}->{$snapname} = 1
3274 if defined($snapname);
3275 $volhash->{$volid}->{size
} = $size if $size;
3278 foreach_drive
($conf, sub {
3279 my ($ds, $drive) = @_;
3280 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, undef, $drive->{size
});
3283 foreach my $snapname (keys %{$conf->{snapshots
}}) {
3284 my $snap = $conf->{snapshots
}->{$snapname};
3285 $test_volid->($snap->{vmstate
}, 0, 1, $snapname);
3286 foreach_drive
($snap, sub {
3287 my ($ds, $drive) = @_;
3288 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, $snapname);
3292 foreach my $volid (keys %$volhash) {
3293 &$func($volid, $volhash->{$volid}, @param);
3297 sub conf_has_serial
{
3300 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3301 if ($conf->{"serial$i"}) {
3309 sub conf_has_audio
{
3310 my ($conf, $id) = @_;
3313 my $audio = $conf->{"audio$id"};
3314 return undef if !defined($audio);
3316 my $audioproperties = PVE
::JSONSchema
::parse_property_string
($audio_fmt, $audio);
3317 my $audiodriver = $audioproperties->{driver
} // 'spice';
3320 dev
=> $audioproperties->{device
},
3321 dev_id
=> "audiodev$id",
3322 backend
=> $audiodriver,
3323 backend_id
=> "$audiodriver-backend${id}",
3327 sub vga_conf_has_spice
{
3330 my $vgaconf = parse_vga
($vga);
3331 my $vgatype = $vgaconf->{type
};
3332 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3339 return get_host_arch
() eq $arch;
3342 my $default_machines = {
3347 sub get_basic_machine_info
{
3348 my ($conf, $forcemachine) = @_;
3350 my $arch = $conf->{arch
} // get_host_arch
();
3351 my $machine = $forcemachine || $conf->{machine
} || $default_machines->{$arch};
3352 return ($arch, $machine);
3355 sub get_ovmf_files
($) {
3358 my $ovmf = $OVMF->{$arch}
3359 or die "no OVMF images known for architecture '$arch'\n";
3365 aarch64
=> '/usr/bin/qemu-system-aarch64',
3366 x86_64
=> '/usr/bin/qemu-system-x86_64',
3368 sub get_command_for_arch
($) {
3370 return '/usr/bin/kvm' if is_native
($arch);
3372 my $cmd = $Arch2Qemu->{$arch}
3373 or die "don't know how to emulate architecture '$arch'\n";
3377 sub get_cpu_options
{
3378 my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3381 my $ostype = $conf->{ostype
};
3383 my $cpu = $kvm ?
"kvm64" : "qemu64";
3384 if ($arch eq 'aarch64') {
3385 $cpu = 'cortex-a57';
3388 if (my $cputype = $conf->{cpu
}) {
3389 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
3390 or die "Cannot parse cpu description: $cputype\n";
3391 $cpu = $cpuconf->{cputype
};
3392 $kvm_off = 1 if $cpuconf->{hidden
};
3393 $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3395 if (defined(my $flags = $cpuconf->{flags
})) {
3396 push @$cpuFlags, split(";", $flags);
3400 push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3402 push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3404 push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3406 push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3408 if (min_version
($machine_version, 2, 3) && $arch eq 'x86_64') {
3410 push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3411 push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3414 add_hyperv_enlightenments
($cpuFlags, $winversion, $machine_version, $conf->{bios
}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3416 push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3418 push @$cpuFlags, 'kvm=off' if $kvm_off;
3420 if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3421 push @$cpuFlags, "vendor=${cpu_vendor}"
3422 if $cpu_vendor ne 'default';
3423 } elsif ($arch ne 'aarch64') {
3424 die "internal error"; # should not happen
3427 $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3429 return ('-cpu', $cpu);
3432 sub config_to_command
{
3433 my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3436 my $globalFlags = [];
3437 my $machineFlags = [];
3442 my $vernum = 0; # unknown
3443 my $ostype = $conf->{ostype
};
3444 my $winversion = windows_version
($ostype);
3445 my $kvm = $conf->{kvm
};
3447 my ($arch, $machine_type) = get_basic_machine_info
($conf, $forcemachine);
3448 my $kvm_binary = get_command_for_arch
($arch);
3449 my $kvmver = kvm_user_version
($kvm_binary);
3450 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type) // $kvmver;
3451 $kvm //= 1 if is_native
($arch);
3454 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3455 if !defined kvm_version
();
3458 if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3459 $vernum = $1*1000000+$2*1000;
3460 } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3461 $vernum = $1*1000000+$2*1000+$3;
3464 die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3466 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3467 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3468 my $use_old_bios_files = undef;
3469 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3471 my $cpuunits = defined($conf->{cpuunits
}) ?
3472 $conf->{cpuunits
} : $defaults->{cpuunits
};
3474 push @$cmd, $kvm_binary;
3476 push @$cmd, '-id', $vmid;
3478 my $vmname = $conf->{name
} || "vm$vmid";
3480 push @$cmd, '-name', $vmname;
3484 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3485 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3486 push @$cmd, '-mon', "chardev=qmp,mode=control";
3488 if (min_version
($machine_version, 2, 12)) {
3489 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3490 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3493 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3495 push @$cmd, '-daemonize';
3497 if ($conf->{smbios1
}) {
3498 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3499 if ($smbios_conf->{base64
}) {
3500 # Do not pass base64 flag to qemu
3501 delete $smbios_conf->{base64
};
3502 my $smbios_string = "";
3503 foreach my $key (keys %$smbios_conf) {
3505 if ($key eq "uuid") {
3506 $value = $smbios_conf->{uuid
}
3508 $value = decode_base64
($smbios_conf->{$key});
3510 # qemu accepts any binary data, only commas need escaping by double comma
3512 $smbios_string .= "," . $key . "=" . $value if $value;
3514 push @$cmd, '-smbios', "type=1" . $smbios_string;
3516 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3520 if ($conf->{vmgenid
}) {
3521 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3524 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3525 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3526 die "uefi base image not found\n" if ! -f
$ovmf_code;
3530 if (my $efidisk = $conf->{efidisk0
}) {
3531 my $d = PVE
::JSONSchema
::parse_property_string
($efidisk_fmt, $efidisk);
3532 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3533 $format = $d->{format
};
3535 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3536 if (!defined($format)) {
3537 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3538 $format = qemu_img_format
($scfg, $volname);
3542 die "efidisk format must be specified\n"
3543 if !defined($format);
3546 warn "no efidisk configured! Using temporary efivars disk.\n";
3547 $path = "/tmp/$vmid-ovmf.fd";
3548 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3552 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3553 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3558 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3559 if (min_version
($machine_version, 4, 0)) {
3560 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3562 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3566 # add usb controllers
3567 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
($conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3568 push @$devices, @usbcontrollers if @usbcontrollers;
3569 my $vga = parse_vga
($conf->{vga
});
3571 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3572 $vga->{type
} = 'qxl' if $qxlnum;
3574 if (!$vga->{type
}) {
3575 if ($arch eq 'aarch64') {
3576 $vga->{type
} = 'virtio';
3577 } elsif (min_version
($machine_version, 2, 9)) {
3578 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3580 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3584 # enable absolute mouse coordinates (needed by vnc)
3586 if (defined($conf->{tablet
})) {
3587 $tablet = $conf->{tablet
};
3589 $tablet = $defaults->{tablet
};
3590 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3591 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3595 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3596 my $kbd = print_keyboarddevice_full
($conf, $arch);
3597 push @$devices, '-device', $kbd if defined($kbd);
3601 my $gpu_passthrough;
3604 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
3605 my $id = "hostpci$i";
3606 my $d = parse_hostpci
($conf->{$id});
3609 if (my $pcie = $d->{pcie
}) {
3610 die "q35 machine model is not enabled" if !$q35;
3611 # win7 wants to have the pcie devices directly on the pcie bus
3612 # instead of in the root port
3613 if ($winversion == 7) {
3614 $pciaddr = print_pcie_addr
("${id}bus0");
3616 # add more root ports if needed, 4 are present by default
3617 # by pve-q35 cfgs, rest added here on demand.
3619 push @$devices, '-device', print_pcie_root_port
($i);
3621 $pciaddr = print_pcie_addr
($id);
3624 $pciaddr = print_pci_addr
($id, $bridges, $arch, $machine_type);
3628 if ($d->{'x-vga'}) {
3629 $xvga = ',x-vga=on' if !($conf->{bios
} && $conf->{bios
} eq 'ovmf');
3631 $vga->{type
} = 'none' if !defined($conf->{vga
});
3632 $gpu_passthrough = 1;
3635 my $pcidevices = $d->{pciid
};
3636 my $multifunction = 1 if @$pcidevices > 1;
3639 if ($d->{mdev
} && scalar(@$pcidevices) == 1) {
3640 my $pci_id = $pcidevices->[0]->{id
};
3641 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $i);
3642 $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3643 } elsif ($d->{mdev
}) {
3644 warn "ignoring mediated device '$id' with multifunction device\n";
3648 foreach my $pcidevice (@$pcidevices) {
3649 my $devicestr = "vfio-pci";
3652 $devicestr .= ",sysfsdev=$sysfspath";
3654 $devicestr .= ",host=$pcidevice->{id}";
3657 my $mf_addr = $multifunction ?
".$j" : '';
3658 $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3661 $devicestr .= ',rombar=0' if defined($d->{rombar
}) && !$d->{rombar
};
3662 $devicestr .= "$xvga";
3663 $devicestr .= ",multifunction=on" if $multifunction;
3664 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile
};
3667 push @$devices, '-device', $devicestr;
3673 my $usb_dev_features = {};
3674 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3676 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
($conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features);
3677 push @$devices, @usbdevices if @usbdevices;
3679 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3680 if (my $path = $conf->{"serial$i"}) {
3681 if ($path eq 'socket') {
3682 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3683 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3684 # On aarch64, serial0 is the UART device. Qemu only allows
3685 # connecting UART devices via the '-serial' command line, as
3686 # the device has a fixed slot on the hardware...
3687 if ($arch eq 'aarch64' && $i == 0) {
3688 push @$devices, '-serial', "chardev:serial$i";
3690 push @$devices, '-device', "isa-serial,chardev=serial$i";
3693 die "no such serial device\n" if ! -c
$path;
3694 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3695 push @$devices, '-device', "isa-serial,chardev=serial$i";
3701 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3702 if (my $path = $conf->{"parallel$i"}) {
3703 die "no such parallel device\n" if ! -c
$path;
3704 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3705 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3706 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3710 if (my $audio = conf_has_audio
($conf)) {
3712 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3714 my $id = $audio->{dev_id
};
3715 if ($audio->{dev
} eq 'AC97') {
3716 push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3717 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3718 push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3719 push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3720 push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3722 die "unkown audio device '$audio->{dev}', implement me!";
3725 push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3729 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3730 $sockets = $conf->{sockets
} if $conf->{sockets
};
3732 my $cores = $conf->{cores
} || 1;
3734 my $maxcpus = $sockets * $cores;
3736 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3738 my $allowed_vcpus = $cpuinfo->{cpus
};
3740 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3741 if ($allowed_vcpus < $maxcpus);
3743 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3745 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3746 for (my $i = 2; $i <= $vcpus; $i++) {
3747 my $cpustr = print_cpu_device
($conf,$i);
3748 push @$cmd, '-device', $cpustr;
3753 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3755 push @$cmd, '-nodefaults';
3757 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
3759 my $bootindex_hash = {};
3761 foreach my $o (split(//, $bootorder)) {
3762 $bootindex_hash->{$o} = $i*100;
3766 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3768 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3770 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3772 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3773 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3774 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3775 push @$cmd, '-vnc', "unix:$socket,password";
3777 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3778 push @$cmd, '-nographic';
3782 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3784 my $useLocaltime = $conf->{localtime};
3786 if ($winversion >= 5) { # windows
3787 $useLocaltime = 1 if !defined($conf->{localtime});
3789 # use time drift fix when acpi is enabled
3790 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3791 $tdf = 1 if !defined($conf->{tdf
});
3795 if ($winversion >= 6) {
3796 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3797 push @$cmd, '-no-hpet';
3800 push @$rtcFlags, 'driftfix=slew' if $tdf;
3803 push @$machineFlags, 'accel=tcg';
3806 if ($machine_type) {
3807 push @$machineFlags, "type=${machine_type}";
3810 if (($conf->{startdate
}) && ($conf->{startdate
} ne 'now')) {
3811 push @$rtcFlags, "base=$conf->{startdate}";
3812 } elsif ($useLocaltime) {
3813 push @$rtcFlags, 'base=localtime';
3816 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3818 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3820 push @$cmd, '-S' if $conf->{freeze
};
3822 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3824 my $guest_agent = parse_guest_agent
($conf);
3826 if ($guest_agent->{enabled
}) {
3827 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3828 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3830 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3831 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3832 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3833 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3834 } elsif ($guest_agent->{type
} eq 'isa') {
3835 push @$devices, '-device', "isa-serial,chardev=qga0";
3844 for(my $i = 1; $i < $qxlnum; $i++){
3845 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3848 # assume other OS works like Linux
3849 my ($ram, $vram) = ("134217728", "67108864");
3850 if ($vga->{memory
}) {
3851 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3852 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3854 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3855 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3859 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3861 my $nodename = PVE
::INotify
::nodename
();
3862 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3863 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3864 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3866 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3867 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3868 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3870 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3871 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3873 my $spice_enhancement = PVE
::JSONSchema
::parse_property_string
($spice_enhancements_fmt, $conf->{spice_enhancements
} // '');
3874 if ($spice_enhancement->{foldersharing
}) {
3875 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3876 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3879 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3880 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming
};
3881 push @$devices, '-spice', "$spice_opts";
3884 # enable balloon by default, unless explicitly disabled
3885 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3886 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3887 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3890 if ($conf->{watchdog
}) {
3891 my $wdopts = parse_watchdog
($conf->{watchdog
});
3892 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3893 my $watchdog = $wdopts->{model
} || 'i6300esb';
3894 push @$devices, '-device', "$watchdog$pciaddr";
3895 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3899 my $scsicontroller = {};
3900 my $ahcicontroller = {};
3901 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3903 # Add iscsi initiator name if available
3904 if (my $initiator = get_initiator_name
()) {
3905 push @$devices, '-iscsi', "initiator-name=$initiator";
3908 foreach_drive
($conf, sub {
3909 my ($ds, $drive) = @_;
3911 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3912 push @$vollist, $drive->{file
};
3915 # ignore efidisk here, already added in bios/fw handling code above
3916 return if $drive->{interface
} eq 'efidisk';
3918 $use_virtio = 1 if $ds =~ m/^virtio/;
3920 if (drive_is_cdrom
($drive)) {
3921 if ($bootindex_hash->{d
}) {
3922 $drive->{bootindex
} = $bootindex_hash->{d
};
3923 $bootindex_hash->{d
} += 1;
3926 if ($bootindex_hash->{c
}) {
3927 $drive->{bootindex
} = $bootindex_hash->{c
} if $conf->{bootdisk
} && ($conf->{bootdisk
} eq $ds);
3928 $bootindex_hash->{c
} += 1;
3932 if($drive->{interface
} eq 'virtio'){
3933 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3936 if ($drive->{interface
} eq 'scsi') {
3938 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3940 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3941 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3944 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3945 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3946 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3947 } elsif ($drive->{iothread
}) {
3948 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3952 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3953 $queues = ",num_queues=$drive->{queues}";
3956 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3957 $scsicontroller->{$controller}=1;
3960 if ($drive->{interface
} eq 'sata') {
3961 my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3962 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3963 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3964 $ahcicontroller->{$controller}=1;
3967 my $drive_cmd = print_drive_full
($storecfg, $vmid, $drive);
3968 push @$devices, '-drive',$drive_cmd;
3969 push @$devices, '-device', print_drivedevice_full
($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3972 for (my $i = 0; $i < $MAX_NETS; $i++) {
3973 next if !$conf->{"net$i"};
3974 my $d = parse_net
($conf->{"net$i"});
3977 $use_virtio = 1 if $d->{model
} eq 'virtio';
3979 if ($bootindex_hash->{n
}) {
3980 $d->{bootindex
} = $bootindex_hash->{n
};
3981 $bootindex_hash->{n
} += 1;
3984 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, "net$i");
3985 push @$devices, '-netdev', $netdevfull;
3987 my $netdevicefull = print_netdevice_full
($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3988 push @$devices, '-device', $netdevicefull;
3991 if ($conf->{ivshmem
}) {
3992 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3996 $bus = print_pcie_addr
("ivshmem");
3998 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4001 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4002 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4004 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4005 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4010 if (min_version
($machine_version, 2, 3)) {
4015 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4017 for my $k (sort {$b cmp $a} keys %$bridges) {
4018 $pciaddr = print_pci_addr
("pci.$k", undef, $arch, $machine_type);
4019 unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4023 push @$cmd, @$devices;
4024 push @$cmd, '-rtc', join(',', @$rtcFlags)
4025 if scalar(@$rtcFlags);
4026 push @$cmd, '-machine', join(',', @$machineFlags)
4027 if scalar(@$machineFlags);
4028 push @$cmd, '-global', join(',', @$globalFlags)
4029 if scalar(@$globalFlags);
4031 if (my $vmstate = $conf->{vmstate
}) {
4032 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4033 push @$vollist, $vmstate;
4034 push @$cmd, '-loadstate', $statepath;
4038 if ($conf->{args
}) {
4039 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4043 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
4049 my $res = mon_cmd
($vmid, 'query-spice');
4051 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4054 sub vm_devices_list
{
4057 my $res = mon_cmd
($vmid, 'query-pci');
4058 my $devices_to_check = [];
4060 foreach my $pcibus (@$res) {
4061 push @$devices_to_check, @{$pcibus->{devices
}},
4064 while (@$devices_to_check) {
4066 for my $d (@$devices_to_check) {
4067 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4068 next if !$d->{'pci_bridge'};
4070 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4071 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4073 $devices_to_check = $to_check;
4076 my $resblock = mon_cmd
($vmid, 'query-block');
4077 foreach my $block (@$resblock) {
4078 if($block->{device
} =~ m/^drive-(\S+)/){
4083 my $resmice = mon_cmd
($vmid, 'query-mice');
4084 foreach my $mice (@$resmice) {
4085 if ($mice->{name
} eq 'QEMU HID Tablet') {
4086 $devices->{tablet
} = 1;
4091 # for usb devices there is no query-usb
4092 # but we can iterate over the entries in
4093 # qom-list path=/machine/peripheral
4094 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4095 foreach my $per (@$resperipheral) {
4096 if ($per->{name
} =~ m/^usb\d+$/) {
4097 $devices->{$per->{name
}} = 1;
4105 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4107 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4109 my $devices_list = vm_devices_list
($vmid);
4110 return 1 if defined($devices_list->{$deviceid});
4112 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4114 if ($deviceid eq 'tablet') {
4116 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4118 } elsif ($deviceid eq 'keyboard') {
4120 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4122 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4124 die "usb hotplug currently not reliable\n";
4125 # since we can't reliably hot unplug all added usb devices
4126 # and usb passthrough disables live migration
4127 # we disable usb hotplugging for now
4128 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device));
4130 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4132 qemu_iothread_add
($vmid, $deviceid, $device);
4134 qemu_driveadd
($storecfg, $vmid, $device);
4135 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4137 qemu_deviceadd
($vmid, $devicefull);
4138 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4140 eval { qemu_drivedel
($vmid, $deviceid); };
4145 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4148 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4149 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4150 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4152 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4154 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4155 qemu_iothread_add
($vmid, $deviceid, $device);
4156 $devicefull .= ",iothread=iothread-$deviceid";
4159 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4160 $devicefull .= ",num_queues=$device->{queues}";
4163 qemu_deviceadd
($vmid, $devicefull);
4164 qemu_deviceaddverify
($vmid, $deviceid);
4166 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4168 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4169 qemu_driveadd
($storecfg, $vmid, $device);
4171 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4172 eval { qemu_deviceadd
($vmid, $devicefull); };
4174 eval { qemu_drivedel
($vmid, $deviceid); };
4179 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4181 return undef if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4183 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4184 my $use_old_bios_files = undef;
4185 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4187 my $netdevicefull = print_netdevice_full
($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4188 qemu_deviceadd
($vmid, $netdevicefull);
4190 qemu_deviceaddverify
($vmid, $deviceid);
4191 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4194 eval { qemu_netdevdel
($vmid, $deviceid); };
4199 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4202 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4203 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4205 qemu_deviceadd
($vmid, $devicefull);
4206 qemu_deviceaddverify
($vmid, $deviceid);
4209 die "can't hotplug device '$deviceid'\n";
4215 # fixme: this should raise exceptions on error!
4216 sub vm_deviceunplug
{
4217 my ($vmid, $conf, $deviceid) = @_;
4219 my $devices_list = vm_devices_list
($vmid);
4220 return 1 if !defined($devices_list->{$deviceid});
4222 die "can't unplug bootdisk" if $conf->{bootdisk
} && $conf->{bootdisk
} eq $deviceid;
4224 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4226 qemu_devicedel
($vmid, $deviceid);
4228 } elsif ($deviceid =~ m/^usb\d+$/) {
4230 die "usb hotplug currently not reliable\n";
4231 # when unplugging usb devices this way,
4232 # there may be remaining usb controllers/hubs
4233 # so we disable it for now
4234 qemu_devicedel
($vmid, $deviceid);
4235 qemu_devicedelverify
($vmid, $deviceid);
4237 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4239 qemu_devicedel
($vmid, $deviceid);
4240 qemu_devicedelverify
($vmid, $deviceid);
4241 qemu_drivedel
($vmid, $deviceid);
4242 qemu_iothread_del
($conf, $vmid, $deviceid);
4244 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4246 qemu_devicedel
($vmid, $deviceid);
4247 qemu_devicedelverify
($vmid, $deviceid);
4248 qemu_iothread_del
($conf, $vmid, $deviceid);
4250 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4252 qemu_devicedel
($vmid, $deviceid);
4253 qemu_drivedel
($vmid, $deviceid);
4254 qemu_deletescsihw
($conf, $vmid, $deviceid);
4256 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4258 qemu_devicedel
($vmid, $deviceid);
4259 qemu_devicedelverify
($vmid, $deviceid);
4260 qemu_netdevdel
($vmid, $deviceid);
4263 die "can't unplug device '$deviceid'\n";
4269 sub qemu_deviceadd
{
4270 my ($vmid, $devicefull) = @_;
4272 $devicefull = "driver=".$devicefull;
4273 my %options = split(/[=,]/, $devicefull);
4275 mon_cmd
($vmid, "device_add" , %options);
4278 sub qemu_devicedel
{
4279 my ($vmid, $deviceid) = @_;
4281 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4284 sub qemu_iothread_add
{
4285 my($vmid, $deviceid, $device) = @_;
4287 if ($device->{iothread
}) {
4288 my $iothreads = vm_iothreads_list
($vmid);
4289 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4293 sub qemu_iothread_del
{
4294 my($conf, $vmid, $deviceid) = @_;
4296 my $confid = $deviceid;
4297 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4298 $confid = 'scsi' . $1;
4300 my $device = parse_drive
($confid, $conf->{$confid});
4301 if ($device->{iothread
}) {
4302 my $iothreads = vm_iothreads_list
($vmid);
4303 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4307 sub qemu_objectadd
{
4308 my($vmid, $objectid, $qomtype) = @_;
4310 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4315 sub qemu_objectdel
{
4316 my($vmid, $objectid) = @_;
4318 mon_cmd
($vmid, "object-del", id
=> $objectid);
4324 my ($storecfg, $vmid, $device) = @_;
4326 my $drive = print_drive_full
($storecfg, $vmid, $device);
4327 $drive =~ s/\\/\\\\/g;
4328 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4330 # If the command succeeds qemu prints: "OK
"
4331 return 1 if $ret =~ m/OK/s;
4333 die "adding drive failed
: $ret\n";
4337 my($vmid, $deviceid) = @_;
4339 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4342 return 1 if $ret eq "";
4344 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4345 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4347 die "deleting drive
$deviceid failed
: $ret\n";
4350 sub qemu_deviceaddverify {
4351 my ($vmid, $deviceid) = @_;
4353 for (my $i = 0; $i <= 5; $i++) {
4354 my $devices_list = vm_devices_list($vmid);
4355 return 1 if defined($devices_list->{$deviceid});
4359 die "error on hotplug device
'$deviceid'\n";
4363 sub qemu_devicedelverify {
4364 my ($vmid, $deviceid) = @_;
4366 # need to verify that the device is correctly removed as device_del
4367 # is async and empty return is not reliable
4369 for (my $i = 0; $i <= 5; $i++) {
4370 my $devices_list = vm_devices_list($vmid);
4371 return 1 if !defined($devices_list->{$deviceid});
4375 die "error on hot-unplugging device
'$deviceid'\n";
4378 sub qemu_findorcreatescsihw {
4379 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4381 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4383 my $scsihwid="$controller_prefix$controller";
4384 my $devices_list = vm_devices_list($vmid);
4386 if(!defined($devices_list->{$scsihwid})) {
4387 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4393 sub qemu_deletescsihw {
4394 my ($conf, $vmid, $opt) = @_;
4396 my $device = parse_drive($opt, $conf->{$opt});
4398 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4399 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4403 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4405 my $devices_list = vm_devices_list($vmid);
4406 foreach my $opt (keys %{$devices_list}) {
4407 if (PVE::QemuServer::is_valid_drivename($opt)) {
4408 my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4409 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4415 my $scsihwid="scsihw
$controller";
4417 vm_deviceunplug($vmid, $conf, $scsihwid);
4422 sub qemu_add_pci_bridge {
4423 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4429 print_pci_addr($device, $bridges, $arch, $machine_type);
4431 while (my ($k, $v) = each %$bridges) {
4434 return 1 if !defined($bridgeid) || $bridgeid < 1;
4436 my $bridge = "pci
.$bridgeid";
4437 my $devices_list = vm_devices_list($vmid);
4439 if (!defined($devices_list->{$bridge})) {
4440 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4446 sub qemu_set_link_status {
4447 my ($vmid, $device, $up) = @_;
4449 mon_cmd($vmid, "set_link
", name => $device,
4450 up => $up ? JSON::true : JSON::false);
4453 sub qemu_netdevadd {
4454 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4456 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4457 my %options = split(/[=,]/, $netdev);
4459 mon_cmd($vmid, "netdev_add
", %options);
4463 sub qemu_netdevdel {
4464 my ($vmid, $deviceid) = @_;
4466 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4469 sub qemu_usb_hotplug {
4470 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4474 # remove the old one first
4475 vm_deviceunplug($vmid, $conf, $deviceid);
4477 # check if xhci controller is necessary and available
4478 if ($device->{usb3}) {
4480 my $devicelist = vm_devices_list($vmid);
4482 if (!$devicelist->{xhci}) {
4483 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4484 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4487 my $d = parse_usb_device($device->{host});
4488 $d->{usb3} = $device->{usb3};
4491 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4494 sub qemu_cpu_hotplug {
4495 my ($vmid, $conf, $vcpus) = @_;
4497 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4500 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4501 $sockets = $conf->{sockets} if $conf->{sockets};
4502 my $cores = $conf->{cores} || 1;
4503 my $maxcpus = $sockets * $cores;
4505 $vcpus = $maxcpus if !$vcpus;
4507 die "you can
't add more vcpus than maxcpus\n"
4508 if $vcpus > $maxcpus;
4510 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4512 if ($vcpus < $currentvcpus) {
4514 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4516 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4517 qemu_devicedel($vmid, "cpu$i");
4519 my $currentrunningvcpus = undef;
4521 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4522 last if scalar(@{$currentrunningvcpus}) == $i-1;
4523 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4527 #update conf after each succesfull cpu unplug
4528 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4529 PVE::QemuConfig->write_config($vmid, $conf);
4532 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4538 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4539 die "vcpus in running vm does not match its configuration\n"
4540 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4542 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4544 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4545 my $cpustr = print_cpu_device($conf, $i);
4546 qemu_deviceadd($vmid, $cpustr);
4549 my $currentrunningvcpus = undef;
4551 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4552 last if scalar(@{$currentrunningvcpus}) == $i;
4553 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4557 #update conf after each succesfull cpu hotplug
4558 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4559 PVE::QemuConfig->write_config($vmid, $conf);
4563 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4564 mon_cmd($vmid, "cpu-add", id => int($i));
4569 sub qemu_block_set_io_throttle {
4570 my ($vmid, $deviceid,
4571 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4572 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4573 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4574 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4576 return if !check_running($vmid) ;
4578 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4580 bps_rd => int($bps_rd),
4581 bps_wr => int($bps_wr),
4583 iops_rd => int($iops_rd),
4584 iops_wr => int($iops_wr),
4585 bps_max => int($bps_max),
4586 bps_rd_max => int($bps_rd_max),
4587 bps_wr_max => int($bps_wr_max),
4588 iops_max => int($iops_max),
4589 iops_rd_max => int($iops_rd_max),
4590 iops_wr_max => int($iops_wr_max),
4591 bps_max_length => int($bps_max_length),
4592 bps_rd_max_length => int($bps_rd_max_length),
4593 bps_wr_max_length => int($bps_wr_max_length),
4594 iops_max_length => int($iops_max_length),
4595 iops_rd_max_length => int($iops_rd_max_length),
4596 iops_wr_max_length => int($iops_wr_max_length),
4601 # old code, only used to shutdown old VM after update
4603 my ($fh, $timeout) = @_;
4605 my $sel = new IO::Select;
4612 while (scalar (@ready = $sel->can_read($timeout))) {
4614 if ($count = $fh->sysread($buf, 8192)) {
4615 if ($buf =~ /^(.*)\(qemu\) $/s) {
4622 if (!defined($count)) {
4629 die "monitor read timeout\n" if !scalar(@ready);
4634 sub qemu_block_resize {
4635 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4637 my $running = check_running($vmid);
4639 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4641 return if !$running;
4643 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4647 sub qemu_volume_snapshot {
4648 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4650 my $running = check_running($vmid);
4652 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4653 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4655 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4659 sub qemu_volume_snapshot_delete {
4660 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4662 my $running = check_running($vmid);
4667 my $conf = PVE::QemuConfig->load_config($vmid);
4668 foreach_drive($conf, sub {
4669 my ($ds, $drive) = @_;
4670 $running = 1 if $drive->{file} eq $volid;
4674 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4675 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4677 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4681 sub set_migration_caps {
4687 "auto-converge" => 1,
4689 "x-rdma-pin-all" => 0,
4694 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4696 for my $supported_capability (@$supported_capabilities) {
4698 capability => $supported_capability->{capability},
4699 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4703 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4706 my $fast_plug_option = {
4714 'vmstatestorage
' => 1,
4718 # hotplug changes in [PENDING]
4719 # $selection hash can be used to only apply specified options, for
4720 # example: { cores => 1 } (only apply changed 'cores
')
4721 # $errors ref is used to return error messages
4722 sub vmconfig_hotplug_pending {
4723 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4725 my $defaults = load_defaults();
4726 my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4728 # commit values which do not have any impact on running VM first
4729 # Note: those option cannot raise errors, we we do not care about
4730 # $selection and always apply them.
4732 my $add_error = sub {
4733 my ($opt, $msg) = @_;
4734 $errors->{$opt} = "hotplug problem - $msg";
4738 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4739 if ($fast_plug_option->{$opt}) {
4740 $conf->{$opt} = $conf->{pending}->{$opt};
4741 delete $conf->{pending}->{$opt};
4747 PVE::QemuConfig->write_config($vmid, $conf);
4748 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4751 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4753 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4754 foreach my $opt (sort keys %$pending_delete_hash) {
4755 next if $selection && !$selection->{$opt};
4756 my $force = $pending_delete_hash->{$opt}->{force};
4758 if ($opt eq 'hotplug
') {
4759 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4760 } elsif ($opt eq 'tablet
') {
4761 die "skip\n" if !$hotplug_features->{usb};
4762 if ($defaults->{tablet}) {
4763 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4764 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4765 if $arch eq 'aarch64
';
4767 vm_deviceunplug($vmid, $conf, 'tablet
');
4768 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4770 } elsif ($opt =~ m/^usb\d+/) {
4772 # since we cannot reliably hot unplug usb devices
4773 # we are disabling it
4774 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4775 vm_deviceunplug($vmid, $conf, $opt);
4776 } elsif ($opt eq 'vcpus
') {
4777 die "skip\n" if !$hotplug_features->{cpu};
4778 qemu_cpu_hotplug($vmid, $conf, undef);
4779 } elsif ($opt eq 'balloon
') {
4780 # enable balloon device is not hotpluggable
4781 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4782 # here we reset the ballooning value to memory
4783 my $balloon = $conf->{memory} || $defaults->{memory};
4784 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4785 } elsif ($fast_plug_option->{$opt}) {
4787 } elsif ($opt =~ m/^net(\d+)$/) {
4788 die "skip\n" if !$hotplug_features->{network};
4789 vm_deviceunplug($vmid, $conf, $opt);
4790 } elsif (is_valid_drivename($opt)) {
4791 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4792 vm_deviceunplug($vmid, $conf, $opt);
4793 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4794 } elsif ($opt =~ m/^memory$/) {
4795 die "skip\n" if !$hotplug_features->{memory};
4796 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4797 } elsif ($opt eq 'cpuunits
') {
4798 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4799 } elsif ($opt eq 'cpulimit
') {
4800 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4806 &$add_error($opt, $err) if $err ne "skip\n";
4808 # save new config if hotplug was successful
4809 delete $conf->{$opt};
4810 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4811 PVE::QemuConfig->write_config($vmid, $conf);
4812 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4816 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4817 $apply_pending_cloudinit = sub {
4818 return if $apply_pending_cloudinit_done; # once is enough
4819 $apply_pending_cloudinit_done = 1; # once is enough
4821 my ($key, $value) = @_;
4823 my @cloudinit_opts = keys %$confdesc_cloudinit;
4824 foreach my $opt (keys %{$conf->{pending}}) {
4825 next if !grep { $_ eq $opt } @cloudinit_opts;
4826 $conf->{$opt} = delete $conf->{pending}->{$opt};
4829 my $new_conf = { %$conf };
4830 $new_conf->{$key} = $value;
4831 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4834 foreach my $opt (keys %{$conf->{pending}}) {
4835 next if $selection && !$selection->{$opt};
4836 my $value = $conf->{pending}->{$opt};
4838 if ($opt eq 'hotplug
') {
4839 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4840 } elsif ($opt eq 'tablet
') {
4841 die "skip\n" if !$hotplug_features->{usb};
4843 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4844 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4845 if $arch eq 'aarch64
';
4846 } elsif ($value == 0) {
4847 vm_deviceunplug($vmid, $conf, 'tablet
');
4848 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4850 } elsif ($opt =~ m/^usb\d+$/) {
4852 # since we cannot reliably hot unplug usb devices
4853 # we are disabling it
4854 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4855 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4856 die "skip\n" if !$d;
4857 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4858 } elsif ($opt eq 'vcpus
') {
4859 die "skip\n" if !$hotplug_features->{cpu};
4860 qemu_cpu_hotplug($vmid, $conf, $value);
4861 } elsif ($opt eq 'balloon
') {
4862 # enable/disable balloning device is not hotpluggable
4863 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4864 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4865 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4867 # allow manual ballooning if shares is set to zero
4868 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4869 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4870 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4872 } elsif ($opt =~ m/^net(\d+)$/) {
4873 # some changes can be done without hotplug
4874 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4875 $vmid, $opt, $value, $arch, $machine_type);
4876 } elsif (is_valid_drivename($opt)) {
4877 # some changes can be done without hotplug
4878 my $drive = parse_drive($opt, $value);
4879 if (drive_is_cloudinit($drive)) {
4880 &$apply_pending_cloudinit($opt, $value);
4882 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4883 $vmid, $opt, $value, 1, $arch, $machine_type);
4884 } elsif ($opt =~ m/^memory$/) { #dimms
4885 die "skip\n" if !$hotplug_features->{memory};
4886 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4887 } elsif ($opt eq 'cpuunits
') {
4888 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4889 } elsif ($opt eq 'cpulimit
') {
4890 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4891 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4893 die "skip\n"; # skip non-hot-pluggable options
4897 &$add_error($opt, $err) if $err ne "skip\n";
4899 # save new config if hotplug was successful
4900 $conf->{$opt} = $value;
4901 delete $conf->{pending}->{$opt};
4902 PVE::QemuConfig->write_config($vmid, $conf);
4903 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4908 sub try_deallocate_drive {
4909 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4911 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4912 my $volid = $drive->{file};
4913 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4914 my $sid = PVE::Storage::parse_volume_id($volid);
4915 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4917 # check if the disk is really unused
4918 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4919 if is_volume_in_use($storecfg, $conf, $key, $volid);
4920 PVE::Storage::vdisk_free($storecfg, $volid);
4923 # If vm is not owner of this disk remove from config
4931 sub vmconfig_delete_or_detach_drive {
4932 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4934 my $drive = parse_drive($opt, $conf->{$opt});
4936 my $rpcenv = PVE::RPCEnvironment::get();
4937 my $authuser = $rpcenv->get_user();
4940 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4941 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4943 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4949 sub vmconfig_apply_pending {
4950 my ($vmid, $conf, $storecfg) = @_;
4954 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4955 foreach my $opt (sort keys %$pending_delete_hash) {
4956 die "internal error" if $opt =~ m/^unused/;
4957 my $force = $pending_delete_hash->{$opt}->{force};
4958 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4959 if (!defined($conf->{$opt})) {
4960 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4961 PVE::QemuConfig->write_config($vmid, $conf);
4962 } elsif (is_valid_drivename($opt)) {
4963 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4964 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4965 delete $conf->{$opt};
4966 PVE::QemuConfig->write_config($vmid, $conf);
4968 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4969 delete $conf->{$opt};
4970 PVE::QemuConfig->write_config($vmid, $conf);
4974 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4976 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4977 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4979 if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4980 # skip if nothing changed
4981 } elsif (is_valid_drivename($opt)) {
4982 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4983 if defined($conf->{$opt});
4984 $conf->{$opt} = $conf->{pending}->{$opt};
4986 $conf->{$opt} = $conf->{pending}->{$opt};
4989 delete $conf->{pending}->{$opt};
4990 PVE::QemuConfig->write_config($vmid, $conf);
4994 my $safe_num_ne = sub {
4997 return 0 if !defined($a) && !defined($b);
4998 return 1 if !defined($a);
4999 return 1 if !defined($b);
5004 my $safe_string_ne = sub {
5007 return 0 if !defined($a) && !defined($b);
5008 return 1 if !defined($a);
5009 return 1 if !defined($b);
5014 sub vmconfig_update_net {
5015 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5017 my $newnet = parse_net($value);
5019 if ($conf->{$opt}) {
5020 my $oldnet = parse_net($conf->{$opt});
5022 if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5023 &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5024 &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5025 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5027 # for non online change, we try to hot-unplug
5028 die "skip\n" if !$hotplug;
5029 vm_deviceunplug($vmid, $conf, $opt);
5032 die "internal error" if $opt !~ m/net(\d+)/;
5033 my $iface = "tap${vmid}i$1";
5035 if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5036 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5037 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5038 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5039 PVE::Network::tap_unplug($iface);
5040 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5041 } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5042 # Rate can be applied on its own but any change above needs to
5043 # include the rate in tap_plug since OVS resets everything.
5044 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5047 if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5048 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5056 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5062 sub vmconfig_update_disk {
5063 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5065 # fixme: do we need force?
5067 my $drive = parse_drive($opt, $value);
5069 if ($conf->{$opt}) {
5071 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
5073 my $media = $drive->{media} || 'disk
';
5074 my $oldmedia = $old_drive->{media} || 'disk
';
5075 die "unable to change media type\n" if $media ne $oldmedia;
5077 if (!drive_is_cdrom($old_drive)) {
5079 if ($drive->{file} ne $old_drive->{file}) {
5081 die "skip\n" if !$hotplug;
5083 # unplug and register as unused
5084 vm_deviceunplug($vmid, $conf, $opt);
5085 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5088 # update existing disk
5090 # skip non hotpluggable value
5091 if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5092 &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5093 &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5094 &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5099 if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5100 &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5101 &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5102 &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5103 &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5104 &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5105 &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5106 &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5107 &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5108 &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5109 &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5110 &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5111 &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5112 &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5113 &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5114 &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5115 &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5116 &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5118 qemu_block_set_io_throttle($vmid,"drive-$opt",
5119 ($drive->{mbps} || 0)*1024*1024,
5120 ($drive->{mbps_rd} || 0)*1024*1024,
5121 ($drive->{mbps_wr} || 0)*1024*1024,
5122 $drive->{iops} || 0,
5123 $drive->{iops_rd} || 0,
5124 $drive->{iops_wr} || 0,
5125 ($drive->{mbps_max} || 0)*1024*1024,
5126 ($drive->{mbps_rd_max} || 0)*1024*1024,
5127 ($drive->{mbps_wr_max} || 0)*1024*1024,
5128 $drive->{iops_max} || 0,
5129 $drive->{iops_rd_max} || 0,
5130 $drive->{iops_wr_max} || 0,
5131 $drive->{bps_max_length} || 1,
5132 $drive->{bps_rd_max_length} || 1,
5133 $drive->{bps_wr_max_length} || 1,
5134 $drive->{iops_max_length} || 1,
5135 $drive->{iops_rd_max_length} || 1,
5136 $drive->{iops_wr_max_length} || 1);
5145 if ($drive->{file} eq 'none
') {
5146 mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5147 if (drive_is_cloudinit($old_drive)) {
5148 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5151 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5152 mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5153 mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5161 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5163 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5164 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5168 my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5169 $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5171 PVE::QemuConfig->lock_config($vmid, sub {
5172 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5174 die "you can't start a vm
if it
's a template\n" if PVE::QemuConfig->is_template($conf);
5176 my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended
');
5178 PVE::QemuConfig->check_lock($conf)
5179 if !($skiplock || $is_suspended);
5181 die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5183 # clean up leftover reboot request files
5184 eval { clear_reboot_request($vmid); };
5187 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5188 vmconfig_apply_pending($vmid, $conf, $storecfg);
5189 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5192 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5194 my $defaults = load_defaults();
5196 # set environment variable useful inside network script
5197 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5199 my $local_volumes = {};
5201 if ($targetstorage) {
5202 foreach_drive($conf, sub {
5203 my ($ds, $drive) = @_;
5205 return if drive_is_cdrom($drive);
5207 my $volid = $drive->{file};
5211 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5213 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5214 return if $scfg->{shared};
5215 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5220 foreach my $opt (sort keys %$local_volumes) {
5222 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5223 my $drive = parse_drive($opt, $conf->{$opt});
5225 #if remote storage is specified, use default format
5226 if ($targetstorage && $targetstorage ne "1") {
5227 $storeid = $targetstorage;
5228 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5229 $format = $defFormat;
5231 #else we use same format than original
5232 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5233 $format = qemu_img_format($scfg, $volid);
5236 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5237 my $newdrive = $drive;
5238 $newdrive->{format} = $format;
5239 $newdrive->{file} = $newvolid;
5240 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5241 $local_volumes->{$opt} = $drivestr;
5242 #pass drive to conf for command line
5243 $conf->{$opt} = $drivestr;
5247 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
5249 if ($is_suspended) {
5250 # enforce machine type on suspended vm to ensure HW compatibility
5251 $forcemachine = $conf->{runningmachine};
5252 print "Resuming suspended VM\n";
5255 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5258 my $get_migration_ip = sub {
5259 my ($cidr, $nodename) = @_;
5261 return $migration_ip if defined($migration_ip);
5263 if (!defined($cidr)) {
5264 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5265 $cidr = $dc_conf->{migration}->{network};
5268 if (defined($cidr)) {
5269 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5271 die "could not get IP: no address configured on local " .
5272 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5274 die "could not get IP: multiple addresses configured on local " .
5275 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5277 $migration_ip = @$ips[0];
5280 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5281 if !defined($migration_ip);
5283 return $migration_ip;
5288 if ($statefile eq 'tcp
') {
5289 my $localip = "localhost";
5290 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5291 my $nodename = PVE::INotify::nodename();
5293 if (!defined($migration_type)) {
5294 if (defined($datacenterconf->{migration}->{type})) {
5295 $migration_type = $datacenterconf->{migration}->{type};
5297 $migration_type = 'secure
';
5301 if ($migration_type eq 'insecure
') {
5302 $localip = $get_migration_ip->($migration_network, $nodename);
5303 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5306 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5307 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5308 $migrate_uri = "tcp:${localip}:${migrate_port}";
5309 push @$cmd, '-incoming
', $migrate_uri;
5312 } elsif ($statefile eq 'unix
') {
5313 # should be default for secure migrations as a ssh TCP forward
5314 # tunnel is not deterministic reliable ready and fails regurarly
5315 # to set up in time, so use UNIX socket forwards
5316 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5317 unlink $socket_addr;
5319 $migrate_uri = "unix:$socket_addr";
5321 push @$cmd, '-incoming
', $migrate_uri;
5324 } elsif (-e $statefile) {
5325 push @$cmd, '-loadstate
', $statefile;
5327 my $statepath = PVE::Storage::path($storecfg, $statefile);
5328 push @$vollist, $statefile;
5329 push @$cmd, '-loadstate
', $statepath;
5336 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
5337 my $d = parse_hostpci($conf->{"hostpci$i"});
5339 my $pcidevices = $d->{pciid};
5340 foreach my $pcidevice (@$pcidevices) {
5341 my $pciid = $pcidevice->{id};
5343 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5344 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5345 die "no pci device info for device '$pciid'\n" if !$info;
5348 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5349 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5351 die "can't unbind
/bind pci group to vfio
'$pciid'\n"
5352 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5353 die "can
't reset pci device '$pciid'\n"
5354 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5359 PVE::Storage::activate_volumes($storecfg, $vollist);
5362 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5363 outfunc => sub {}, errfunc => sub {});
5365 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5366 # timeout should be more than enough here...
5367 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5369 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5370 : $defaults->{cpuunits};
5372 my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5373 my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5376 Slice => 'qemu
.slice
',
5378 CPUShares => $cpuunits
5381 if (my $cpulimit = $conf->{cpulimit}) {
5382 $properties{CPUQuota} = int($cpulimit * 100);
5384 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5386 my $run_qemu = sub {
5387 PVE::Tools::run_fork sub {
5388 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5389 run_command($cmd, %run_params);
5393 if ($conf->{hugepages}) {
5396 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5397 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5399 PVE::QemuServer::Memory::hugepages_mount();
5400 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5402 eval { $run_qemu->() };
5404 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5408 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5410 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5413 eval { $run_qemu->() };
5417 # deactivate volumes if start fails
5418 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5419 die "start failed: $err";
5422 print "migration listens on $migrate_uri\n" if $migrate_uri;
5424 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5425 eval { mon_cmd($vmid, "cont"); };
5429 #start nbd server for storage migration
5430 if ($targetstorage) {
5431 my $nodename = PVE::INotify::nodename();
5432 my $localip = $get_migration_ip->($migration_network, $nodename);
5433 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5434 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5436 mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet
', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5438 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5440 foreach my $opt (sort keys %$local_volumes) {
5441 my $volid = $local_volumes->{$opt};
5442 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5443 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5444 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5448 if ($migratedfrom) {
5450 set_migration_caps($vmid);
5455 print "spice listens on port $spice_port\n";
5456 if ($spice_ticket) {
5457 mon_cmd($vmid, "set_password", protocol => 'spice
', password => $spice_ticket);
5458 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5463 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5464 if !$statefile && $conf->{balloon};
5466 foreach my $opt (keys %$conf) {
5467 next if $opt !~ m/^net\d+$/;
5468 my $nicconf = parse_net($conf->{$opt});
5469 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5473 mon_cmd($vmid, 'qom-set
',
5474 path => "machine/peripheral/balloon0",
5475 property => "guest-stats-polling-interval",
5476 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5478 if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5479 print "Resumed VM, removing state\n";
5480 delete $conf->@{qw(lock vmstate runningmachine)};
5481 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5482 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5483 PVE
::QemuConfig-
>write_config($vmid, $conf);
5486 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5490 sub vm_commandline
{
5491 my ($storecfg, $vmid, $snapname) = @_;
5493 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5497 my $snapshot = $conf->{snapshots
}->{$snapname};
5498 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5500 # check for a 'runningmachine' in snapshot
5501 $forcemachine = $snapshot->{runningmachine
} if $snapshot->{runningmachine
};
5503 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5508 my $defaults = load_defaults
();
5510 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine);
5512 return PVE
::Tools
::cmd2string
($cmd);
5516 my ($vmid, $skiplock) = @_;
5518 PVE
::QemuConfig-
>lock_config($vmid, sub {
5520 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5522 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5524 mon_cmd
($vmid, "system_reset");
5528 sub get_vm_volumes
{
5532 foreach_volid
($conf, sub {
5533 my ($volid, $attr) = @_;
5535 return if $volid =~ m
|^/|;
5537 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5540 push @$vollist, $volid;
5546 sub vm_stop_cleanup
{
5547 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5552 my $vollist = get_vm_volumes
($conf);
5553 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5556 foreach my $ext (qw(mon qmp pid vnc qga)) {
5557 unlink "/var/run/qemu-server/${vmid}.$ext";
5560 if ($conf->{ivshmem
}) {
5561 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5562 # just delete it for now, VMs which have this already open do not
5563 # are affected, but new VMs will get a separated one. If this
5564 # becomes an issue we either add some sort of ref-counting or just
5565 # add a "don't delete on stop" flag to the ivshmem format.
5566 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5569 foreach my $key (keys %$conf) {
5570 next if $key !~ m/^hostpci(\d+)$/;
5571 my $hostpciindex = $1;
5572 my $d = parse_hostpci
($conf->{$key});
5573 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5575 foreach my $pci (@{$d->{pciid
}}) {
5576 my $pciid = $pci->{id
};
5577 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5581 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5583 warn $@ if $@; # avoid errors - just warn
5586 # call only in locked context
5588 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5590 my $pid = check_running
($vmid, $nocheck);
5595 $conf = PVE
::QemuConfig-
>load_config($vmid);
5596 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5597 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5598 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5599 $timeout = $opts->{down
} if $opts->{down
};
5601 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5606 if (defined($conf) && parse_guest_agent
($conf)->{enabled
}) {
5607 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5609 mon_cmd
($vmid, "system_powerdown");
5612 mon_cmd
($vmid, "quit");
5618 $timeout = 60 if !defined($timeout);
5621 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5626 if ($count >= $timeout) {
5628 warn "VM still running - terminating now with SIGTERM\n";
5631 die "VM quit/powerdown failed - got timeout\n";
5634 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5639 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5642 die "VM quit/powerdown failed\n";
5650 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5655 if ($count >= $timeout) {
5656 warn "VM still running - terminating now with SIGKILL\n";
5661 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5664 # Note: use $nocheck to skip tests if VM configuration file exists.
5665 # We need that when migration VMs to other nodes (files already moved)
5666 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5668 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5670 $force = 1 if !defined($force) && !$shutdown;
5673 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5674 kill 15, $pid if $pid;
5675 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5676 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5680 PVE
::QemuConfig-
>lock_config($vmid, sub {
5681 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5686 my ($vmid, $timeout) = @_;
5688 PVE
::QemuConfig-
>lock_config($vmid, sub {
5691 # only reboot if running, as qmeventd starts it again on a stop event
5692 return if !check_running
($vmid);
5694 create_reboot_request
($vmid);
5696 my $storecfg = PVE
::Storage
::config
();
5697 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5701 # avoid that the next normal shutdown will be confused for a reboot
5702 clear_reboot_request
($vmid);
5709 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5716 PVE
::QemuConfig-
>lock_config($vmid, sub {
5718 $conf = PVE
::QemuConfig-
>load_config($vmid);
5720 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5721 PVE
::QemuConfig-
>check_lock($conf)
5722 if !($skiplock || $is_backing_up);
5724 die "cannot suspend to disk during backup\n"
5725 if $is_backing_up && $includestate;
5727 if ($includestate) {
5728 $conf->{lock} = 'suspending';
5729 my $date = strftime
("%Y-%m-%d", localtime(time()));
5730 $storecfg = PVE
::Storage
::config
();
5731 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5732 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5733 PVE
::QemuConfig-
>write_config($vmid, $conf);
5735 mon_cmd
($vmid, "stop");
5739 if ($includestate) {
5741 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5744 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5746 my $state = mon_cmd
($vmid, "query-savevm");
5747 if (!$state->{status
}) {
5748 die "savevm not active\n";
5749 } elsif ($state->{status
} eq 'active') {
5752 } elsif ($state->{status
} eq 'completed') {
5753 print "State saved, quitting\n";
5755 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5756 die "query-savevm failed with error '$state->{error}'\n"
5758 die "query-savevm returned status '$state->{status}'\n";
5764 PVE
::QemuConfig-
>lock_config($vmid, sub {
5765 $conf = PVE
::QemuConfig-
>load_config($vmid);
5767 # cleanup, but leave suspending lock, to indicate something went wrong
5769 mon_cmd
($vmid, "savevm-end");
5770 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5771 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5772 delete $conf->@{qw(vmstate runningmachine)};
5773 PVE
::QemuConfig-
>write_config($vmid, $conf);
5779 die "lock changed unexpectedly\n"
5780 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5782 mon_cmd
($vmid, "quit");
5783 $conf->{lock} = 'suspended';
5784 PVE
::QemuConfig-
>write_config($vmid, $conf);
5790 my ($vmid, $skiplock, $nocheck) = @_;
5792 PVE
::QemuConfig-
>lock_config($vmid, sub {
5793 my $res = mon_cmd
($vmid, 'query-status');
5794 my $resume_cmd = 'cont';
5796 if ($res->{status
} && $res->{status
} eq 'suspended') {
5797 $resume_cmd = 'system_wakeup';
5802 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5804 PVE
::QemuConfig-
>check_lock($conf)
5805 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5808 mon_cmd
($vmid, $resume_cmd);
5813 my ($vmid, $skiplock, $key) = @_;
5815 PVE
::QemuConfig-
>lock_config($vmid, sub {
5817 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5819 # there is no qmp command, so we use the human monitor command
5820 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5821 die $res if $res ne '';
5825 # vzdump restore implementaion
5827 sub tar_archive_read_firstfile
{
5828 my $archive = shift;
5830 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5832 # try to detect archive type first
5833 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5834 die "unable to open file '$archive'\n";
5835 my $firstfile = <$fh>;
5839 die "ERROR: archive contaions no data\n" if !$firstfile;
5845 sub tar_restore_cleanup
{
5846 my ($storecfg, $statfile) = @_;
5848 print STDERR
"starting cleanup\n";
5850 if (my $fd = IO
::File-
>new($statfile, "r")) {
5851 while (defined(my $line = <$fd>)) {
5852 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5855 if ($volid =~ m
|^/|) {
5856 unlink $volid || die 'unlink failed\n';
5858 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5860 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5862 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5864 print STDERR
"unable to parse line in statfile - $line";
5871 sub restore_archive
{
5872 my ($archive, $vmid, $user, $opts) = @_;
5874 my $format = $opts->{format
};
5877 if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5878 $format = 'tar' if !$format;
5880 } elsif ($archive =~ m/\.tar$/) {
5881 $format = 'tar' if !$format;
5882 } elsif ($archive =~ m/.tar.lzo$/) {
5883 $format = 'tar' if !$format;
5885 } elsif ($archive =~ m/\.vma$/) {
5886 $format = 'vma' if !$format;
5887 } elsif ($archive =~ m/\.vma\.gz$/) {
5888 $format = 'vma' if !$format;
5890 } elsif ($archive =~ m/\.vma\.lzo$/) {
5891 $format = 'vma' if !$format;
5894 $format = 'vma' if !$format; # default
5897 # try to detect archive format
5898 if ($format eq 'tar') {
5899 return restore_tar_archive
($archive, $vmid, $user, $opts);
5901 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5905 sub restore_update_config_line
{
5906 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5908 return if $line =~ m/^\#qmdump\#/;
5909 return if $line =~ m/^\#vzdump\#/;
5910 return if $line =~ m/^lock:/;
5911 return if $line =~ m/^unused\d+:/;
5912 return if $line =~ m/^parent:/;
5914 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5915 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5916 # try to convert old 1.X settings
5917 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5918 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5919 my ($model, $macaddr) = split(/\=/, $devconfig);
5920 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5923 bridge
=> "vmbr$ind",
5924 macaddr
=> $macaddr,
5926 my $netstr = print_net
($net);
5928 print $outfd "net$cookie->{netcount}: $netstr\n";
5929 $cookie->{netcount
}++;
5931 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5932 my ($id, $netstr) = ($1, $2);
5933 my $net = parse_net
($netstr);
5934 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5935 $netstr = print_net
($net);
5936 print $outfd "$id: $netstr\n";
5937 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5940 my $di = parse_drive
($virtdev, $value);
5941 if (defined($di->{backup
}) && !$di->{backup
}) {
5942 print $outfd "#$line";
5943 } elsif ($map->{$virtdev}) {
5944 delete $di->{format
}; # format can change on restore
5945 $di->{file
} = $map->{$virtdev};
5946 $value = print_drive
($vmid, $di);
5947 print $outfd "$virtdev: $value\n";
5951 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5953 if ($vmgenid ne '0') {
5954 # always generate a new vmgenid if there was a valid one setup
5955 $vmgenid = generate_uuid
();
5957 print $outfd "vmgenid: $vmgenid\n";
5958 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5959 my ($uuid, $uuid_str);
5960 UUID
::generate
($uuid);
5961 UUID
::unparse
($uuid, $uuid_str);
5962 my $smbios1 = parse_smbios1
($2);
5963 $smbios1->{uuid
} = $uuid_str;
5964 print $outfd $1.print_smbios1
($smbios1)."\n";
5971 my ($cfg, $vmid) = @_;
5973 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5975 my $volid_hash = {};
5976 foreach my $storeid (keys %$info) {
5977 foreach my $item (@{$info->{$storeid}}) {
5978 next if !($item->{volid
} && $item->{size
});
5979 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5980 $volid_hash->{$item->{volid
}} = $item;
5987 sub is_volume_in_use
{
5988 my ($storecfg, $conf, $skip_drive, $volid) = @_;
5990 my $path = PVE
::Storage
::path
($storecfg, $volid);
5992 my $scan_config = sub {
5993 my ($cref, $snapname) = @_;
5995 foreach my $key (keys %$cref) {
5996 my $value = $cref->{$key};
5997 if (is_valid_drivename
($key)) {
5998 next if $skip_drive && $key eq $skip_drive;
5999 my $drive = parse_drive
($key, $value);
6000 next if !$drive || !$drive->{file
} || drive_is_cdrom
($drive);
6001 return 1 if $volid eq $drive->{file
};
6002 if ($drive->{file
} =~ m!^/!) {
6003 return 1 if $drive->{file
} eq $path;
6005 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
}, 1);
6007 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid, 1);
6009 return 1 if $path eq PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
6017 return 1 if &$scan_config($conf);
6021 foreach my $snapname (keys %{$conf->{snapshots
}}) {
6022 return 1 if &$scan_config($conf->{snapshots
}->{$snapname}, $snapname);
6028 sub update_disksize
{
6029 my ($vmid, $conf, $volid_hash) = @_;
6032 my $prefix = "VM $vmid:";
6034 # used and unused disks
6035 my $referenced = {};
6037 # Note: it is allowed to define multiple storages with same path (alias), so
6038 # we need to check both 'volid' and real 'path' (two different volid can point
6039 # to the same path).
6041 my $referencedpath = {};
6044 foreach my $opt (keys %$conf) {
6045 if (is_valid_drivename
($opt)) {
6046 my $drive = parse_drive
($opt, $conf->{$opt});
6047 my $volid = $drive->{file
};
6050 $referenced->{$volid} = 1;
6051 if ($volid_hash->{$volid} &&
6052 (my $path = $volid_hash->{$volid}->{path
})) {
6053 $referencedpath->{$path} = 1;
6056 next if drive_is_cdrom
($drive);
6057 next if !$volid_hash->{$volid};
6059 $drive->{size
} = $volid_hash->{$volid}->{size
};
6060 my $new = print_drive
($vmid, $drive);
6061 if ($new ne $conf->{$opt}) {
6063 $conf->{$opt} = $new;
6064 print "$prefix update disk '$opt' information.\n";
6069 # remove 'unusedX' entry if volume is used
6070 foreach my $opt (keys %$conf) {
6071 next if $opt !~ m/^unused\d+$/;
6072 my $volid = $conf->{$opt};
6073 my $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6074 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6075 print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6077 delete $conf->{$opt};
6080 $referenced->{$volid} = 1;
6081 $referencedpath->{$path} = 1 if $path;
6084 foreach my $volid (sort keys %$volid_hash) {
6085 next if $volid =~ m/vm-$vmid-state-/;
6086 next if $referenced->{$volid};
6087 my $path = $volid_hash->{$volid}->{path
};
6088 next if !$path; # just to be sure
6089 next if $referencedpath->{$path};
6091 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6092 print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6093 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6100 my ($vmid, $nolock, $dryrun) = @_;
6102 my $cfg = PVE
::Storage
::config
();
6104 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6105 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6106 foreach my $stor (keys %{$cfg->{ids
}}) {
6107 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
6110 print "rescan volumes...\n";
6111 my $volid_hash = scan_volids
($cfg, $vmid);
6113 my $updatefn = sub {
6116 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6118 PVE
::QemuConfig-
>check_lock($conf);
6121 foreach my $volid (keys %$volid_hash) {
6122 my $info = $volid_hash->{$volid};
6123 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6126 my $changes = update_disksize
($vmid, $conf, $vm_volids);
6128 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6131 if (defined($vmid)) {
6135 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6138 my $vmlist = config_list
();
6139 foreach my $vmid (keys %$vmlist) {
6143 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6149 sub restore_vma_archive
{
6150 my ($archive, $vmid, $user, $opts, $comp) = @_;
6152 my $readfrom = $archive;
6154 my $cfg = PVE
::Storage
::config
();
6156 my $bwlimit = $opts->{bwlimit
};
6158 my $dbg_cmdstring = '';
6159 my $add_pipe = sub {
6161 push @$commands, $cmd;
6162 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6163 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6168 if ($archive eq '-') {
6171 # If we use a backup from a PVE defined storage we also consider that
6172 # storage's rate limit:
6173 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6174 if (defined($volid)) {
6175 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6176 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6178 print STDERR
"applying read rate limit: $readlimit\n";
6179 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6180 $add_pipe->($cstream);
6187 if ($comp eq 'gzip') {
6188 $cmd = ['zcat', $readfrom];
6189 } elsif ($comp eq 'lzop') {
6190 $cmd = ['lzop', '-d', '-c', $readfrom];
6192 die "unknown compression method '$comp'\n";
6197 my $tmpdir = "/var/tmp/vzdumptmp$$";
6200 # disable interrupts (always do cleanups)
6204 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6206 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6207 POSIX
::mkfifo
($mapfifo, 0600);
6210 my $openfifo = sub {
6211 open($fifofh, '>', $mapfifo) || die $!;
6214 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6221 my $rpcenv = PVE
::RPCEnvironment
::get
();
6223 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6224 my $tmpfn = "$conffile.$$.tmp";
6226 # Note: $oldconf is undef if VM does not exists
6227 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6228 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6232 my $print_devmap = sub {
6233 my $virtdev_hash = {};
6235 my $cfgfn = "$tmpdir/qemu-server.conf";
6237 # we can read the config - that is already extracted
6238 my $fh = IO
::File-
>new($cfgfn, "r") ||
6239 "unable to read qemu-server.conf - $!\n";
6241 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6243 my $pve_firewall_dir = '/etc/pve/firewall';
6244 mkdir $pve_firewall_dir; # make sure the dir exists
6245 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6248 while (defined(my $line = <$fh>)) {
6249 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6250 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6251 die "archive does not contain data for drive '$virtdev'\n"
6252 if !$devinfo->{$devname};
6253 if (defined($opts->{storage
})) {
6254 $storeid = $opts->{storage
} || 'local';
6255 } elsif (!$storeid) {
6258 $format = 'raw' if !$format;
6259 $devinfo->{$devname}->{devname
} = $devname;
6260 $devinfo->{$devname}->{virtdev
} = $virtdev;
6261 $devinfo->{$devname}->{format
} = $format;
6262 $devinfo->{$devname}->{storeid
} = $storeid;
6264 # check permission on storage
6265 my $pool = $opts->{pool
}; # todo: do we need that?
6266 if ($user ne 'root@pam') {
6267 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6270 $storage_limits{$storeid} = $bwlimit;
6272 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6273 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6275 my $drive = parse_drive
($virtdev, $2);
6276 if (drive_is_cloudinit
($drive)) {
6277 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6278 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
6279 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6283 storeid
=> $opts->{storage
} // $storeid,
6284 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6285 file
=> $drive->{file
}, # to make drive_is_cloudinit check possible
6286 name
=> "vm-$vmid-cloudinit",
6289 $virtdev_hash->{$virtdev} = $d;
6294 foreach my $key (keys %storage_limits) {
6295 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6297 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6298 $storage_limits{$key} = $limit * 1024;
6301 foreach my $devname (keys %$devinfo) {
6302 die "found no device mapping information for device '$devname'\n"
6303 if !$devinfo->{$devname}->{virtdev
};
6306 # create empty/temp config
6308 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6309 foreach_drive
($oldconf, sub {
6310 my ($ds, $drive) = @_;
6312 return if drive_is_cdrom
($drive, 1);
6314 my $volid = $drive->{file
};
6315 return if !$volid || $volid =~ m
|^/|;
6317 my ($path, $owner) = PVE
::Storage
::path
($cfg, $volid);
6318 return if !$path || !$owner || ($owner != $vmid);
6320 # Note: only delete disk we want to restore
6321 # other volumes will become unused
6322 if ($virtdev_hash->{$ds}) {
6323 eval { PVE
::Storage
::vdisk_free
($cfg, $volid); };
6330 # delete vmstate files, after the restore we have no snapshots anymore
6331 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
6332 my $snap = $oldconf->{snapshots
}->{$snapname};
6333 if ($snap->{vmstate
}) {
6334 eval { PVE
::Storage
::vdisk_free
($cfg, $snap->{vmstate
}); };
6343 foreach my $virtdev (sort keys %$virtdev_hash) {
6344 my $d = $virtdev_hash->{$virtdev};
6345 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6346 my $storeid = $d->{storeid
};
6347 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
6350 if (my $limit = $storage_limits{$storeid}) {
6351 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6354 # test if requested format is supported
6355 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($cfg, $storeid);
6356 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6357 $d->{format
} = $defFormat if !$supported;
6360 if ($d->{is_cloudinit
}) {
6362 $name .= ".$d->{format}" if $d->{format
} ne 'raw';
6365 my $volid = PVE
::Storage
::vdisk_alloc
($cfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6366 print STDERR
"new volume ID is '$volid'\n";
6367 $d->{volid
} = $volid;
6369 PVE
::Storage
::activate_volumes
($cfg, [$volid]);
6371 my $write_zeros = 1;
6372 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6376 if (!$d->{is_cloudinit
}) {
6377 my $path = PVE
::Storage
::path
($cfg, $volid);
6379 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6381 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6383 $map->{$virtdev} = $volid;
6386 $fh->seek(0, 0) || die "seek failed - $!\n";
6388 my $outfd = new IO
::File
($tmpfn, "w") ||
6389 die "unable to write config for VM $vmid\n";
6391 my $cookie = { netcount
=> 0 };
6392 while (defined(my $line = <$fh>)) {
6393 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6406 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6407 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6409 $oldtimeout = alarm($timeout);
6416 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6417 my ($dev_id, $size, $devname) = ($1, $2, $3);
6418 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6419 } elsif ($line =~ m/^CTIME: /) {
6420 # we correctly received the vma config, so we can disable
6421 # the timeout now for disk allocation (set to 10 minutes, so
6422 # that we always timeout if something goes wrong)
6425 print $fifofh "done\n";
6426 my $tmp = $oldtimeout || 0;
6427 $oldtimeout = undef;
6433 print "restore vma archive: $dbg_cmdstring\n";
6434 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6438 alarm($oldtimeout) if $oldtimeout;
6441 foreach my $devname (keys %$devinfo) {
6442 my $volid = $devinfo->{$devname}->{volid
};
6443 push @$vollist, $volid if $volid;
6446 PVE
::Storage
::deactivate_volumes
($cfg, $vollist);
6454 foreach my $devname (keys %$devinfo) {
6455 my $volid = $devinfo->{$devname}->{volid
};
6458 if ($volid =~ m
|^/|) {
6459 unlink $volid || die 'unlink failed\n';
6461 PVE
::Storage
::vdisk_free
($cfg, $volid);
6463 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6465 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6472 rename($tmpfn, $conffile) ||
6473 die "unable to commit configuration file '$conffile'\n";
6475 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6477 eval { rescan
($vmid, 1); };
6481 sub restore_tar_archive
{
6482 my ($archive, $vmid, $user, $opts) = @_;
6484 if ($archive ne '-') {
6485 my $firstfile = tar_archive_read_firstfile
($archive);
6486 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6487 if $firstfile ne 'qemu-server.conf';
6490 my $storecfg = PVE
::Storage
::config
();
6492 # avoid zombie disks when restoring over an existing VM -> cleanup first
6493 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6494 # skiplock=1 because qmrestore has set the 'create' lock itself already
6495 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6496 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6498 my $tocmd = "/usr/lib/qemu-server/qmextract";
6500 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6501 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6502 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6503 $tocmd .= ' --info' if $opts->{info
};
6505 # tar option "xf" does not autodetect compression when read from STDIN,
6506 # so we pipe to zcat
6507 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6508 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6510 my $tmpdir = "/var/tmp/vzdumptmp$$";
6513 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6514 local $ENV{VZDUMP_VMID
} = $vmid;
6515 local $ENV{VZDUMP_USER
} = $user;
6517 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6518 my $tmpfn = "$conffile.$$.tmp";
6520 # disable interrupts (always do cleanups)
6524 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6532 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6534 if ($archive eq '-') {
6535 print "extracting archive from STDIN\n";
6536 run_command
($cmd, input
=> "<&STDIN");
6538 print "extracting archive '$archive'\n";
6542 return if $opts->{info
};
6546 my $statfile = "$tmpdir/qmrestore.stat";
6547 if (my $fd = IO
::File-
>new($statfile, "r")) {
6548 while (defined (my $line = <$fd>)) {
6549 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6550 $map->{$1} = $2 if $1;
6552 print STDERR
"unable to parse line in statfile - $line\n";
6558 my $confsrc = "$tmpdir/qemu-server.conf";
6560 my $srcfd = new IO
::File
($confsrc, "r") ||
6561 die "unable to open file '$confsrc'\n";
6563 my $outfd = new IO
::File
($tmpfn, "w") ||
6564 die "unable to write config for VM $vmid\n";
6566 my $cookie = { netcount
=> 0 };
6567 while (defined (my $line = <$srcfd>)) {
6568 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6576 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6582 rename $tmpfn, $conffile ||
6583 die "unable to commit configuration file '$conffile'\n";
6585 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6587 eval { rescan
($vmid, 1); };
6591 sub foreach_storage_used_by_vm
{
6592 my ($conf, $func) = @_;
6596 foreach_drive
($conf, sub {
6597 my ($ds, $drive) = @_;
6598 return if drive_is_cdrom
($drive);
6600 my $volid = $drive->{file
};
6602 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6603 $sidhash->{$sid} = $sid if $sid;
6606 foreach my $sid (sort keys %$sidhash) {
6611 my $qemu_snap_storage = {
6614 sub do_snapshots_with_qemu
{
6615 my ($storecfg, $volid) = @_;
6617 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6618 my $scfg = $storecfg->{ids
}->{$storage_name};
6620 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6624 if ($volid =~ m/\.(qcow2|qed)$/){
6631 sub qga_check_running
{
6632 my ($vmid, $nowarn) = @_;
6634 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6636 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6642 sub template_create
{
6643 my ($vmid, $conf, $disk) = @_;
6645 my $storecfg = PVE
::Storage
::config
();
6647 foreach_drive
($conf, sub {
6648 my ($ds, $drive) = @_;
6650 return if drive_is_cdrom
($drive);
6651 return if $disk && $ds ne $disk;
6653 my $volid = $drive->{file
};
6654 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6656 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6657 $drive->{file
} = $voliddst;
6658 $conf->{$ds} = print_drive
($vmid, $drive);
6659 PVE
::QemuConfig-
>write_config($vmid, $conf);
6663 sub convert_iscsi_path
{
6666 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6671 my $initiator_name = get_initiator_name
();
6673 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6674 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6677 die "cannot convert iscsi path '$path', unkown format\n";
6680 sub qemu_img_convert
{
6681 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6683 my $storecfg = PVE
::Storage
::config
();
6684 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6685 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6687 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6691 my $src_is_iscsi = 0;
6692 my $src_format = 'raw';
6695 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6696 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6697 $src_format = qemu_img_format
($src_scfg, $src_volname);
6698 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6699 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6700 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6701 } elsif (-f
$src_volid) {
6702 $src_path = $src_volid;
6703 if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6708 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6710 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6711 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6712 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6713 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6716 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6717 push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6718 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6719 push @$cmd, '-T', $cachemode if defined($cachemode);
6721 if ($src_is_iscsi) {
6722 push @$cmd, '--image-opts';
6723 $src_path = convert_iscsi_path
($src_path);
6725 push @$cmd, '-f', $src_format;
6728 if ($dst_is_iscsi) {
6729 push @$cmd, '--target-image-opts';
6730 $dst_path = convert_iscsi_path
($dst_path);
6732 push @$cmd, '-O', $dst_format;
6735 push @$cmd, $src_path;
6737 if (!$dst_is_iscsi && $is_zero_initialized) {
6738 push @$cmd, "zeroinit:$dst_path";
6740 push @$cmd, $dst_path;
6745 if($line =~ m/\((\S+)\/100\
%\)/){
6747 my $transferred = int($size * $percent / 100);
6748 my $remaining = $size - $transferred;
6750 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6755 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6757 die "copy failed: $err" if $err;
6760 sub qemu_img_format
{
6761 my ($scfg, $volname) = @_;
6763 if ($scfg->{path
} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6770 sub qemu_drive_mirror
{
6771 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6773 $jobs = {} if !$jobs;
6777 $jobs->{"drive-$drive"} = {};
6779 if ($dst_volid =~ /^nbd:/) {
6780 $qemu_target = $dst_volid;
6783 my $storecfg = PVE
::Storage
::config
();
6784 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6786 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6788 $format = qemu_img_format
($dst_scfg, $dst_volname);
6790 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6792 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6795 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6796 $opts->{format
} = $format if $format;
6798 if (defined($bwlimit)) {
6799 $opts->{speed
} = $bwlimit * 1024;
6800 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6802 print "drive mirror is starting for drive-$drive\n";
6805 # if a job already runs for this device we get an error, catch it for cleanup
6806 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
6808 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6810 die "mirroring error: $err\n";
6813 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6816 sub qemu_drive_mirror_monitor
{
6817 my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6820 my $err_complete = 0;
6823 die "storage migration timed out\n" if $err_complete > 300;
6825 my $stats = mon_cmd
($vmid, "query-block-jobs");
6827 my $running_mirror_jobs = {};
6828 foreach my $stat (@$stats) {
6829 next if $stat->{type
} ne 'mirror';
6830 $running_mirror_jobs->{$stat->{device
}} = $stat;
6833 my $readycounter = 0;
6835 foreach my $job (keys %$jobs) {
6837 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6838 print "$job : finished\n";
6839 delete $jobs->{$job};
6843 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6845 my $busy = $running_mirror_jobs->{$job}->{busy
};
6846 my $ready = $running_mirror_jobs->{$job}->{ready
};
6847 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6848 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6849 my $remaining = $total - $transferred;
6850 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6852 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6855 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6858 last if scalar(keys %$jobs) == 0;
6860 if ($readycounter == scalar(keys %$jobs)) {
6861 print "all mirroring jobs are ready \n";
6862 last if $skipcomplete; #do the complete later
6864 if ($vmiddst && $vmiddst != $vmid) {
6865 my $agent_running = $qga && qga_check_running
($vmid);
6866 if ($agent_running) {
6867 print "freeze filesystem\n";
6868 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6870 print "suspend vm\n";
6871 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6874 # if we clone a disk for a new target vm, we don't switch the disk
6875 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6877 if ($agent_running) {
6878 print "unfreeze filesystem\n";
6879 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6881 print "resume vm\n";
6882 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6888 foreach my $job (keys %$jobs) {
6889 # try to switch the disk if source and destination are on the same guest
6890 print "$job: Completing block job...\n";
6892 eval { mon_cmd
($vmid, "block-job-complete", device
=> $job) };
6893 if ($@ =~ m/cannot be completed/) {
6894 print "$job: Block job cannot be completed, try again.\n";
6897 print "$job: Completed successfully.\n";
6898 $jobs->{$job}->{complete
} = 1;
6909 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6910 die "mirroring error: $err";
6915 sub qemu_blockjobs_cancel
{
6916 my ($vmid, $jobs) = @_;
6918 foreach my $job (keys %$jobs) {
6919 print "$job: Cancelling block job\n";
6920 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6921 $jobs->{$job}->{cancel
} = 1;
6925 my $stats = mon_cmd
($vmid, "query-block-jobs");
6927 my $running_jobs = {};
6928 foreach my $stat (@$stats) {
6929 $running_jobs->{$stat->{device
}} = $stat;
6932 foreach my $job (keys %$jobs) {
6934 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6935 print "$job: Done.\n";
6936 delete $jobs->{$job};
6940 last if scalar(keys %$jobs) == 0;
6947 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6948 $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6953 print "create linked clone of drive $drivename ($drive->{file})\n";
6954 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6955 push @$newvollist, $newvolid;
6958 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6959 $storeid = $storage if $storage;
6961 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6962 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
6964 print "create full clone of drive $drivename ($drive->{file})\n";
6966 if (drive_is_cloudinit
($drive)) {
6967 $name = "vm-$newvmid-cloudinit";
6968 $name .= ".$dst_format" if $dst_format ne 'raw';
6970 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
6972 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6973 push @$newvollist, $newvolid;
6975 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
6977 if (drive_is_cloudinit
($drive)) {
6981 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6982 if (!$running || $snapname) {
6983 # TODO: handle bwlimits
6984 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
6987 my $kvmver = get_running_qemu_version
($vmid);
6988 if (!min_version
($kvmver, 2, 7)) {
6989 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6990 if $drive->{iothread
};
6993 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
6998 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
7001 $disk->{format
} = undef;
7002 $disk->{file
} = $newvolid;
7003 $disk->{size
} = $size;
7008 sub get_running_qemu_version
{
7010 my $res = mon_cmd
($vmid, "query-version");
7011 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7014 sub qemu_use_old_bios_files
{
7015 my ($machine_type) = @_;
7017 return if !$machine_type;
7019 my $use_old_bios_files = undef;
7021 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7023 $use_old_bios_files = 1;
7025 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type) // kvm_user_version
();
7026 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7027 # load new efi bios files on migration. So this hack is required to allow
7028 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7029 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7030 $use_old_bios_files = !min_version
($version, 2, 4);
7033 return ($use_old_bios_files, $machine_type);
7036 sub create_efidisk
($$$$$) {
7037 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7039 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7040 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7042 my $vars_size_b = -s
$ovmf_vars;
7043 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7044 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7045 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7047 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7049 return ($volid, $vars_size);
7052 sub vm_iothreads_list
{
7055 my $res = mon_cmd
($vmid, 'query-iothreads');
7058 foreach my $iothread (@$res) {
7059 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7066 my ($conf, $drive) = @_;
7070 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7072 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7078 my $controller = int($drive->{index} / $maxdev);
7079 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single') ?
"virtioscsi" : "scsihw";
7081 return ($maxdev, $controller, $controller_prefix);
7084 sub add_hyperv_enlightenments
{
7085 my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7087 return if $winversion < 6;
7088 return if $bios && $bios eq 'ovmf' && $winversion < 8;
7090 if ($gpu_passthrough || defined($hv_vendor_id)) {
7091 $hv_vendor_id //= 'proxmox';
7092 push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7095 if (min_version
($machine_version, 2, 3)) {
7096 push @$cpuFlags , 'hv_spinlocks=0x1fff';
7097 push @$cpuFlags , 'hv_vapic';
7098 push @$cpuFlags , 'hv_time';
7100 push @$cpuFlags , 'hv_spinlocks=0xffff';
7103 if (min_version
($machine_version, 2, 6)) {
7104 push @$cpuFlags , 'hv_reset';
7105 push @$cpuFlags , 'hv_vpindex';
7106 push @$cpuFlags , 'hv_runtime';
7109 if ($winversion >= 7) {
7110 push @$cpuFlags , 'hv_relaxed';
7112 if (min_version
($machine_version, 2, 12)) {
7113 push @$cpuFlags , 'hv_synic';
7114 push @$cpuFlags , 'hv_stimer';
7117 if (min_version
($machine_version, 3, 1)) {
7118 push @$cpuFlags , 'hv_ipi';
7123 sub windows_version
{
7126 return 0 if !$ostype;
7130 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7132 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7134 } elsif ($ostype =~ m/^win(\d+)$/) {
7141 sub resolve_dst_disk_format
{
7142 my ($storecfg, $storeid, $src_volname, $format) = @_;
7143 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7146 # if no target format is specified, use the source disk format as hint
7148 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7149 $format = qemu_img_format
($scfg, $src_volname);
7155 # test if requested format is supported - else use default
7156 my $supported = grep { $_ eq $format } @$validFormats;
7157 $format = $defFormat if !$supported;
7161 sub resolve_first_disk
{
7163 my @disks = PVE
::QemuServer
::valid_drive_names
();
7165 foreach my $ds (reverse @disks) {
7166 next if !$conf->{$ds};
7167 my $disk = PVE
::QemuServer
::parse_drive
($ds, $conf->{$ds});
7168 next if PVE
::QemuServer
::drive_is_cdrom
($disk);
7175 my ($uuid, $uuid_str);
7176 UUID
::generate
($uuid);
7177 UUID
::unparse
($uuid, $uuid_str);
7181 sub generate_smbios1_uuid
{
7182 return "uuid=".generate_uuid
();
7188 mon_cmd
($vmid, 'nbd-server-stop');
7191 sub create_reboot_request
{
7193 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7194 or die "failed to create reboot trigger file: $!\n";
7198 sub clear_reboot_request
{
7200 my $path = "/run/qemu-server/$vmid.reboot";
7203 $res = unlink($path);
7204 die "could not remove reboot request for $vmid: $!"
7205 if !$res && $! != POSIX
::ENOENT
;
7210 # bash completion helper
7212 sub complete_backup_archives
{
7213 my ($cmdname, $pname, $cvalue) = @_;
7215 my $cfg = PVE
::Storage
::config
();
7219 if ($cvalue =~ m/^([^:]+):/) {
7223 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7226 foreach my $id (keys %$data) {
7227 foreach my $item (@{$data->{$id}}) {
7228 next if $item->{format
} !~ m/^vma\.(gz|lzo)$/;
7229 push @$res, $item->{volid
} if defined($item->{volid
});
7236 my $complete_vmid_full = sub {
7239 my $idlist = vmstatus
();
7243 foreach my $id (keys %$idlist) {
7244 my $d = $idlist->{$id};
7245 if (defined($running)) {
7246 next if $d->{template
};
7247 next if $running && $d->{status
} ne 'running';
7248 next if !$running && $d->{status
} eq 'running';
7257 return &$complete_vmid_full();
7260 sub complete_vmid_stopped
{
7261 return &$complete_vmid_full(0);
7264 sub complete_vmid_running
{
7265 return &$complete_vmid_full(1);
7268 sub complete_storage
{
7270 my $cfg = PVE
::Storage
::config
();
7271 my $ids = $cfg->{ids
};
7274 foreach my $sid (keys %$ids) {
7275 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7276 next if !$ids->{$sid}->{content
}->{images
};
7283 sub complete_migration_storage
{
7284 my ($cmd, $param, $current_value, $all_args) = @_;
7286 my $targetnode = @$all_args[1];
7288 my $cfg = PVE
::Storage
::config
();
7289 my $ids = $cfg->{ids
};
7292 foreach my $sid (keys %$ids) {
7293 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7294 next if !$ids->{$sid}->{content
}->{images
};