1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
30 use PVE
::DataCenterConfig
;
31 use PVE
::Exception
qw(raise raise_param_exc);
32 use PVE
::GuestHelpers
;
34 use PVE
::JSONSchema
qw(get_standard_option);
36 use PVE
::RPCEnvironment
;
40 use PVE
::Tools
qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
44 use PVE
::QemuServer
::Helpers
qw(min_version);
45 use PVE
::QemuServer
::Cloudinit
;
46 use PVE
::QemuServer
::Machine
;
47 use PVE
::QemuServer
::Memory
;
48 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
49 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port);
50 use PVE
::QemuServer
::USB
qw(parse_usb_device);
52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
55 "$EDK2_FW_BASE/OVMF_CODE.fd",
56 "$EDK2_FW_BASE/OVMF_VARS.fd"
59 "$EDK2_FW_BASE/AAVMF_CODE.fd",
60 "$EDK2_FW_BASE/AAVMF_VARS.fd"
64 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
68 # Note about locking: we use flock on the config file protect
69 # against concurent actions.
70 # Aditionaly, we have a 'lock' setting in the config file. This
71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
72 # allowed when such lock is set. But you can ignore this kind of
73 # lock with the --skiplock flag.
75 cfs_register_file
('/qemu-server/',
79 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
80 description
=> "Some command save/restore state from this location.",
86 PVE
::JSONSchema
::register_standard_option
('pve-qm-image-format', {
88 enum
=> [qw(raw cow qcow qed qcow2 vmdk cloop)],
89 description
=> "The drive's backing file's data format.",
93 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
94 description
=> "Specifies the Qemu machine type.",
96 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
101 #no warnings 'redefine';
104 my ($controller, $vmid, $option, $value) = @_;
106 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
107 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
111 my $nodename = PVE
::INotify
::nodename
();
113 my $cpu_vendor_list = {
115 486 => 'GenuineIntel',
116 pentium
=> 'GenuineIntel',
117 pentium2
=> 'GenuineIntel',
118 pentium3
=> 'GenuineIntel',
119 coreduo
=> 'GenuineIntel',
120 core2duo
=> 'GenuineIntel',
121 Conroe
=> 'GenuineIntel',
122 Penryn
=> 'GenuineIntel',
123 Nehalem
=> 'GenuineIntel',
124 'Nehalem-IBRS' => 'GenuineIntel',
125 Westmere
=> 'GenuineIntel',
126 'Westmere-IBRS' => 'GenuineIntel',
127 SandyBridge
=> 'GenuineIntel',
128 'SandyBridge-IBRS' => 'GenuineIntel',
129 IvyBridge
=> 'GenuineIntel',
130 'IvyBridge-IBRS' => 'GenuineIntel',
131 Haswell
=> 'GenuineIntel',
132 'Haswell-IBRS' => 'GenuineIntel',
133 'Haswell-noTSX' => 'GenuineIntel',
134 'Haswell-noTSX-IBRS' => 'GenuineIntel',
135 Broadwell
=> 'GenuineIntel',
136 'Broadwell-IBRS' => 'GenuineIntel',
137 'Broadwell-noTSX' => 'GenuineIntel',
138 'Broadwell-noTSX-IBRS' => 'GenuineIntel',
139 'Skylake-Client' => 'GenuineIntel',
140 'Skylake-Client-IBRS' => 'GenuineIntel',
141 'Skylake-Server' => 'GenuineIntel',
142 'Skylake-Server-IBRS' => 'GenuineIntel',
143 'Cascadelake-Server' => 'GenuineIntel',
144 KnightsMill
=> 'GenuineIntel',
148 athlon
=> 'AuthenticAMD',
149 phenom
=> 'AuthenticAMD',
150 Opteron_G1
=> 'AuthenticAMD',
151 Opteron_G2
=> 'AuthenticAMD',
152 Opteron_G3
=> 'AuthenticAMD',
153 Opteron_G4
=> 'AuthenticAMD',
154 Opteron_G5
=> 'AuthenticAMD',
155 EPYC
=> 'AuthenticAMD',
156 'EPYC-IBPB' => 'AuthenticAMD',
158 # generic types, use vendor from host node
167 my @supported_cpu_flags = (
181 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
185 description
=> "Emulated CPU type.",
187 enum
=> [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
192 description
=> "Do not identify as a KVM virtual machine.",
199 pattern
=> qr/[a-zA-Z0-9]{1,12}/,
200 format_description
=> 'vendor-id',
201 description
=> 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
205 description
=> "List of additional CPU flags separated by ';'."
206 . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
207 . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
208 format_description
=> '+FLAG[;-FLAG...]',
210 pattern
=> qr/$cpu_flag(;$cpu_flag)*/,
219 enum
=> [qw(i6300esb ib700)],
220 description
=> "Watchdog type to emulate.",
221 default => 'i6300esb',
226 enum
=> [qw(reset shutdown poweroff pause debug none)],
227 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
231 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
235 description
=> "Enable/disable Qemu GuestAgent.",
240 fstrim_cloned_disks
=> {
241 description
=> "Run fstrim after cloning/moving a disk.",
247 description
=> "Select the agent type",
251 enum
=> [qw(virtio isa)],
257 description
=> "Select the VGA type.",
262 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
265 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
277 description
=> "The size of the file in MB.",
281 pattern
=> '[a-zA-Z0-9\-]+',
283 format_description
=> 'string',
284 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
291 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
292 description
=> "Configure an audio device."
299 description
=> "Driver backend for the audio device."
303 my $spice_enhancements_fmt = {
308 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
312 enum
=> ['off', 'all', 'filter'],
315 description
=> "Enable video streaming. Uses compression for detected video streams."
323 description
=> "Specifies whether a VM will be started during system bootup.",
329 description
=> "Automatic restart after crash (currently ignored).",
334 type
=> 'string', format
=> 'pve-hotplug-features',
335 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
336 default => 'network,disk,usb',
341 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
347 description
=> "Lock/unlock the VM.",
348 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
353 description
=> "Limit of CPU usage.",
354 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
362 description
=> "CPU weight for a VM.",
363 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
371 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
378 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
384 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
392 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
393 "It should not be necessary to set it.",
394 enum
=> PVE
::Tools
::kvmkeymaplist
(),
399 type
=> 'string', format
=> 'dns-name',
400 description
=> "Set a name for the VM. Only used on the configuration web interface.",
405 description
=> "SCSI controller model",
406 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
412 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
417 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
418 description
=> "Specify guest operating system.",
419 verbose_description
=> <<EODESC,
420 Specify guest operating system. This is used to enable special
421 optimization/features for specific operating systems:
424 other;; unspecified OS
425 wxp;; Microsoft Windows XP
426 w2k;; Microsoft Windows 2000
427 w2k3;; Microsoft Windows 2003
428 w2k8;; Microsoft Windows 2008
429 wvista;; Microsoft Windows Vista
430 win7;; Microsoft Windows 7
431 win8;; Microsoft Windows 8/2012/2012r2
432 win10;; Microsoft Windows 10/2016
433 l24;; Linux 2.4 Kernel
434 l26;; Linux 2.6 - 5.X Kernel
435 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
441 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
442 pattern
=> '[acdn]{1,4}',
447 type
=> 'string', format
=> 'pve-qm-bootdisk',
448 description
=> "Enable booting from specified disk.",
449 pattern
=> '(ide|sata|scsi|virtio)\d+',
454 description
=> "The number of CPUs. Please use option -sockets instead.",
461 description
=> "The number of CPU sockets.",
468 description
=> "The number of cores per socket.",
475 description
=> "Enable/disable NUMA.",
481 description
=> "Enable/disable hugepages memory.",
482 enum
=> [qw(any 2 1024)],
487 description
=> "Number of hotplugged vcpus.",
494 description
=> "Enable/disable ACPI.",
499 description
=> "Enable/disable Qemu GuestAgent and its properties.",
501 format
=> $agent_fmt,
506 description
=> "Enable/disable KVM hardware virtualization.",
512 description
=> "Enable/disable time drift fix.",
518 description
=> "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
523 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
527 type
=> 'string', format
=> $vga_fmt,
528 description
=> "Configure the VGA hardware.",
529 verbose_description
=> "Configure the VGA Hardware. If you want to use ".
530 "high resolution modes (>= 1280x1024x16) you may need to increase " .
531 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
532 "is 'std' for all OS types besides some Windows versions (XP and " .
533 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
534 "display server. For win* OS you can select how many independent " .
535 "displays you want, Linux guests can add displays them self.\n".
536 "You can also run without any graphic card, using a serial device as terminal.",
540 type
=> 'string', format
=> 'pve-qm-watchdog',
541 description
=> "Create a virtual hardware watchdog device.",
542 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled" .
543 " (by a guest action), the watchdog must be periodically polled " .
544 "by an agent inside the guest or else the watchdog will reset " .
545 "the guest (or execute the respective action specified)",
550 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
551 description
=> "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
552 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
555 startup
=> get_standard_option
('pve-startup-order'),
559 description
=> "Enable/disable Template.",
565 description
=> "Arbitrary arguments passed to kvm.",
566 verbose_description
=> <<EODESCR,
567 Arbitrary arguments passed to kvm, for example:
569 args: -no-reboot -no-hpet
571 NOTE: this option is for experts only.
578 description
=> "Enable/disable the USB tablet device.",
579 verbose_description
=> "Enable/disable the USB tablet device. This device is " .
580 "usually needed to allow absolute mouse positioning with VNC. " .
581 "Else the mouse runs out of sync with normal VNC clients. " .
582 "If you're running lots of console-only guests on one host, " .
583 "you may consider disabling this to save some context switches. " .
584 "This is turned off by default if you use spice (-vga=qxl).",
589 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
593 migrate_downtime
=> {
596 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
602 type
=> 'string', format
=> 'pve-qm-ide',
603 typetext
=> '<volume>',
604 description
=> "This is an alias for option -ide2",
608 description
=> "Emulated CPU type.",
612 parent
=> get_standard_option
('pve-snapshot-name', {
614 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
618 description
=> "Timestamp for snapshots.",
624 type
=> 'string', format
=> 'pve-volume-id',
625 description
=> "Reference to a volume which stores the VM state. This is used internally for snapshots.",
627 vmstatestorage
=> get_standard_option
('pve-storage-id', {
628 description
=> "Default storage for VM state volumes/files.",
631 runningmachine
=> get_standard_option
('pve-qemu-machine', {
632 description
=> "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
634 machine
=> get_standard_option
('pve-qemu-machine'),
636 description
=> "Virtual processor architecture. Defaults to the host.",
639 enum
=> [qw(x86_64 aarch64)],
642 description
=> "Specify SMBIOS type 1 fields.",
643 type
=> 'string', format
=> 'pve-qm-smbios1',
650 description
=> "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
656 enum
=> [ qw(seabios ovmf) ],
657 description
=> "Select BIOS implementation.",
658 default => 'seabios',
662 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
663 format_description
=> 'UUID',
664 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
665 verbose_description
=> "The VM generation ID (vmgenid) device exposes a".
666 " 128-bit integer value identifier to the guest OS. This allows to".
667 " notify the guest operating system when the virtual machine is".
668 " executed with a different configuration (e.g. snapshot execution".
669 " or creation from a template). The guest operating system notices".
670 " the change, and is then able to react as appropriate by marking".
671 " its copies of distributed databases as dirty, re-initializing its".
672 " random number generator, etc.\n".
673 "Note that auto-creation only works when done throug API/CLI create".
674 " or update methods, but not when manually editing the config file.",
675 default => "1 (autogenerated)",
680 format
=> 'pve-volume-id',
682 description
=> "Script that will be executed during various steps in the vms lifetime.",
686 format
=> $ivshmem_fmt,
687 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
692 format
=> $audio_fmt,
693 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
696 spice_enhancements
=> {
698 format
=> $spice_enhancements_fmt,
699 description
=> "Configure additional enhancements for SPICE.",
703 type
=> 'string', format
=> 'pve-tag-list',
704 description
=> 'Tags of the VM. This is only meta information.',
713 description
=> 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
714 format
=> 'pve-volume-id',
715 format_description
=> 'volume',
720 description
=> 'Specify a custom file containing all network data passed to the VM via cloud-init.',
721 format
=> 'pve-volume-id',
722 format_description
=> 'volume',
727 description
=> 'Specify a custom file containing all user data passed to the VM via cloud-init.',
728 format
=> 'pve-volume-id',
729 format_description
=> 'volume',
732 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
734 my $confdesc_cloudinit = {
738 description
=> 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
739 enum
=> ['configdrive2', 'nocloud'],
744 description
=> "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
749 description
=> 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
754 description
=> 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
755 format
=> 'pve-qm-cicustom',
760 description
=> "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
764 type
=> 'string', format
=> 'address-list',
765 description
=> "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
770 format
=> 'urlencoded',
771 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
775 # what about other qemu settings ?
777 #machine => 'string',
790 ##soundhw => 'string',
792 while (my ($k, $v) = each %$confdesc) {
793 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
796 my $MAX_IDE_DISKS = 4;
797 my $MAX_SCSI_DISKS = 14;
798 my $MAX_VIRTIO_DISKS = 16;
799 my $MAX_SATA_DISKS = 6;
800 my $MAX_USB_DEVICES = 5;
802 my $MAX_UNUSED_DISKS = 256;
803 my $MAX_HOSTPCI_DEVICES = 16;
804 my $MAX_SERIAL_PORTS = 4;
805 my $MAX_PARALLEL_PORTS = 3;
811 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
812 description
=> "CPUs accessing this NUMA node.",
813 format_description
=> "id[-id];...",
817 description
=> "Amount of memory this NUMA node provides.",
822 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
823 description
=> "Host NUMA nodes to use.",
824 format_description
=> "id[-id];...",
829 enum
=> [qw(preferred bind interleave)],
830 description
=> "NUMA allocation policy.",
834 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
837 type
=> 'string', format
=> $numa_fmt,
838 description
=> "NUMA topology.",
840 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
842 for (my $i = 0; $i < $MAX_NUMA; $i++) {
843 $confdesc->{"numa$i"} = $numadesc;
846 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
847 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
848 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
849 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
851 my $net_fmt_bridge_descr = <<__EOD__;
852 Bridge to attach the network device to. The Proxmox VE standard bridge
855 If you do not specify a bridge, we create a kvm user (NATed) network
856 device, which provides DHCP and DNS services. The following addresses
863 The DHCP server assign addresses to the guest starting from 10.0.2.15.
867 macaddr
=> get_standard_option
('mac-addr', {
868 description
=> "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
872 description
=> "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
873 enum
=> $nic_model_list,
876 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
879 description
=> $net_fmt_bridge_descr,
880 format_description
=> 'bridge',
885 minimum
=> 0, maximum
=> 16,
886 description
=> 'Number of packet queues to be used on the device.',
892 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
897 minimum
=> 1, maximum
=> 4094,
898 description
=> 'VLAN tag to apply to packets on this interface.',
903 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
904 description
=> 'VLAN trunks to pass through this interface.',
905 format_description
=> 'vlanid[;vlanid...]',
910 description
=> 'Whether this interface should be protected by the firewall.',
915 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
922 type
=> 'string', format
=> $net_fmt,
923 description
=> "Specify network devices.",
926 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
931 format
=> 'pve-ipv4-config',
932 format_description
=> 'IPv4Format/CIDR',
933 description
=> 'IPv4 address in CIDR format.',
940 format_description
=> 'GatewayIPv4',
941 description
=> 'Default gateway for IPv4 traffic.',
947 format
=> 'pve-ipv6-config',
948 format_description
=> 'IPv6Format/CIDR',
949 description
=> 'IPv6 address in CIDR format.',
956 format_description
=> 'GatewayIPv6',
957 description
=> 'Default gateway for IPv6 traffic.',
962 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
965 type
=> 'string', format
=> 'pve-qm-ipconfig',
966 description
=> <<'EODESCR',
967 cloud-init: Specify IP addresses and gateways for the corresponding interface.
969 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
971 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
977 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
979 for (my $i = 0; $i < $MAX_NETS; $i++) {
980 $confdesc->{"net$i"} = $netdesc;
981 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
984 foreach my $key (keys %$confdesc_cloudinit) {
985 $confdesc->{$key} = $confdesc_cloudinit->{$key};
988 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
989 sub verify_volume_id_or_qm_path
{
990 my ($volid, $noerr) = @_;
992 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
996 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
997 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
999 return undef if $noerr;
1007 my %drivedesc_base = (
1008 volume
=> { alias
=> 'file' },
1011 format
=> 'pve-volume-id-or-qm-path',
1013 format_description
=> 'volume',
1014 description
=> "The drive's backing volume.",
1018 enum
=> [qw(cdrom disk)],
1019 description
=> "The drive's media type.",
1025 description
=> "Force the drive's physical geometry to have a specific cylinder count.",
1030 description
=> "Force the drive's physical geometry to have a specific head count.",
1035 description
=> "Force the drive's physical geometry to have a specific sector count.",
1040 enum
=> [qw(none lba auto)],
1041 description
=> "Force disk geometry bios translation mode.",
1046 description
=> "Controls qemu's snapshot mode feature."
1047 . " If activated, changes made to the disk are temporary and will"
1048 . " be discarded when the VM is shutdown.",
1053 enum
=> [qw(none writethrough writeback unsafe directsync)],
1054 description
=> "The drive's cache mode",
1057 format
=> get_standard_option
('pve-qm-image-format'),
1060 format
=> 'disk-size',
1061 format_description
=> 'DiskSize',
1062 description
=> "Disk size. This is purely informational and has no effect.",
1067 description
=> "Whether the drive should be included when making backups.",
1072 description
=> 'Whether the drive should considered for replication jobs.',
1078 enum
=> [qw(ignore report stop)],
1079 description
=> 'Read error action.',
1084 enum
=> [qw(enospc ignore report stop)],
1085 description
=> 'Write error action.',
1090 enum
=> [qw(native threads)],
1091 description
=> 'AIO type to use.',
1096 enum
=> [qw(ignore on)],
1097 description
=> 'Controls whether to pass discard/trim requests to the underlying storage.',
1102 description
=> 'Controls whether to detect and try to optimize writes of zeroes.',
1107 format
=> 'urlencoded',
1108 format_description
=> 'serial',
1109 maxLength
=> 20*3, # *3 since it's %xx url enoded
1110 description
=> "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1115 description
=> 'Mark this locally-managed volume as available on all nodes',
1116 verbose_description
=> "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1122 my %iothread_fmt = ( iothread
=> {
1124 description
=> "Whether to use iothreads for this drive",
1131 format
=> 'urlencoded',
1132 format_description
=> 'model',
1133 maxLength
=> 40*3, # *3 since it's %xx url enoded
1134 description
=> "The drive's reported model name, url-encoded, up to 40 bytes long.",
1142 description
=> "Number of queues.",
1148 my %scsiblock_fmt = (
1151 description
=> "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1160 description
=> "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1168 pattern
=> qr/^(0x)[0-9a-fA-F]{16}/,
1169 format_description
=> 'wwn',
1170 description
=> "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1175 my $add_throttle_desc = sub {
1176 my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1179 format_description
=> $unit,
1180 description
=> "Maximum $what in $longunit.",
1183 $d->{minimum
} = $minimum if defined($minimum);
1184 $drivedesc_base{$key} = $d;
1186 # throughput: (leaky bucket)
1187 $add_throttle_desc->('bps', 'integer', 'r/w speed', 'bps', 'bytes per second');
1188 $add_throttle_desc->('bps_rd', 'integer', 'read speed', 'bps', 'bytes per second');
1189 $add_throttle_desc->('bps_wr', 'integer', 'write speed', 'bps', 'bytes per second');
1190 $add_throttle_desc->('mbps', 'number', 'r/w speed', 'mbps', 'megabytes per second');
1191 $add_throttle_desc->('mbps_rd', 'number', 'read speed', 'mbps', 'megabytes per second');
1192 $add_throttle_desc->('mbps_wr', 'number', 'write speed', 'mbps', 'megabytes per second');
1193 $add_throttle_desc->('iops', 'integer', 'r/w I/O', 'iops', 'operations per second');
1194 $add_throttle_desc->('iops_rd', 'integer', 'read I/O', 'iops', 'operations per second');
1195 $add_throttle_desc->('iops_wr', 'integer', 'write I/O', 'iops', 'operations per second');
1197 # pools: (pool of IO before throttling starts taking effect)
1198 $add_throttle_desc->('mbps_max', 'number', 'unthrottled r/w pool', 'mbps', 'megabytes per second');
1199 $add_throttle_desc->('mbps_rd_max', 'number', 'unthrottled read pool', 'mbps', 'megabytes per second');
1200 $add_throttle_desc->('mbps_wr_max', 'number', 'unthrottled write pool', 'mbps', 'megabytes per second');
1201 $add_throttle_desc->('iops_max', 'integer', 'unthrottled r/w I/O pool', 'iops', 'operations per second');
1202 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool', 'iops', 'operations per second');
1203 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1206 $add_throttle_desc->('bps_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1207 $add_throttle_desc->('bps_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1208 $add_throttle_desc->('bps_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1210 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1211 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1214 $drivedesc_base{'bps_rd_length'} = { alias
=> 'bps_rd_max_length' };
1215 $drivedesc_base{'bps_wr_length'} = { alias
=> 'bps_wr_max_length' };
1216 $drivedesc_base{'iops_rd_length'} = { alias
=> 'iops_rd_max_length' };
1217 $drivedesc_base{'iops_wr_length'} = { alias
=> 'iops_wr_max_length' };
1225 PVE
::JSONSchema
::register_format
("pve-qm-ide", $ide_fmt);
1229 type
=> 'string', format
=> $ide_fmt,
1230 description
=> "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1232 PVE
::JSONSchema
::register_standard_option
("pve-qm-ide", $idedesc);
1244 type
=> 'string', format
=> $scsi_fmt,
1245 description
=> "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1247 PVE
::JSONSchema
::register_standard_option
("pve-qm-scsi", $scsidesc);
1256 type
=> 'string', format
=> $sata_fmt,
1257 description
=> "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1259 PVE
::JSONSchema
::register_standard_option
("pve-qm-sata", $satadesc);
1267 type
=> 'string', format
=> $virtio_fmt,
1268 description
=> "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1270 PVE
::JSONSchema
::register_standard_option
("pve-qm-virtio", $virtiodesc);
1272 my $alldrive_fmt = {
1283 volume
=> { alias
=> 'file' },
1286 format
=> 'pve-volume-id-or-qm-path',
1288 format_description
=> 'volume',
1289 description
=> "The drive's backing volume.",
1291 format
=> get_standard_option
('pve-qm-image-format'),
1294 format
=> 'disk-size',
1295 format_description
=> 'DiskSize',
1296 description
=> "Disk size. This is purely informational and has no effect.",
1301 my $efidisk_desc = {
1303 type
=> 'string', format
=> $efidisk_fmt,
1304 description
=> "Configure a Disk for storing EFI vars",
1307 PVE
::JSONSchema
::register_standard_option
("pve-qm-efidisk", $efidisk_desc);
1312 type
=> 'string', format
=> 'pve-qm-usb-device',
1313 format_description
=> 'HOSTUSBDEVICE|spice',
1314 description
=> <<EODESCR,
1315 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1317 'bus-port(.port)*' (decimal numbers) or
1318 'vendor_id:product_id' (hexadeciaml numbers) or
1321 You can use the 'lsusb -t' command to list existing usb devices.
1323 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1325 The value 'spice' can be used to add a usb redirection devices for spice.
1331 description
=> "Specifies whether if given host option is a USB3 device or port.",
1338 type
=> 'string', format
=> $usb_fmt,
1339 description
=> "Configure an USB device (n is 0 to 4).",
1341 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1343 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1348 pattern
=> qr/$PCIRE(;$PCIRE)*/,
1349 format_description
=> 'HOSTPCIID[;HOSTPCIID2...]',
1350 description
=> <<EODESCR,
1351 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1352 of PCI virtual functions of the host. HOSTPCIID syntax is:
1354 'bus:dev.func' (hexadecimal numbers)
1356 You can us the 'lspci' command to list existing PCI devices.
1361 description
=> "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1367 pattern
=> '[^,;]+',
1368 format_description
=> 'string',
1369 description
=> "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1374 description
=> "Choose the PCI-express bus (needs the 'q35' machine model).",
1380 description
=> "Enable vfio-vga device support.",
1386 format_description
=> 'string',
1387 pattern
=> '[^/\.:]+',
1389 description
=> <<EODESCR
1390 The type of mediated device to use.
1391 An instance of this type will be created on startup of the VM and
1392 will be cleaned up when the VM stops.
1396 PVE
::JSONSchema
::register_format
('pve-qm-hostpci', $hostpci_fmt);
1400 type
=> 'string', format
=> 'pve-qm-hostpci',
1401 description
=> "Map host PCI devices into guest.",
1402 verbose_description
=> <<EODESCR,
1403 Map host PCI devices into guest.
1405 NOTE: This option allows direct access to host hardware. So it is no longer
1406 possible to migrate such machines - use with special care.
1408 CAUTION: Experimental! User reported problems with this option.
1411 PVE
::JSONSchema
::register_standard_option
("pve-qm-hostpci", $hostpcidesc);
1416 pattern
=> '(/dev/.+|socket)',
1417 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1418 verbose_description
=> <<EODESCR,
1419 Create a serial device inside the VM (n is 0 to 3), and pass through a
1420 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1421 host side (use 'qm terminal' to open a terminal connection).
1423 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1425 CAUTION: Experimental! User reported problems with this option.
1432 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1433 description
=> "Map host parallel devices (n is 0 to 2).",
1434 verbose_description
=> <<EODESCR,
1435 Map host parallel devices (n is 0 to 2).
1437 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1439 CAUTION: Experimental! User reported problems with this option.
1443 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1444 $confdesc->{"parallel$i"} = $paralleldesc;
1447 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1448 $confdesc->{"serial$i"} = $serialdesc;
1451 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
1452 $confdesc->{"hostpci$i"} = $hostpcidesc;
1455 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) {
1456 $drivename_hash->{"ide$i"} = 1;
1457 $confdesc->{"ide$i"} = $idedesc;
1460 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) {
1461 $drivename_hash->{"sata$i"} = 1;
1462 $confdesc->{"sata$i"} = $satadesc;
1465 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) {
1466 $drivename_hash->{"scsi$i"} = 1;
1467 $confdesc->{"scsi$i"} = $scsidesc ;
1470 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) {
1471 $drivename_hash->{"virtio$i"} = 1;
1472 $confdesc->{"virtio$i"} = $virtiodesc;
1475 $drivename_hash->{efidisk0
} = 1;
1476 $confdesc->{efidisk0
} = $efidisk_desc;
1478 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1479 $confdesc->{"usb$i"} = $usbdesc;
1484 type
=> 'string', format
=> 'pve-volume-id',
1485 description
=> "Reference to unused volumes. This is used internally, and should not be modified manually.",
1488 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1489 $confdesc->{"unused$i"} = $unuseddesc;
1492 my $kvm_api_version = 0;
1495 return $kvm_api_version if $kvm_api_version;
1497 open my $fh, '<', '/dev/kvm'
1500 # 0xae00 => KVM_GET_API_VERSION
1501 $kvm_api_version = ioctl($fh, 0xae00, 0);
1503 return $kvm_api_version;
1506 my $kvm_user_version = {};
1509 sub kvm_user_version
{
1512 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1513 my $st = stat($binary);
1515 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1516 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1517 $cachedmtime == $st->mtime;
1519 $kvm_user_version->{$binary} = 'unknown';
1520 $kvm_mtime->{$binary} = $st->mtime;
1524 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1525 $kvm_user_version->{$binary} = $2;
1529 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1532 return $kvm_user_version->{$binary};
1536 sub kernel_has_vhost_net
{
1537 return -c
'/dev/vhost-net';
1540 sub valid_drive_names
{
1541 # order is important - used to autoselect boot disk
1542 return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1543 (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1544 (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1545 (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1549 sub is_valid_drivename
{
1552 return defined($drivename_hash->{$dev});
1557 return defined($confdesc->{$key});
1561 sub get_cdrom_path
{
1563 return $cdrom_path if $cdrom_path;
1565 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1566 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1567 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1571 my ($storecfg, $vmid, $cdrom) = @_;
1573 if ($cdrom eq 'cdrom') {
1574 return get_cdrom_path
();
1575 } elsif ($cdrom eq 'none') {
1577 } elsif ($cdrom =~ m
|^/|) {
1580 return PVE
::Storage
::path
($storecfg, $cdrom);
1584 # try to convert old style file names to volume IDs
1585 sub filename_to_volume_id
{
1586 my ($vmid, $file, $media) = @_;
1588 if (!($file eq 'none' || $file eq 'cdrom' ||
1589 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1591 return undef if $file =~ m
|/|;
1593 if ($media && $media eq 'cdrom') {
1594 $file = "local:iso/$file";
1596 $file = "local:$vmid/$file";
1603 sub verify_media_type
{
1604 my ($opt, $vtype, $media) = @_;
1609 if ($media eq 'disk') {
1611 } elsif ($media eq 'cdrom') {
1614 die "internal error";
1617 return if ($vtype eq $etype);
1619 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1622 sub cleanup_drive_path
{
1623 my ($opt, $storecfg, $drive) = @_;
1625 # try to convert filesystem paths to volume IDs
1627 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1628 ($drive->{file
} !~ m
|^/dev/.+|) &&
1629 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1630 ($drive->{file
} !~ m/^\d+$/)) {
1631 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1632 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1633 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1634 verify_media_type
($opt, $vtype, $drive->{media
});
1635 $drive->{file
} = $volid;
1638 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1641 sub parse_hotplug_features
{
1646 return $res if $data eq '0';
1648 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1650 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1651 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1654 die "invalid hotplug feature '$feature'\n";
1660 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1661 sub pve_verify_hotplug_features
{
1662 my ($value, $noerr) = @_;
1664 return $value if parse_hotplug_features
($value);
1666 return undef if $noerr;
1668 die "unable to parse hotplug option\n";
1671 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1672 # [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1673 # [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1674 # [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1675 # [,iothread=on][,serial=serial][,model=model]
1678 my ($key, $data) = @_;
1680 my ($interface, $index);
1682 if ($key =~ m/^([^\d]+)(\d+)$/) {
1689 my $desc = $key =~ /^unused\d+$/ ?
$alldrive_fmt
1690 : $confdesc->{$key}->{format
};
1692 warn "invalid drive key: $key\n";
1695 my $res = eval { PVE
::JSONSchema
::parse_property_string
($desc, $data) };
1696 return undef if !$res;
1697 $res->{interface
} = $interface;
1698 $res->{index} = $index;
1701 foreach my $opt (qw(bps bps_rd bps_wr)) {
1702 if (my $bps = defined(delete $res->{$opt})) {
1703 if (defined($res->{"m$opt"})) {
1704 warn "both $opt and m$opt specified\n";
1708 $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1712 # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1713 for my $requirement (
1714 [mbps_max
=> 'mbps'],
1715 [mbps_rd_max
=> 'mbps_rd'],
1716 [mbps_wr_max
=> 'mbps_wr'],
1717 [miops_max
=> 'miops'],
1718 [miops_rd_max
=> 'miops_rd'],
1719 [miops_wr_max
=> 'miops_wr'],
1720 [bps_max_length
=> 'mbps_max'],
1721 [bps_rd_max_length
=> 'mbps_rd_max'],
1722 [bps_wr_max_length
=> 'mbps_wr_max'],
1723 [iops_max_length
=> 'iops_max'],
1724 [iops_rd_max_length
=> 'iops_rd_max'],
1725 [iops_wr_max_length
=> 'iops_wr_max']) {
1726 my ($option, $requires) = @$requirement;
1727 if ($res->{$option} && !$res->{$requires}) {
1728 warn "$option requires $requires\n";
1733 return undef if $error;
1735 return undef if $res->{mbps_rd
} && $res->{mbps
};
1736 return undef if $res->{mbps_wr
} && $res->{mbps
};
1737 return undef if $res->{iops_rd
} && $res->{iops
};
1738 return undef if $res->{iops_wr
} && $res->{iops
};
1740 if ($res->{media
} && ($res->{media
} eq 'cdrom')) {
1741 return undef if $res->{snapshot
} || $res->{trans
} || $res->{format
};
1742 return undef if $res->{heads
} || $res->{secs
} || $res->{cyls
};
1743 return undef if $res->{interface
} eq 'virtio';
1746 if (my $size = $res->{size
}) {
1747 return undef if !defined($res->{size
} = PVE
::JSONSchema
::parse_size
($size));
1754 my ($vmid, $drive) = @_;
1755 my $data = { %$drive };
1756 delete $data->{$_} for qw(index interface);
1757 return PVE
::JSONSchema
::print_property_string
($data, $alldrive_fmt);
1761 my($fh, $noerr) = @_;
1764 my $SG_GET_VERSION_NUM = 0x2282;
1766 my $versionbuf = "\x00" x
8;
1767 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1769 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1772 my $version = unpack("I", $versionbuf);
1773 if ($version < 30000) {
1774 die "scsi generic interface too old\n" if !$noerr;
1778 my $buf = "\x00" x
36;
1779 my $sensebuf = "\x00" x
8;
1780 my $cmd = pack("C x3 C x1", 0x12, 36);
1782 # see /usr/include/scsi/sg.h
1783 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1785 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1786 length($sensebuf), 0, length($buf), $buf,
1787 $cmd, $sensebuf, 6000);
1789 $ret = ioctl($fh, $SG_IO, $packet);
1791 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1795 my @res = unpack($sg_io_hdr_t, $packet);
1796 if ($res[17] || $res[18]) {
1797 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1802 (my $byte0, my $byte1, $res->{vendor
},
1803 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1805 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1806 $res->{type
} = $byte0 & 31;
1814 my $fh = IO
::File-
>new("+<$path") || return undef;
1815 my $res = scsi_inquiry
($fh, 1);
1821 sub print_tabletdevice_full
{
1822 my ($conf, $arch) = @_;
1824 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1826 # we use uhci for old VMs because tablet driver was buggy in older qemu
1828 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1834 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1837 sub print_keyboarddevice_full
{
1838 my ($conf, $arch, $machine) = @_;
1840 return undef if $arch ne 'aarch64';
1842 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1845 sub print_drivedevice_full
{
1846 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1851 if ($drive->{interface
} eq 'virtio') {
1852 my $pciaddr = print_pci_addr
("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1853 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1854 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread
};
1855 } elsif ($drive->{interface
} eq 'scsi') {
1857 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1858 my $unit = $drive->{index} % $maxdev;
1859 my $devicetype = 'hd';
1861 if (drive_is_cdrom
($drive)) {
1864 if ($drive->{file
} =~ m
|^/|) {
1865 $path = $drive->{file
};
1866 if (my $info = path_is_scsi
($path)) {
1867 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1868 $devicetype = 'block';
1869 } elsif ($info->{type
} == 1) { # tape
1870 $devicetype = 'generic';
1874 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1877 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1878 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, kvm_user_version
());
1879 if ($path =~ m/^iscsi\:\/\
// &&
1880 !min_version
($version, 4, 1)) {
1881 $devicetype = 'generic';
1885 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1886 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1888 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1891 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1892 $device .= ",rotation_rate=1";
1894 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1896 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1897 my $maxdev = ($drive->{interface
} eq 'sata') ?
$MAX_SATA_DISKS : 2;
1898 my $controller = int($drive->{index} / $maxdev);
1899 my $unit = $drive->{index} % $maxdev;
1900 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1902 $device = "ide-$devicetype";
1903 if ($drive->{interface
} eq 'ide') {
1904 $device .= ",bus=ide.$controller,unit=$unit";
1906 $device .= ",bus=ahci$controller.$unit";
1908 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1910 if ($devicetype eq 'hd') {
1911 if (my $model = $drive->{model
}) {
1912 $model = URI
::Escape
::uri_unescape
($model);
1913 $device .= ",model=$model";
1915 if ($drive->{ssd
}) {
1916 $device .= ",rotation_rate=1";
1919 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1920 } elsif ($drive->{interface
} eq 'usb') {
1922 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1924 die "unsupported interface type";
1927 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1929 if (my $serial = $drive->{serial
}) {
1930 $serial = URI
::Escape
::uri_unescape
($serial);
1931 $device .= ",serial=$serial";
1938 sub get_initiator_name
{
1941 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return undef;
1942 while (defined(my $line = <$fh>)) {
1943 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1952 sub print_drive_full
{
1953 my ($storecfg, $vmid, $drive) = @_;
1956 my $volid = $drive->{file
};
1959 if (drive_is_cdrom
($drive)) {
1960 $path = get_iso_path
($storecfg, $vmid, $volid);
1962 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1964 $path = PVE
::Storage
::path
($storecfg, $volid);
1965 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1966 $format = qemu_img_format
($scfg, $volname);
1974 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1975 foreach my $o (@qemu_drive_options) {
1976 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1979 # snapshot only accepts on|off
1980 if (defined($drive->{snapshot
})) {
1981 my $v = $drive->{snapshot
} ?
'on' : 'off';
1982 $opts .= ",snapshot=$v";
1985 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1986 my ($dir, $qmpname) = @$type;
1987 if (my $v = $drive->{"mbps$dir"}) {
1988 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1990 if (my $v = $drive->{"mbps${dir}_max"}) {
1991 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1993 if (my $v = $drive->{"bps${dir}_max_length"}) {
1994 $opts .= ",throttling.bps$qmpname-max-length=$v";
1996 if (my $v = $drive->{"iops${dir}"}) {
1997 $opts .= ",throttling.iops$qmpname=$v";
1999 if (my $v = $drive->{"iops${dir}_max"}) {
2000 $opts .= ",throttling.iops$qmpname-max=$v";
2002 if (my $v = $drive->{"iops${dir}_max_length"}) {
2003 $opts .= ",throttling.iops$qmpname-max-length=$v";
2007 $opts .= ",format=$format" if $format && !$drive->{format
};
2009 my $cache_direct = 0;
2011 if (my $cache = $drive->{cache
}) {
2012 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2013 } elsif (!drive_is_cdrom
($drive)) {
2014 $opts .= ",cache=none";
2018 # aio native works only with O_DIRECT
2019 if (!$drive->{aio
}) {
2021 $opts .= ",aio=native";
2023 $opts .= ",aio=threads";
2027 if (!drive_is_cdrom
($drive)) {
2029 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
2030 $detectzeroes = 'off';
2031 } elsif ($drive->{discard
}) {
2032 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
2034 # This used to be our default with discard not being specified:
2035 $detectzeroes = 'on';
2037 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2040 my $pathinfo = $path ?
"file=$path," : '';
2042 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2045 sub print_netdevice_full
{
2046 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2048 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
2050 my $device = $net->{model
};
2051 if ($net->{model
} eq 'virtio') {
2052 $device = 'virtio-net-pci';
2055 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
2056 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2057 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
2058 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2059 my $vectors = $net->{queues
} * 2 + 2;
2060 $tmpstr .= ",vectors=$vectors,mq=on";
2062 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
2064 if ($use_old_bios_files) {
2066 if ($device eq 'virtio-net-pci') {
2067 $romfile = 'pxe-virtio.rom';
2068 } elsif ($device eq 'e1000') {
2069 $romfile = 'pxe-e1000.rom';
2070 } elsif ($device eq 'ne2k') {
2071 $romfile = 'pxe-ne2k_pci.rom';
2072 } elsif ($device eq 'pcnet') {
2073 $romfile = 'pxe-pcnet.rom';
2074 } elsif ($device eq 'rtl8139') {
2075 $romfile = 'pxe-rtl8139.rom';
2077 $tmpstr .= ",romfile=$romfile" if $romfile;
2083 sub print_netdev_full
{
2084 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2087 if ($netid =~ m/^net(\d+)$/) {
2091 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
2093 my $ifname = "tap${vmid}i$i";
2095 # kvm uses TUNSETIFF ioctl, and that limits ifname length
2096 die "interface name '$ifname' is too long (max 15 character)\n"
2097 if length($ifname) >= 16;
2099 my $vhostparam = '';
2100 if (is_native
($arch)) {
2101 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
2104 my $vmname = $conf->{name
} || "vm$vmid";
2107 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
2109 if ($net->{bridge
}) {
2110 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2112 $netdev = "type=user,id=$netid,hostname=$vmname";
2115 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
2121 sub print_cpu_device
{
2122 my ($conf, $id) = @_;
2124 my $kvm = $conf->{kvm
} // 1;
2125 my $cpu = $kvm ?
"kvm64" : "qemu64";
2126 if (my $cputype = $conf->{cpu
}) {
2127 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
2128 or die "Cannot parse cpu description: $cputype\n";
2129 $cpu = $cpuconf->{cputype
};
2132 my $cores = $conf->{cores
} || 1;
2134 my $current_core = ($id - 1) % $cores;
2135 my $current_socket = int(($id - 1 - $current_core)/$cores);
2137 return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2141 'cirrus' => 'cirrus-vga',
2143 'vmware' => 'vmware-svga',
2144 'virtio' => 'virtio-vga',
2147 sub print_vga_device
{
2148 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2150 my $type = $vga_map->{$vga->{type
}};
2151 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2152 $type = 'virtio-gpu';
2154 my $vgamem_mb = $vga->{memory
};
2156 my $max_outputs = '';
2158 $type = $id ?
'qxl' : 'qxl-vga';
2160 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
2161 # set max outputs so linux can have up to 4 qxl displays with one device
2162 if (min_version
($machine_version, 4, 1)) {
2163 $max_outputs = ",max_outputs=4";
2168 die "no devicetype for $vga->{type}\n" if !$type;
2172 if ($vga->{type
} eq 'virtio') {
2173 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
2174 $memory = ",max_hostmem=$bytes";
2176 # from https://www.spice-space.org/multiple-monitors.html
2177 $memory = ",vgamem_mb=$vga->{memory}";
2178 my $ram = $vgamem_mb * 4;
2179 my $vram = $vgamem_mb * 2;
2180 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2182 $memory = ",vgamem_mb=$vga->{memory}";
2184 } elsif ($qxlnum && $id) {
2185 $memory = ",ram_size=67108864,vram_size=33554432";
2188 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2189 my $vgaid = "vga" . ($id // '');
2192 if ($q35 && $vgaid eq 'vga') {
2193 # the first display uses pcie.0 bus on q35 machines
2194 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
2196 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
2199 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2202 sub drive_is_cloudinit
{
2204 return $drive->{file
} =~ m
@[:/]vm-\d
+-cloudinit
(?
:\
.$QEMU_FORMAT_RE)?
$@;
2207 sub drive_is_cdrom
{
2208 my ($drive, $exclude_cloudinit) = @_;
2210 return 0 if $exclude_cloudinit && drive_is_cloudinit
($drive);
2212 return $drive && $drive->{media
} && ($drive->{media
} eq 'cdrom');
2216 sub parse_number_sets
{
2219 foreach my $part (split(/;/, $set)) {
2220 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2221 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2222 push @$res, [ $1, $2 ];
2224 die "invalid range: $part\n";
2233 my $res = PVE
::JSONSchema
::parse_property_string
($numa_fmt, $data);
2234 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
2235 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
2242 return undef if !$value;
2244 my $res = PVE
::JSONSchema
::parse_property_string
($hostpci_fmt, $value);
2246 my @idlist = split(/;/, $res->{host
});
2247 delete $res->{host
};
2248 foreach my $id (@idlist) {
2249 if ($id =~ m/\./) { # full id 00:00.1
2250 push @{$res->{pciid
}}, {
2253 } else { # partial id 00:00
2254 $res->{pciid
} = PVE
::SysFSTools
::lspci
($id);
2260 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2264 my $res = eval { PVE
::JSONSchema
::parse_property_string
($net_fmt, $data) };
2269 if (!defined($res->{macaddr
})) {
2270 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
2271 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
2276 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2277 sub parse_ipconfig
{
2280 my $res = eval { PVE
::JSONSchema
::parse_property_string
($ipconfig_fmt, $data) };
2286 if ($res->{gw
} && !$res->{ip
}) {
2287 warn 'gateway specified without specifying an IP address';
2290 if ($res->{gw6
} && !$res->{ip6
}) {
2291 warn 'IPv6 gateway specified without specifying an IPv6 address';
2294 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
2295 warn 'gateway specified together with DHCP';
2298 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
2300 warn "IPv6 gateway specified together with $res->{ip6} address";
2304 if (!$res->{ip
} && !$res->{ip6
}) {
2305 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
2314 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
2317 sub add_random_macs
{
2318 my ($settings) = @_;
2320 foreach my $opt (keys %$settings) {
2321 next if $opt !~ m/^net(\d+)$/;
2322 my $net = parse_net
($settings->{$opt});
2324 $settings->{$opt} = print_net
($net);
2328 sub vm_is_volid_owner
{
2329 my ($storecfg, $vmid, $volid) = @_;
2331 if ($volid !~ m
|^/|) {
2333 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
2334 if ($owner && ($owner == $vmid)) {
2342 sub vmconfig_register_unused_drive
{
2343 my ($storecfg, $vmid, $conf, $drive) = @_;
2345 if (drive_is_cloudinit
($drive)) {
2346 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2348 } elsif (!drive_is_cdrom
($drive)) {
2349 my $volid = $drive->{file
};
2350 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2351 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2356 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2360 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2361 format_description
=> 'UUID',
2362 description
=> "Set SMBIOS1 UUID.",
2367 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2368 format_description
=> 'Base64 encoded string',
2369 description
=> "Set SMBIOS1 version.",
2374 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2375 format_description
=> 'Base64 encoded string',
2376 description
=> "Set SMBIOS1 serial number.",
2381 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2382 format_description
=> 'Base64 encoded string',
2383 description
=> "Set SMBIOS1 manufacturer.",
2388 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2389 format_description
=> 'Base64 encoded string',
2390 description
=> "Set SMBIOS1 product ID.",
2395 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2396 format_description
=> 'Base64 encoded string',
2397 description
=> "Set SMBIOS1 SKU string.",
2402 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2403 format_description
=> 'Base64 encoded string',
2404 description
=> "Set SMBIOS1 family string.",
2409 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2417 my $res = eval { PVE
::JSONSchema
::parse_property_string
($smbios1_fmt, $data) };
2424 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2427 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2429 PVE
::JSONSchema
::register_format
('pve-qm-bootdisk', \
&verify_bootdisk
);
2430 sub verify_bootdisk
{
2431 my ($value, $noerr) = @_;
2433 return $value if is_valid_drivename
($value);
2435 return undef if $noerr;
2437 die "invalid boot disk '$value'\n";
2440 sub parse_watchdog
{
2443 return undef if !$value;
2445 my $res = eval { PVE
::JSONSchema
::parse_property_string
($watchdog_fmt, $value) };
2450 sub parse_guest_agent
{
2453 return {} if !defined($value->{agent
});
2455 my $res = eval { PVE
::JSONSchema
::parse_property_string
($agent_fmt, $value->{agent
}) };
2458 # if the agent is disabled ignore the other potentially set properties
2459 return {} if !$res->{enabled
};
2466 return {} if !$value;
2467 my $res = eval { PVE
::JSONSchema
::parse_property_string
($vga_fmt, $value) };
2472 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2473 sub verify_usb_device
{
2474 my ($value, $noerr) = @_;
2476 return $value if parse_usb_device
($value);
2478 return undef if $noerr;
2480 die "unable to parse usb device\n";
2483 # add JSON properties for create and set function
2484 sub json_config_properties
{
2487 foreach my $opt (keys %$confdesc) {
2488 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2489 $prop->{$opt} = $confdesc->{$opt};
2495 # return copy of $confdesc_cloudinit to generate documentation
2496 sub cloudinit_config_properties
{
2498 return dclone
($confdesc_cloudinit);
2502 my ($key, $value) = @_;
2504 die "unknown setting '$key'\n" if !$confdesc->{$key};
2506 my $type = $confdesc->{$key}->{type
};
2508 if (!defined($value)) {
2509 die "got undefined value\n";
2512 if ($value =~ m/[\n\r]/) {
2513 die "property contains a line feed\n";
2516 if ($type eq 'boolean') {
2517 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2518 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2519 die "type check ('boolean') failed - got '$value'\n";
2520 } elsif ($type eq 'integer') {
2521 return int($1) if $value =~ m/^(\d+)$/;
2522 die "type check ('integer') failed - got '$value'\n";
2523 } elsif ($type eq 'number') {
2524 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2525 die "type check ('number') failed - got '$value'\n";
2526 } elsif ($type eq 'string') {
2527 if (my $fmt = $confdesc->{$key}->{format
}) {
2528 PVE
::JSONSchema
::check_format
($fmt, $value);
2531 $value =~ s/^\"(.*)\"$/$1/;
2534 die "internal error"
2539 my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2541 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2543 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2545 if ($conf->{template
}) {
2546 # check if any base image is still used by a linked clone
2547 foreach_drive
($conf, sub {
2548 my ($ds, $drive) = @_;
2549 return if drive_is_cdrom
($drive);
2551 my $volid = $drive->{file
};
2552 return if !$volid || $volid =~ m
|^/|;
2554 die "base volume '$volid' is still in use by linked cloned\n"
2555 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2560 # only remove disks owned by this VM
2561 foreach_drive
($conf, sub {
2562 my ($ds, $drive) = @_;
2563 return if drive_is_cdrom
($drive, 1);
2565 my $volid = $drive->{file
};
2566 return if !$volid || $volid =~ m
|^/|;
2568 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2569 return if !$path || !$owner || ($owner != $vmid);
2571 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2572 warn "Could not remove disk '$volid', check manually: $@" if $@;
2575 # also remove unused disk
2576 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2577 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2578 my ($volid, $sid, $volname, $d) = @_;
2579 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2583 if (defined $replacement_conf) {
2584 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2586 PVE
::QemuConfig-
>destroy_config($vmid);
2590 sub parse_vm_config
{
2591 my ($filename, $raw) = @_;
2593 return undef if !defined($raw);
2596 digest
=> Digest
::SHA
::sha1_hex
($raw),
2601 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2602 || die "got strange filename '$filename'";
2610 my @lines = split(/\n/, $raw);
2611 foreach my $line (@lines) {
2612 next if $line =~ m/^\s*$/;
2614 if ($line =~ m/^\[PENDING\]\s*$/i) {
2615 $section = 'pending';
2616 if (defined($descr)) {
2618 $conf->{description
} = $descr;
2621 $conf = $res->{$section} = {};
2624 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2626 if (defined($descr)) {
2628 $conf->{description
} = $descr;
2631 $conf = $res->{snapshots
}->{$section} = {};
2635 if ($line =~ m/^\#(.*)\s*$/) {
2636 $descr = '' if !defined($descr);
2637 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2641 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2642 $descr = '' if !defined($descr);
2643 $descr .= PVE
::Tools
::decode_text
($2);
2644 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2645 $conf->{snapstate
} = $1;
2646 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2649 $conf->{$key} = $value;
2650 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2652 if ($section eq 'pending') {
2653 $conf->{delete} = $value; # we parse this later
2655 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2657 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2660 eval { $value = check_type
($key, $value); };
2662 warn "vm $vmid - unable to parse value of '$key' - $@";
2664 $key = 'ide2' if $key eq 'cdrom';
2665 my $fmt = $confdesc->{$key}->{format
};
2666 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2667 my $v = parse_drive
($key, $value);
2668 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2669 $v->{file
} = $volid;
2670 $value = print_drive
($vmid, $v);
2672 warn "vm $vmid - unable to parse value of '$key'\n";
2677 $conf->{$key} = $value;
2682 if (defined($descr)) {
2684 $conf->{description
} = $descr;
2686 delete $res->{snapstate
}; # just to be sure
2691 sub write_vm_config
{
2692 my ($filename, $conf) = @_;
2694 delete $conf->{snapstate
}; # just to be sure
2696 if ($conf->{cdrom
}) {
2697 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2698 $conf->{ide2
} = $conf->{cdrom
};
2699 delete $conf->{cdrom
};
2702 # we do not use 'smp' any longer
2703 if ($conf->{sockets
}) {
2704 delete $conf->{smp
};
2705 } elsif ($conf->{smp
}) {
2706 $conf->{sockets
} = $conf->{smp
};
2707 delete $conf->{cores
};
2708 delete $conf->{smp
};
2711 my $used_volids = {};
2713 my $cleanup_config = sub {
2714 my ($cref, $pending, $snapname) = @_;
2716 foreach my $key (keys %$cref) {
2717 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2718 $key eq 'snapstate' || $key eq 'pending';
2719 my $value = $cref->{$key};
2720 if ($key eq 'delete') {
2721 die "propertry 'delete' is only allowed in [PENDING]\n"
2723 # fixme: check syntax?
2726 eval { $value = check_type
($key, $value); };
2727 die "unable to parse value of '$key' - $@" if $@;
2729 $cref->{$key} = $value;
2731 if (!$snapname && is_valid_drivename
($key)) {
2732 my $drive = parse_drive
($key, $value);
2733 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2738 &$cleanup_config($conf);
2740 &$cleanup_config($conf->{pending
}, 1);
2742 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2743 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2744 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2747 # remove 'unusedX' settings if we re-add a volume
2748 foreach my $key (keys %$conf) {
2749 my $value = $conf->{$key};
2750 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2751 delete $conf->{$key};
2755 my $generate_raw_config = sub {
2756 my ($conf, $pending) = @_;
2760 # add description as comment to top of file
2761 if (defined(my $descr = $conf->{description
})) {
2763 foreach my $cl (split(/\n/, $descr)) {
2764 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2767 $raw .= "#\n" if $pending;
2771 foreach my $key (sort keys %$conf) {
2772 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2773 $raw .= "$key: $conf->{$key}\n";
2778 my $raw = &$generate_raw_config($conf);
2780 if (scalar(keys %{$conf->{pending
}})){
2781 $raw .= "\n[PENDING]\n";
2782 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2785 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2786 $raw .= "\n[$snapname]\n";
2787 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2797 # we use static defaults from our JSON schema configuration
2798 foreach my $key (keys %$confdesc) {
2799 if (defined(my $default = $confdesc->{$key}->{default})) {
2800 $res->{$key} = $default;
2808 my $vmlist = PVE
::Cluster
::get_vmlist
();
2810 return $res if !$vmlist || !$vmlist->{ids
};
2811 my $ids = $vmlist->{ids
};
2813 foreach my $vmid (keys %$ids) {
2814 my $d = $ids->{$vmid};
2815 next if !$d->{node
} || $d->{node
} ne $nodename;
2816 next if !$d->{type
} || $d->{type
} ne 'qemu';
2817 $res->{$vmid}->{exists} = 1;
2822 # test if VM uses local resources (to prevent migration)
2823 sub check_local_resources
{
2824 my ($conf, $noerr) = @_;
2828 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2829 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2831 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2833 foreach my $k (keys %$conf) {
2834 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2835 # sockets are safe: they will recreated be on the target side post-migrate
2836 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2837 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2840 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2845 # check if used storages are available on all nodes (use by migrate)
2846 sub check_storage_availability
{
2847 my ($storecfg, $conf, $node) = @_;
2849 foreach_drive
($conf, sub {
2850 my ($ds, $drive) = @_;
2852 my $volid = $drive->{file
};
2855 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2858 # check if storage is available on both nodes
2859 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2860 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2864 # list nodes where all VM images are available (used by has_feature API)
2866 my ($conf, $storecfg) = @_;
2868 my $nodelist = PVE
::Cluster
::get_nodelist
();
2869 my $nodehash = { map { $_ => 1 } @$nodelist };
2870 my $nodename = PVE
::INotify
::nodename
();
2872 foreach_drive
($conf, sub {
2873 my ($ds, $drive) = @_;
2875 my $volid = $drive->{file
};
2878 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2880 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2881 if ($scfg->{disable
}) {
2883 } elsif (my $avail = $scfg->{nodes
}) {
2884 foreach my $node (keys %$nodehash) {
2885 delete $nodehash->{$node} if !$avail->{$node};
2887 } elsif (!$scfg->{shared
}) {
2888 foreach my $node (keys %$nodehash) {
2889 delete $nodehash->{$node} if $node ne $nodename
2898 sub check_local_storage_availability
{
2899 my ($conf, $storecfg) = @_;
2901 my $nodelist = PVE
::Cluster
::get_nodelist
();
2902 my $nodehash = { map { $_ => {} } @$nodelist };
2904 foreach_drive
($conf, sub {
2905 my ($ds, $drive) = @_;
2907 my $volid = $drive->{file
};
2910 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2912 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2914 if ($scfg->{disable
}) {
2915 foreach my $node (keys %$nodehash) {
2916 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2918 } elsif (my $avail = $scfg->{nodes
}) {
2919 foreach my $node (keys %$nodehash) {
2920 if (!$avail->{$node}) {
2921 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2928 foreach my $node (values %$nodehash) {
2929 if (my $unavail = $node->{unavailable_storages
}) {
2930 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2937 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2939 my ($vmid, $nocheck, $node) = @_;
2941 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2942 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2947 my $vzlist = config_list
();
2949 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2951 while (defined(my $de = $fd->read)) {
2952 next if $de !~ m/^(\d+)\.pid$/;
2954 next if !defined($vzlist->{$vmid});
2955 if (my $pid = check_running
($vmid)) {
2956 $vzlist->{$vmid}->{pid
} = $pid;
2964 my ($storecfg, $conf) = @_;
2966 my $bootdisk = $conf->{bootdisk
};
2967 return undef if !$bootdisk;
2968 return undef if !is_valid_drivename
($bootdisk);
2970 return undef if !$conf->{$bootdisk};
2972 my $drive = parse_drive
($bootdisk, $conf->{$bootdisk});
2973 return undef if !defined($drive);
2975 return undef if drive_is_cdrom
($drive);
2977 my $volid = $drive->{file
};
2978 return undef if !$volid;
2980 return $drive->{size
};
2983 our $vmstatus_return_properties = {
2984 vmid
=> get_standard_option
('pve-vmid'),
2986 description
=> "Qemu process status.",
2988 enum
=> ['stopped', 'running'],
2991 description
=> "Maximum memory in bytes.",
2994 renderer
=> 'bytes',
2997 description
=> "Root disk size in bytes.",
3000 renderer
=> 'bytes',
3003 description
=> "VM name.",
3008 description
=> "Qemu QMP agent status.",
3013 description
=> "PID of running qemu process.",
3018 description
=> "Uptime.",
3021 renderer
=> 'duration',
3024 description
=> "Maximum usable CPUs.",
3029 description
=> "The current config lock, if any.",
3034 description
=> "The current configured tags, if any",
3040 my $last_proc_pid_stat;
3042 # get VM status information
3043 # This must be fast and should not block ($full == false)
3044 # We only query KVM using QMP if $full == true (this can be slow)
3046 my ($opt_vmid, $full) = @_;
3050 my $storecfg = PVE
::Storage
::config
();
3052 my $list = vzlist
();
3053 my $defaults = load_defaults
();
3055 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
3057 my $cpucount = $cpuinfo->{cpus
} || 1;
3059 foreach my $vmid (keys %$list) {
3060 next if $opt_vmid && ($vmid ne $opt_vmid);
3062 my $conf = PVE
::QemuConfig-
>load_config($vmid);
3064 my $d = { vmid
=> $vmid };
3065 $d->{pid
} = $list->{$vmid}->{pid
};
3067 # fixme: better status?
3068 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
3070 my $size = disksize
($storecfg, $conf);
3071 if (defined($size)) {
3072 $d->{disk
} = 0; # no info available
3073 $d->{maxdisk
} = $size;
3079 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
3080 * ($conf->{cores
} || $defaults->{cores
});
3081 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
3082 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
3084 $d->{name
} = $conf->{name
} || "VM $vmid";
3085 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
3086 : $defaults->{memory
}*(1024*1024);
3088 if ($conf->{balloon
}) {
3089 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
3090 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
3091 : $defaults->{shares
};
3102 $d->{diskwrite
} = 0;
3104 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
3106 $d->{serial
} = 1 if conf_has_serial
($conf);
3107 $d->{lock} = $conf->{lock} if $conf->{lock};
3108 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
3113 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
3114 foreach my $dev (keys %$netdev) {
3115 next if $dev !~ m/^tap([1-9]\d*)i/;
3117 my $d = $res->{$vmid};
3120 $d->{netout
} += $netdev->{$dev}->{receive
};
3121 $d->{netin
} += $netdev->{$dev}->{transmit
};
3124 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
3125 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
3130 my $ctime = gettimeofday
;
3132 foreach my $vmid (keys %$list) {
3134 my $d = $res->{$vmid};
3135 my $pid = $d->{pid
};
3138 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
3139 next if !$pstat; # not running
3141 my $used = $pstat->{utime} + $pstat->{stime
};
3143 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
3145 if ($pstat->{vsize
}) {
3146 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
3149 my $old = $last_proc_pid_stat->{$pid};
3151 $last_proc_pid_stat->{$pid} = {
3159 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
3161 if ($dtime > 1000) {
3162 my $dutime = $used - $old->{used
};
3164 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
3165 $last_proc_pid_stat->{$pid} = {
3171 $d->{cpu
} = $old->{cpu
};
3175 return $res if !$full;
3177 my $qmpclient = PVE
::QMPClient-
>new();
3179 my $ballooncb = sub {
3180 my ($vmid, $resp) = @_;
3182 my $info = $resp->{'return'};
3183 return if !$info->{max_mem
};
3185 my $d = $res->{$vmid};
3187 # use memory assigned to VM
3188 $d->{maxmem
} = $info->{max_mem
};
3189 $d->{balloon
} = $info->{actual
};
3191 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
3192 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
3193 $d->{freemem
} = $info->{free_mem
};
3196 $d->{ballooninfo
} = $info;
3199 my $blockstatscb = sub {
3200 my ($vmid, $resp) = @_;
3201 my $data = $resp->{'return'} || [];
3202 my $totalrdbytes = 0;
3203 my $totalwrbytes = 0;
3205 for my $blockstat (@$data) {
3206 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
3207 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
3209 $blockstat->{device
} =~ s/drive-//;
3210 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
3212 $res->{$vmid}->{diskread
} = $totalrdbytes;
3213 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
3216 my $statuscb = sub {
3217 my ($vmid, $resp) = @_;
3219 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3220 # this fails if ballon driver is not loaded, so this must be
3221 # the last commnand (following command are aborted if this fails).
3222 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3224 my $status = 'unknown';
3225 if (!defined($status = $resp->{'return'}->{status
})) {
3226 warn "unable to get VM status\n";
3230 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3233 foreach my $vmid (keys %$list) {
3234 next if $opt_vmid && ($vmid ne $opt_vmid);
3235 next if !$res->{$vmid}->{pid
}; # not running
3236 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3239 $qmpclient->queue_execute(undef, 2);
3241 foreach my $vmid (keys %$list) {
3242 next if $opt_vmid && ($vmid ne $opt_vmid);
3243 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3250 my ($conf, $func, @param) = @_;
3252 foreach my $ds (valid_drive_names
()) {
3253 next if !defined($conf->{$ds});
3255 my $drive = parse_drive
($ds, $conf->{$ds});
3258 &$func($ds, $drive, @param);
3263 my ($conf, $func, @param) = @_;
3267 my $test_volid = sub {
3268 my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3272 $volhash->{$volid}->{cdrom
} //= 1;
3273 $volhash->{$volid}->{cdrom
} = 0 if !$is_cdrom;
3275 $volhash->{$volid}->{replicate
} //= 0;
3276 $volhash->{$volid}->{replicate
} = 1 if $replicate;
3278 $volhash->{$volid}->{shared
} //= 0;
3279 $volhash->{$volid}->{shared
} = 1 if $shared;
3281 $volhash->{$volid}->{referenced_in_config
} //= 0;
3282 $volhash->{$volid}->{referenced_in_config
} = 1 if !defined($snapname);
3284 $volhash->{$volid}->{referenced_in_snapshot
}->{$snapname} = 1
3285 if defined($snapname);
3286 $volhash->{$volid}->{size
} = $size if $size;
3289 foreach_drive
($conf, sub {
3290 my ($ds, $drive) = @_;
3291 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, undef, $drive->{size
});
3294 foreach my $snapname (keys %{$conf->{snapshots
}}) {
3295 my $snap = $conf->{snapshots
}->{$snapname};
3296 $test_volid->($snap->{vmstate
}, 0, 1, $snapname);
3297 foreach_drive
($snap, sub {
3298 my ($ds, $drive) = @_;
3299 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, $snapname);
3303 foreach my $volid (keys %$volhash) {
3304 &$func($volid, $volhash->{$volid}, @param);
3308 sub conf_has_serial
{
3311 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3312 if ($conf->{"serial$i"}) {
3320 sub conf_has_audio
{
3321 my ($conf, $id) = @_;
3324 my $audio = $conf->{"audio$id"};
3325 return undef if !defined($audio);
3327 my $audioproperties = PVE
::JSONSchema
::parse_property_string
($audio_fmt, $audio);
3328 my $audiodriver = $audioproperties->{driver
} // 'spice';
3331 dev
=> $audioproperties->{device
},
3332 dev_id
=> "audiodev$id",
3333 backend
=> $audiodriver,
3334 backend_id
=> "$audiodriver-backend${id}",
3338 sub vga_conf_has_spice
{
3341 my $vgaconf = parse_vga
($vga);
3342 my $vgatype = $vgaconf->{type
};
3343 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3350 return get_host_arch
() eq $arch;
3355 return $conf->{arch
} // get_host_arch
();
3358 my $default_machines = {
3363 sub get_vm_machine
{
3364 my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
3366 my $machine = $forcemachine || $conf->{machine
};
3368 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3370 $machine ||= $default_machines->{$arch};
3371 $machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
3377 sub get_ovmf_files
($) {
3380 my $ovmf = $OVMF->{$arch}
3381 or die "no OVMF images known for architecture '$arch'\n";
3387 aarch64
=> '/usr/bin/qemu-system-aarch64',
3388 x86_64
=> '/usr/bin/qemu-system-x86_64',
3390 sub get_command_for_arch
($) {
3392 return '/usr/bin/kvm' if is_native
($arch);
3394 my $cmd = $Arch2Qemu->{$arch}
3395 or die "don't know how to emulate architecture '$arch'\n";
3399 sub get_cpu_options
{
3400 my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3403 my $ostype = $conf->{ostype
};
3405 my $cpu = $kvm ?
"kvm64" : "qemu64";
3406 if ($arch eq 'aarch64') {
3407 $cpu = 'cortex-a57';
3410 if (my $cputype = $conf->{cpu
}) {
3411 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
3412 or die "Cannot parse cpu description: $cputype\n";
3413 $cpu = $cpuconf->{cputype
};
3414 $kvm_off = 1 if $cpuconf->{hidden
};
3415 $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3417 if (defined(my $flags = $cpuconf->{flags
})) {
3418 push @$cpuFlags, split(";", $flags);
3422 push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3424 push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3426 push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3428 push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3430 if (min_version
($machine_version, 2, 3) && $arch eq 'x86_64') {
3432 push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3433 push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3436 add_hyperv_enlightenments
($cpuFlags, $winversion, $machine_version, $conf->{bios
}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3438 push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3440 push @$cpuFlags, 'kvm=off' if $kvm_off;
3442 if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3443 push @$cpuFlags, "vendor=${cpu_vendor}"
3444 if $cpu_vendor ne 'default';
3445 } elsif ($arch ne 'aarch64') {
3446 die "internal error"; # should not happen
3449 $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3451 return ('-cpu', $cpu);
3454 sub config_to_command
{
3455 my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3458 my $globalFlags = [];
3459 my $machineFlags = [];
3464 my $vernum = 0; # unknown
3465 my $ostype = $conf->{ostype
};
3466 my $winversion = windows_version
($ostype);
3467 my $kvm = $conf->{kvm
};
3469 my $arch = get_vm_arch
($conf);
3470 my $kvm_binary = get_command_for_arch
($arch);
3471 my $kvmver = kvm_user_version
($kvm_binary);
3473 my $add_pve_version = min_version
($kvmver, 4, 1);
3475 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3476 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, $kvmver);
3477 $kvm //= 1 if is_native
($arch);
3480 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3481 if !defined kvm_version
();
3484 if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3485 $vernum = $1*1000000+$2*1000;
3486 } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3487 $vernum = $1*1000000+$2*1000+$3;
3490 die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3492 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3493 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3494 my $use_old_bios_files = undef;
3495 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3497 my $cpuunits = defined($conf->{cpuunits
}) ?
3498 $conf->{cpuunits
} : $defaults->{cpuunits
};
3500 push @$cmd, $kvm_binary;
3502 push @$cmd, '-id', $vmid;
3504 my $vmname = $conf->{name
} || "vm$vmid";
3506 push @$cmd, '-name', $vmname;
3510 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3511 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3512 push @$cmd, '-mon', "chardev=qmp,mode=control";
3514 if (min_version
($machine_version, 2, 12)) {
3515 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3516 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3519 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3521 push @$cmd, '-daemonize';
3523 if ($conf->{smbios1
}) {
3524 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3525 if ($smbios_conf->{base64
}) {
3526 # Do not pass base64 flag to qemu
3527 delete $smbios_conf->{base64
};
3528 my $smbios_string = "";
3529 foreach my $key (keys %$smbios_conf) {
3531 if ($key eq "uuid") {
3532 $value = $smbios_conf->{uuid
}
3534 $value = decode_base64
($smbios_conf->{$key});
3536 # qemu accepts any binary data, only commas need escaping by double comma
3538 $smbios_string .= "," . $key . "=" . $value if $value;
3540 push @$cmd, '-smbios', "type=1" . $smbios_string;
3542 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3546 if ($conf->{vmgenid
}) {
3547 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3550 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3551 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3552 die "uefi base image not found\n" if ! -f
$ovmf_code;
3556 if (my $efidisk = $conf->{efidisk0
}) {
3557 my $d = PVE
::JSONSchema
::parse_property_string
($efidisk_fmt, $efidisk);
3558 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3559 $format = $d->{format
};
3561 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3562 if (!defined($format)) {
3563 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3564 $format = qemu_img_format
($scfg, $volname);
3568 die "efidisk format must be specified\n"
3569 if !defined($format);
3572 warn "no efidisk configured! Using temporary efivars disk.\n";
3573 $path = "/tmp/$vmid-ovmf.fd";
3574 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3578 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3579 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3584 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3585 if (min_version
($machine_version, 4, 0)) {
3586 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3588 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3592 # add usb controllers
3593 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
($conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3594 push @$devices, @usbcontrollers if @usbcontrollers;
3595 my $vga = parse_vga
($conf->{vga
});
3597 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3598 $vga->{type
} = 'qxl' if $qxlnum;
3600 if (!$vga->{type
}) {
3601 if ($arch eq 'aarch64') {
3602 $vga->{type
} = 'virtio';
3603 } elsif (min_version
($machine_version, 2, 9)) {
3604 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3606 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3610 # enable absolute mouse coordinates (needed by vnc)
3612 if (defined($conf->{tablet
})) {
3613 $tablet = $conf->{tablet
};
3615 $tablet = $defaults->{tablet
};
3616 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3617 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3621 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3622 my $kbd = print_keyboarddevice_full
($conf, $arch);
3623 push @$devices, '-device', $kbd if defined($kbd);
3627 my $gpu_passthrough;
3630 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
3631 my $id = "hostpci$i";
3632 my $d = parse_hostpci
($conf->{$id});
3635 if (my $pcie = $d->{pcie
}) {
3636 die "q35 machine model is not enabled" if !$q35;
3637 # win7 wants to have the pcie devices directly on the pcie bus
3638 # instead of in the root port
3639 if ($winversion == 7) {
3640 $pciaddr = print_pcie_addr
("${id}bus0");
3642 # add more root ports if needed, 4 are present by default
3643 # by pve-q35 cfgs, rest added here on demand.
3645 push @$devices, '-device', print_pcie_root_port
($i);
3647 $pciaddr = print_pcie_addr
($id);
3650 $pciaddr = print_pci_addr
($id, $bridges, $arch, $machine_type);
3654 if ($d->{'x-vga'}) {
3655 $xvga = ',x-vga=on' if !($conf->{bios
} && $conf->{bios
} eq 'ovmf');
3657 $vga->{type
} = 'none' if !defined($conf->{vga
});
3658 $gpu_passthrough = 1;
3661 my $pcidevices = $d->{pciid
};
3662 my $multifunction = 1 if @$pcidevices > 1;
3665 if ($d->{mdev
} && scalar(@$pcidevices) == 1) {
3666 my $pci_id = $pcidevices->[0]->{id
};
3667 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $i);
3668 $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3669 } elsif ($d->{mdev
}) {
3670 warn "ignoring mediated device '$id' with multifunction device\n";
3674 foreach my $pcidevice (@$pcidevices) {
3675 my $devicestr = "vfio-pci";
3678 $devicestr .= ",sysfsdev=$sysfspath";
3680 $devicestr .= ",host=$pcidevice->{id}";
3683 my $mf_addr = $multifunction ?
".$j" : '';
3684 $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3687 $devicestr .= ',rombar=0' if defined($d->{rombar
}) && !$d->{rombar
};
3688 $devicestr .= "$xvga";
3689 $devicestr .= ",multifunction=on" if $multifunction;
3690 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile
};
3693 push @$devices, '-device', $devicestr;
3699 my $usb_dev_features = {};
3700 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3702 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
($conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features);
3703 push @$devices, @usbdevices if @usbdevices;
3705 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3706 if (my $path = $conf->{"serial$i"}) {
3707 if ($path eq 'socket') {
3708 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3709 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3710 # On aarch64, serial0 is the UART device. Qemu only allows
3711 # connecting UART devices via the '-serial' command line, as
3712 # the device has a fixed slot on the hardware...
3713 if ($arch eq 'aarch64' && $i == 0) {
3714 push @$devices, '-serial', "chardev:serial$i";
3716 push @$devices, '-device', "isa-serial,chardev=serial$i";
3719 die "no such serial device\n" if ! -c
$path;
3720 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3721 push @$devices, '-device', "isa-serial,chardev=serial$i";
3727 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3728 if (my $path = $conf->{"parallel$i"}) {
3729 die "no such parallel device\n" if ! -c
$path;
3730 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3731 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3732 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3736 if (my $audio = conf_has_audio
($conf)) {
3738 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3740 my $id = $audio->{dev_id
};
3741 if ($audio->{dev
} eq 'AC97') {
3742 push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3743 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3744 push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3745 push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3746 push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3748 die "unkown audio device '$audio->{dev}', implement me!";
3751 push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3755 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3756 $sockets = $conf->{sockets
} if $conf->{sockets
};
3758 my $cores = $conf->{cores
} || 1;
3760 my $maxcpus = $sockets * $cores;
3762 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3764 my $allowed_vcpus = $cpuinfo->{cpus
};
3766 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3767 if ($allowed_vcpus < $maxcpus);
3769 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3771 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3772 for (my $i = 2; $i <= $vcpus; $i++) {
3773 my $cpustr = print_cpu_device
($conf,$i);
3774 push @$cmd, '-device', $cpustr;
3779 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3781 push @$cmd, '-nodefaults';
3783 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
3785 my $bootindex_hash = {};
3787 foreach my $o (split(//, $bootorder)) {
3788 $bootindex_hash->{$o} = $i*100;
3792 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3794 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3796 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3798 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3799 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3800 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3801 push @$cmd, '-vnc', "unix:$socket,password";
3803 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3804 push @$cmd, '-nographic';
3808 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3810 my $useLocaltime = $conf->{localtime};
3812 if ($winversion >= 5) { # windows
3813 $useLocaltime = 1 if !defined($conf->{localtime});
3815 # use time drift fix when acpi is enabled
3816 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3817 $tdf = 1 if !defined($conf->{tdf
});
3821 if ($winversion >= 6) {
3822 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3823 push @$cmd, '-no-hpet';
3826 push @$rtcFlags, 'driftfix=slew' if $tdf;
3829 push @$machineFlags, 'accel=tcg';
3832 if ($machine_type) {
3833 push @$machineFlags, "type=${machine_type}";
3836 if (($conf->{startdate
}) && ($conf->{startdate
} ne 'now')) {
3837 push @$rtcFlags, "base=$conf->{startdate}";
3838 } elsif ($useLocaltime) {
3839 push @$rtcFlags, 'base=localtime';
3842 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3844 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3846 push @$cmd, '-S' if $conf->{freeze
};
3848 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3850 my $guest_agent = parse_guest_agent
($conf);
3852 if ($guest_agent->{enabled
}) {
3853 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3854 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3856 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3857 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3858 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3859 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3860 } elsif ($guest_agent->{type
} eq 'isa') {
3861 push @$devices, '-device', "isa-serial,chardev=qga0";
3870 for(my $i = 1; $i < $qxlnum; $i++){
3871 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3874 # assume other OS works like Linux
3875 my ($ram, $vram) = ("134217728", "67108864");
3876 if ($vga->{memory
}) {
3877 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3878 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3880 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3881 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3885 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3887 my $nodename = PVE
::INotify
::nodename
();
3888 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3889 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3890 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3892 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3893 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3894 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3896 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3897 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3899 my $spice_enhancement = PVE
::JSONSchema
::parse_property_string
($spice_enhancements_fmt, $conf->{spice_enhancements
} // '');
3900 if ($spice_enhancement->{foldersharing
}) {
3901 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3902 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3905 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3906 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming
};
3907 push @$devices, '-spice', "$spice_opts";
3910 # enable balloon by default, unless explicitly disabled
3911 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3912 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3913 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3916 if ($conf->{watchdog
}) {
3917 my $wdopts = parse_watchdog
($conf->{watchdog
});
3918 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3919 my $watchdog = $wdopts->{model
} || 'i6300esb';
3920 push @$devices, '-device', "$watchdog$pciaddr";
3921 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3925 my $scsicontroller = {};
3926 my $ahcicontroller = {};
3927 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3929 # Add iscsi initiator name if available
3930 if (my $initiator = get_initiator_name
()) {
3931 push @$devices, '-iscsi', "initiator-name=$initiator";
3934 foreach_drive
($conf, sub {
3935 my ($ds, $drive) = @_;
3937 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3938 push @$vollist, $drive->{file
};
3941 # ignore efidisk here, already added in bios/fw handling code above
3942 return if $drive->{interface
} eq 'efidisk';
3944 $use_virtio = 1 if $ds =~ m/^virtio/;
3946 if (drive_is_cdrom
($drive)) {
3947 if ($bootindex_hash->{d
}) {
3948 $drive->{bootindex
} = $bootindex_hash->{d
};
3949 $bootindex_hash->{d
} += 1;
3952 if ($bootindex_hash->{c
}) {
3953 $drive->{bootindex
} = $bootindex_hash->{c
} if $conf->{bootdisk
} && ($conf->{bootdisk
} eq $ds);
3954 $bootindex_hash->{c
} += 1;
3958 if($drive->{interface
} eq 'virtio'){
3959 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3962 if ($drive->{interface
} eq 'scsi') {
3964 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3966 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3967 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3970 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3971 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3972 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3973 } elsif ($drive->{iothread
}) {
3974 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3978 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3979 $queues = ",num_queues=$drive->{queues}";
3982 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3983 $scsicontroller->{$controller}=1;
3986 if ($drive->{interface
} eq 'sata') {
3987 my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3988 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3989 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3990 $ahcicontroller->{$controller}=1;
3993 my $drive_cmd = print_drive_full
($storecfg, $vmid, $drive);
3994 push @$devices, '-drive',$drive_cmd;
3995 push @$devices, '-device', print_drivedevice_full
($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3998 for (my $i = 0; $i < $MAX_NETS; $i++) {
3999 next if !$conf->{"net$i"};
4000 my $d = parse_net
($conf->{"net$i"});
4003 $use_virtio = 1 if $d->{model
} eq 'virtio';
4005 if ($bootindex_hash->{n
}) {
4006 $d->{bootindex
} = $bootindex_hash->{n
};
4007 $bootindex_hash->{n
} += 1;
4010 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, "net$i");
4011 push @$devices, '-netdev', $netdevfull;
4013 my $netdevicefull = print_netdevice_full
($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4014 push @$devices, '-device', $netdevicefull;
4017 if ($conf->{ivshmem
}) {
4018 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4022 $bus = print_pcie_addr
("ivshmem");
4024 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4027 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4028 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4030 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4031 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4036 if (min_version
($machine_version, 2, 3)) {
4041 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4043 for my $k (sort {$b cmp $a} keys %$bridges) {
4044 $pciaddr = print_pci_addr
("pci.$k", undef, $arch, $machine_type);
4045 unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4049 push @$cmd, @$devices;
4050 push @$cmd, '-rtc', join(',', @$rtcFlags)
4051 if scalar(@$rtcFlags);
4052 push @$cmd, '-machine', join(',', @$machineFlags)
4053 if scalar(@$machineFlags);
4054 push @$cmd, '-global', join(',', @$globalFlags)
4055 if scalar(@$globalFlags);
4057 if (my $vmstate = $conf->{vmstate
}) {
4058 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4059 push @$vollist, $vmstate;
4060 push @$cmd, '-loadstate', $statepath;
4064 if ($conf->{args
}) {
4065 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4069 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
4075 my $res = mon_cmd
($vmid, 'query-spice');
4077 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4080 sub vm_devices_list
{
4083 my $res = mon_cmd
($vmid, 'query-pci');
4084 my $devices_to_check = [];
4086 foreach my $pcibus (@$res) {
4087 push @$devices_to_check, @{$pcibus->{devices
}},
4090 while (@$devices_to_check) {
4092 for my $d (@$devices_to_check) {
4093 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4094 next if !$d->{'pci_bridge'};
4096 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4097 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4099 $devices_to_check = $to_check;
4102 my $resblock = mon_cmd
($vmid, 'query-block');
4103 foreach my $block (@$resblock) {
4104 if($block->{device
} =~ m/^drive-(\S+)/){
4109 my $resmice = mon_cmd
($vmid, 'query-mice');
4110 foreach my $mice (@$resmice) {
4111 if ($mice->{name
} eq 'QEMU HID Tablet') {
4112 $devices->{tablet
} = 1;
4117 # for usb devices there is no query-usb
4118 # but we can iterate over the entries in
4119 # qom-list path=/machine/peripheral
4120 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4121 foreach my $per (@$resperipheral) {
4122 if ($per->{name
} =~ m/^usb\d+$/) {
4123 $devices->{$per->{name
}} = 1;
4131 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4133 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4135 my $devices_list = vm_devices_list
($vmid);
4136 return 1 if defined($devices_list->{$deviceid});
4138 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4140 if ($deviceid eq 'tablet') {
4142 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4144 } elsif ($deviceid eq 'keyboard') {
4146 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4148 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4150 die "usb hotplug currently not reliable\n";
4151 # since we can't reliably hot unplug all added usb devices
4152 # and usb passthrough disables live migration
4153 # we disable usb hotplugging for now
4154 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device));
4156 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4158 qemu_iothread_add
($vmid, $deviceid, $device);
4160 qemu_driveadd
($storecfg, $vmid, $device);
4161 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4163 qemu_deviceadd
($vmid, $devicefull);
4164 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4166 eval { qemu_drivedel
($vmid, $deviceid); };
4171 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4174 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4175 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4176 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4178 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4180 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4181 qemu_iothread_add
($vmid, $deviceid, $device);
4182 $devicefull .= ",iothread=iothread-$deviceid";
4185 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4186 $devicefull .= ",num_queues=$device->{queues}";
4189 qemu_deviceadd
($vmid, $devicefull);
4190 qemu_deviceaddverify
($vmid, $deviceid);
4192 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4194 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4195 qemu_driveadd
($storecfg, $vmid, $device);
4197 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4198 eval { qemu_deviceadd
($vmid, $devicefull); };
4200 eval { qemu_drivedel
($vmid, $deviceid); };
4205 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4207 return undef if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4209 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4210 my $use_old_bios_files = undef;
4211 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4213 my $netdevicefull = print_netdevice_full
($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4214 qemu_deviceadd
($vmid, $netdevicefull);
4216 qemu_deviceaddverify
($vmid, $deviceid);
4217 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4220 eval { qemu_netdevdel
($vmid, $deviceid); };
4225 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4228 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4229 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4231 qemu_deviceadd
($vmid, $devicefull);
4232 qemu_deviceaddverify
($vmid, $deviceid);
4235 die "can't hotplug device '$deviceid'\n";
4241 # fixme: this should raise exceptions on error!
4242 sub vm_deviceunplug
{
4243 my ($vmid, $conf, $deviceid) = @_;
4245 my $devices_list = vm_devices_list
($vmid);
4246 return 1 if !defined($devices_list->{$deviceid});
4248 die "can't unplug bootdisk" if $conf->{bootdisk
} && $conf->{bootdisk
} eq $deviceid;
4250 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4252 qemu_devicedel
($vmid, $deviceid);
4254 } elsif ($deviceid =~ m/^usb\d+$/) {
4256 die "usb hotplug currently not reliable\n";
4257 # when unplugging usb devices this way,
4258 # there may be remaining usb controllers/hubs
4259 # so we disable it for now
4260 qemu_devicedel
($vmid, $deviceid);
4261 qemu_devicedelverify
($vmid, $deviceid);
4263 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4265 qemu_devicedel
($vmid, $deviceid);
4266 qemu_devicedelverify
($vmid, $deviceid);
4267 qemu_drivedel
($vmid, $deviceid);
4268 qemu_iothread_del
($conf, $vmid, $deviceid);
4270 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4272 qemu_devicedel
($vmid, $deviceid);
4273 qemu_devicedelverify
($vmid, $deviceid);
4274 qemu_iothread_del
($conf, $vmid, $deviceid);
4276 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4278 qemu_devicedel
($vmid, $deviceid);
4279 qemu_drivedel
($vmid, $deviceid);
4280 qemu_deletescsihw
($conf, $vmid, $deviceid);
4282 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4284 qemu_devicedel
($vmid, $deviceid);
4285 qemu_devicedelverify
($vmid, $deviceid);
4286 qemu_netdevdel
($vmid, $deviceid);
4289 die "can't unplug device '$deviceid'\n";
4295 sub qemu_deviceadd
{
4296 my ($vmid, $devicefull) = @_;
4298 $devicefull = "driver=".$devicefull;
4299 my %options = split(/[=,]/, $devicefull);
4301 mon_cmd
($vmid, "device_add" , %options);
4304 sub qemu_devicedel
{
4305 my ($vmid, $deviceid) = @_;
4307 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4310 sub qemu_iothread_add
{
4311 my($vmid, $deviceid, $device) = @_;
4313 if ($device->{iothread
}) {
4314 my $iothreads = vm_iothreads_list
($vmid);
4315 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4319 sub qemu_iothread_del
{
4320 my($conf, $vmid, $deviceid) = @_;
4322 my $confid = $deviceid;
4323 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4324 $confid = 'scsi' . $1;
4326 my $device = parse_drive
($confid, $conf->{$confid});
4327 if ($device->{iothread
}) {
4328 my $iothreads = vm_iothreads_list
($vmid);
4329 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4333 sub qemu_objectadd
{
4334 my($vmid, $objectid, $qomtype) = @_;
4336 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4341 sub qemu_objectdel
{
4342 my($vmid, $objectid) = @_;
4344 mon_cmd
($vmid, "object-del", id
=> $objectid);
4350 my ($storecfg, $vmid, $device) = @_;
4352 my $drive = print_drive_full
($storecfg, $vmid, $device);
4353 $drive =~ s/\\/\\\\/g;
4354 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4356 # If the command succeeds qemu prints: "OK
"
4357 return 1 if $ret =~ m/OK/s;
4359 die "adding drive failed
: $ret\n";
4363 my($vmid, $deviceid) = @_;
4365 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4368 return 1 if $ret eq "";
4370 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4371 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4373 die "deleting drive
$deviceid failed
: $ret\n";
4376 sub qemu_deviceaddverify {
4377 my ($vmid, $deviceid) = @_;
4379 for (my $i = 0; $i <= 5; $i++) {
4380 my $devices_list = vm_devices_list($vmid);
4381 return 1 if defined($devices_list->{$deviceid});
4385 die "error on hotplug device
'$deviceid'\n";
4389 sub qemu_devicedelverify {
4390 my ($vmid, $deviceid) = @_;
4392 # need to verify that the device is correctly removed as device_del
4393 # is async and empty return is not reliable
4395 for (my $i = 0; $i <= 5; $i++) {
4396 my $devices_list = vm_devices_list($vmid);
4397 return 1 if !defined($devices_list->{$deviceid});
4401 die "error on hot-unplugging device
'$deviceid'\n";
4404 sub qemu_findorcreatescsihw {
4405 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4407 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4409 my $scsihwid="$controller_prefix$controller";
4410 my $devices_list = vm_devices_list($vmid);
4412 if(!defined($devices_list->{$scsihwid})) {
4413 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4419 sub qemu_deletescsihw {
4420 my ($conf, $vmid, $opt) = @_;
4422 my $device = parse_drive($opt, $conf->{$opt});
4424 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4425 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4429 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4431 my $devices_list = vm_devices_list($vmid);
4432 foreach my $opt (keys %{$devices_list}) {
4433 if (PVE::QemuServer::is_valid_drivename($opt)) {
4434 my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4435 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4441 my $scsihwid="scsihw
$controller";
4443 vm_deviceunplug($vmid, $conf, $scsihwid);
4448 sub qemu_add_pci_bridge {
4449 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4455 print_pci_addr($device, $bridges, $arch, $machine_type);
4457 while (my ($k, $v) = each %$bridges) {
4460 return 1 if !defined($bridgeid) || $bridgeid < 1;
4462 my $bridge = "pci
.$bridgeid";
4463 my $devices_list = vm_devices_list($vmid);
4465 if (!defined($devices_list->{$bridge})) {
4466 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4472 sub qemu_set_link_status {
4473 my ($vmid, $device, $up) = @_;
4475 mon_cmd($vmid, "set_link
", name => $device,
4476 up => $up ? JSON::true : JSON::false);
4479 sub qemu_netdevadd {
4480 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4482 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4483 my %options = split(/[=,]/, $netdev);
4485 mon_cmd($vmid, "netdev_add
", %options);
4489 sub qemu_netdevdel {
4490 my ($vmid, $deviceid) = @_;
4492 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4495 sub qemu_usb_hotplug {
4496 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4500 # remove the old one first
4501 vm_deviceunplug($vmid, $conf, $deviceid);
4503 # check if xhci controller is necessary and available
4504 if ($device->{usb3}) {
4506 my $devicelist = vm_devices_list($vmid);
4508 if (!$devicelist->{xhci}) {
4509 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4510 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4513 my $d = parse_usb_device($device->{host});
4514 $d->{usb3} = $device->{usb3};
4517 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4520 sub qemu_cpu_hotplug {
4521 my ($vmid, $conf, $vcpus) = @_;
4523 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4526 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4527 $sockets = $conf->{sockets} if $conf->{sockets};
4528 my $cores = $conf->{cores} || 1;
4529 my $maxcpus = $sockets * $cores;
4531 $vcpus = $maxcpus if !$vcpus;
4533 die "you can
't add more vcpus than maxcpus\n"
4534 if $vcpus > $maxcpus;
4536 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4538 if ($vcpus < $currentvcpus) {
4540 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4542 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4543 qemu_devicedel($vmid, "cpu$i");
4545 my $currentrunningvcpus = undef;
4547 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4548 last if scalar(@{$currentrunningvcpus}) == $i-1;
4549 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4553 #update conf after each succesfull cpu unplug
4554 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4555 PVE::QemuConfig->write_config($vmid, $conf);
4558 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4564 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4565 die "vcpus in running vm does not match its configuration\n"
4566 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4568 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4570 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4571 my $cpustr = print_cpu_device($conf, $i);
4572 qemu_deviceadd($vmid, $cpustr);
4575 my $currentrunningvcpus = undef;
4577 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4578 last if scalar(@{$currentrunningvcpus}) == $i;
4579 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4583 #update conf after each succesfull cpu hotplug
4584 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4585 PVE::QemuConfig->write_config($vmid, $conf);
4589 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4590 mon_cmd($vmid, "cpu-add", id => int($i));
4595 sub qemu_block_set_io_throttle {
4596 my ($vmid, $deviceid,
4597 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4598 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4599 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4600 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4602 return if !check_running($vmid) ;
4604 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4606 bps_rd => int($bps_rd),
4607 bps_wr => int($bps_wr),
4609 iops_rd => int($iops_rd),
4610 iops_wr => int($iops_wr),
4611 bps_max => int($bps_max),
4612 bps_rd_max => int($bps_rd_max),
4613 bps_wr_max => int($bps_wr_max),
4614 iops_max => int($iops_max),
4615 iops_rd_max => int($iops_rd_max),
4616 iops_wr_max => int($iops_wr_max),
4617 bps_max_length => int($bps_max_length),
4618 bps_rd_max_length => int($bps_rd_max_length),
4619 bps_wr_max_length => int($bps_wr_max_length),
4620 iops_max_length => int($iops_max_length),
4621 iops_rd_max_length => int($iops_rd_max_length),
4622 iops_wr_max_length => int($iops_wr_max_length),
4627 # old code, only used to shutdown old VM after update
4629 my ($fh, $timeout) = @_;
4631 my $sel = new IO::Select;
4638 while (scalar (@ready = $sel->can_read($timeout))) {
4640 if ($count = $fh->sysread($buf, 8192)) {
4641 if ($buf =~ /^(.*)\(qemu\) $/s) {
4648 if (!defined($count)) {
4655 die "monitor read timeout\n" if !scalar(@ready);
4660 sub qemu_block_resize {
4661 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4663 my $running = check_running($vmid);
4665 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4667 return if !$running;
4669 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4673 sub qemu_volume_snapshot {
4674 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4676 my $running = check_running($vmid);
4678 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4679 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4681 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4685 sub qemu_volume_snapshot_delete {
4686 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4688 my $running = check_running($vmid);
4693 my $conf = PVE::QemuConfig->load_config($vmid);
4694 foreach_drive($conf, sub {
4695 my ($ds, $drive) = @_;
4696 $running = 1 if $drive->{file} eq $volid;
4700 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4701 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4703 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4707 sub set_migration_caps {
4713 "auto-converge" => 1,
4715 "x-rdma-pin-all" => 0,
4720 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4722 for my $supported_capability (@$supported_capabilities) {
4724 capability => $supported_capability->{capability},
4725 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4729 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4732 my $fast_plug_option = {
4740 'vmstatestorage
' => 1,
4745 # hotplug changes in [PENDING]
4746 # $selection hash can be used to only apply specified options, for
4747 # example: { cores => 1 } (only apply changed 'cores
')
4748 # $errors ref is used to return error messages
4749 sub vmconfig_hotplug_pending {
4750 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4752 my $defaults = load_defaults();
4753 my $arch = get_vm_arch($conf);
4754 my $machine_type = get_vm_machine($conf, undef, $arch);
4756 # commit values which do not have any impact on running VM first
4757 # Note: those option cannot raise errors, we we do not care about
4758 # $selection and always apply them.
4760 my $add_error = sub {
4761 my ($opt, $msg) = @_;
4762 $errors->{$opt} = "hotplug problem - $msg";
4766 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4767 if ($fast_plug_option->{$opt}) {
4768 $conf->{$opt} = $conf->{pending}->{$opt};
4769 delete $conf->{pending}->{$opt};
4775 PVE::QemuConfig->write_config($vmid, $conf);
4776 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4779 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4781 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4782 foreach my $opt (sort keys %$pending_delete_hash) {
4783 next if $selection && !$selection->{$opt};
4784 my $force = $pending_delete_hash->{$opt}->{force};
4786 if ($opt eq 'hotplug
') {
4787 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4788 } elsif ($opt eq 'tablet
') {
4789 die "skip\n" if !$hotplug_features->{usb};
4790 if ($defaults->{tablet}) {
4791 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4792 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4793 if $arch eq 'aarch64
';
4795 vm_deviceunplug($vmid, $conf, 'tablet
');
4796 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4798 } elsif ($opt =~ m/^usb\d+/) {
4800 # since we cannot reliably hot unplug usb devices
4801 # we are disabling it
4802 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4803 vm_deviceunplug($vmid, $conf, $opt);
4804 } elsif ($opt eq 'vcpus
') {
4805 die "skip\n" if !$hotplug_features->{cpu};
4806 qemu_cpu_hotplug($vmid, $conf, undef);
4807 } elsif ($opt eq 'balloon
') {
4808 # enable balloon device is not hotpluggable
4809 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4810 # here we reset the ballooning value to memory
4811 my $balloon = $conf->{memory} || $defaults->{memory};
4812 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4813 } elsif ($fast_plug_option->{$opt}) {
4815 } elsif ($opt =~ m/^net(\d+)$/) {
4816 die "skip\n" if !$hotplug_features->{network};
4817 vm_deviceunplug($vmid, $conf, $opt);
4818 } elsif (is_valid_drivename($opt)) {
4819 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4820 vm_deviceunplug($vmid, $conf, $opt);
4821 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4822 } elsif ($opt =~ m/^memory$/) {
4823 die "skip\n" if !$hotplug_features->{memory};
4824 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4825 } elsif ($opt eq 'cpuunits
') {
4826 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4827 } elsif ($opt eq 'cpulimit
') {
4828 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4834 &$add_error($opt, $err) if $err ne "skip\n";
4836 # save new config if hotplug was successful
4837 delete $conf->{$opt};
4838 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4839 PVE::QemuConfig->write_config($vmid, $conf);
4840 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4844 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4845 $apply_pending_cloudinit = sub {
4846 return if $apply_pending_cloudinit_done; # once is enough
4847 $apply_pending_cloudinit_done = 1; # once is enough
4849 my ($key, $value) = @_;
4851 my @cloudinit_opts = keys %$confdesc_cloudinit;
4852 foreach my $opt (keys %{$conf->{pending}}) {
4853 next if !grep { $_ eq $opt } @cloudinit_opts;
4854 $conf->{$opt} = delete $conf->{pending}->{$opt};
4857 my $new_conf = { %$conf };
4858 $new_conf->{$key} = $value;
4859 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4862 foreach my $opt (keys %{$conf->{pending}}) {
4863 next if $selection && !$selection->{$opt};
4864 my $value = $conf->{pending}->{$opt};
4866 if ($opt eq 'hotplug
') {
4867 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4868 } elsif ($opt eq 'tablet
') {
4869 die "skip\n" if !$hotplug_features->{usb};
4871 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4872 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4873 if $arch eq 'aarch64
';
4874 } elsif ($value == 0) {
4875 vm_deviceunplug($vmid, $conf, 'tablet
');
4876 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4878 } elsif ($opt =~ m/^usb\d+$/) {
4880 # since we cannot reliably hot unplug usb devices
4881 # we are disabling it
4882 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4883 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4884 die "skip\n" if !$d;
4885 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4886 } elsif ($opt eq 'vcpus
') {
4887 die "skip\n" if !$hotplug_features->{cpu};
4888 qemu_cpu_hotplug($vmid, $conf, $value);
4889 } elsif ($opt eq 'balloon
') {
4890 # enable/disable balloning device is not hotpluggable
4891 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4892 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4893 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4895 # allow manual ballooning if shares is set to zero
4896 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4897 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4898 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4900 } elsif ($opt =~ m/^net(\d+)$/) {
4901 # some changes can be done without hotplug
4902 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4903 $vmid, $opt, $value, $arch, $machine_type);
4904 } elsif (is_valid_drivename($opt)) {
4905 # some changes can be done without hotplug
4906 my $drive = parse_drive($opt, $value);
4907 if (drive_is_cloudinit($drive)) {
4908 &$apply_pending_cloudinit($opt, $value);
4910 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4911 $vmid, $opt, $value, 1, $arch, $machine_type);
4912 } elsif ($opt =~ m/^memory$/) { #dimms
4913 die "skip\n" if !$hotplug_features->{memory};
4914 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4915 } elsif ($opt eq 'cpuunits
') {
4916 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4917 } elsif ($opt eq 'cpulimit
') {
4918 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4919 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4921 die "skip\n"; # skip non-hot-pluggable options
4925 &$add_error($opt, $err) if $err ne "skip\n";
4927 # save new config if hotplug was successful
4928 $conf->{$opt} = $value;
4929 delete $conf->{pending}->{$opt};
4930 PVE::QemuConfig->write_config($vmid, $conf);
4931 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4936 sub try_deallocate_drive {
4937 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4939 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4940 my $volid = $drive->{file};
4941 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4942 my $sid = PVE::Storage::parse_volume_id($volid);
4943 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4945 # check if the disk is really unused
4946 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4947 if is_volume_in_use($storecfg, $conf, $key, $volid);
4948 PVE::Storage::vdisk_free($storecfg, $volid);
4951 # If vm is not owner of this disk remove from config
4959 sub vmconfig_delete_or_detach_drive {
4960 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4962 my $drive = parse_drive($opt, $conf->{$opt});
4964 my $rpcenv = PVE::RPCEnvironment::get();
4965 my $authuser = $rpcenv->get_user();
4968 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4969 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4971 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4977 sub vmconfig_apply_pending {
4978 my ($vmid, $conf, $storecfg) = @_;
4982 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4983 foreach my $opt (sort keys %$pending_delete_hash) {
4984 die "internal error" if $opt =~ m/^unused/;
4985 my $force = $pending_delete_hash->{$opt}->{force};
4986 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4987 if (!defined($conf->{$opt})) {
4988 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4989 PVE::QemuConfig->write_config($vmid, $conf);
4990 } elsif (is_valid_drivename($opt)) {
4991 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4992 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4993 delete $conf->{$opt};
4994 PVE::QemuConfig->write_config($vmid, $conf);
4996 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4997 delete $conf->{$opt};
4998 PVE::QemuConfig->write_config($vmid, $conf);
5002 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5004 foreach my $opt (keys %{$conf->{pending}}) { # add/change
5005 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5007 if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5008 # skip if nothing changed
5009 } elsif (is_valid_drivename($opt)) {
5010 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5011 if defined($conf->{$opt});
5012 $conf->{$opt} = $conf->{pending}->{$opt};
5014 $conf->{$opt} = $conf->{pending}->{$opt};
5017 delete $conf->{pending}->{$opt};
5018 PVE::QemuConfig->write_config($vmid, $conf);
5022 my $safe_num_ne = sub {
5025 return 0 if !defined($a) && !defined($b);
5026 return 1 if !defined($a);
5027 return 1 if !defined($b);
5032 my $safe_string_ne = sub {
5035 return 0 if !defined($a) && !defined($b);
5036 return 1 if !defined($a);
5037 return 1 if !defined($b);
5042 sub vmconfig_update_net {
5043 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5045 my $newnet = parse_net($value);
5047 if ($conf->{$opt}) {
5048 my $oldnet = parse_net($conf->{$opt});
5050 if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5051 &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5052 &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5053 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5055 # for non online change, we try to hot-unplug
5056 die "skip\n" if !$hotplug;
5057 vm_deviceunplug($vmid, $conf, $opt);
5060 die "internal error" if $opt !~ m/net(\d+)/;
5061 my $iface = "tap${vmid}i$1";
5063 if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5064 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5065 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5066 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5067 PVE::Network::tap_unplug($iface);
5068 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5069 } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5070 # Rate can be applied on its own but any change above needs to
5071 # include the rate in tap_plug since OVS resets everything.
5072 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5075 if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5076 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5084 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5090 sub vmconfig_update_disk {
5091 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5093 # fixme: do we need force?
5095 my $drive = parse_drive($opt, $value);
5097 if ($conf->{$opt}) {
5099 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
5101 my $media = $drive->{media} || 'disk
';
5102 my $oldmedia = $old_drive->{media} || 'disk
';
5103 die "unable to change media type\n" if $media ne $oldmedia;
5105 if (!drive_is_cdrom($old_drive)) {
5107 if ($drive->{file} ne $old_drive->{file}) {
5109 die "skip\n" if !$hotplug;
5111 # unplug and register as unused
5112 vm_deviceunplug($vmid, $conf, $opt);
5113 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5116 # update existing disk
5118 # skip non hotpluggable value
5119 if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5120 &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5121 &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5122 &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5127 if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5128 &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5129 &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5130 &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5131 &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5132 &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5133 &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5134 &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5135 &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5136 &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5137 &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5138 &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5139 &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5140 &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5141 &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5142 &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5143 &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5144 &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5146 qemu_block_set_io_throttle($vmid,"drive-$opt",
5147 ($drive->{mbps} || 0)*1024*1024,
5148 ($drive->{mbps_rd} || 0)*1024*1024,
5149 ($drive->{mbps_wr} || 0)*1024*1024,
5150 $drive->{iops} || 0,
5151 $drive->{iops_rd} || 0,
5152 $drive->{iops_wr} || 0,
5153 ($drive->{mbps_max} || 0)*1024*1024,
5154 ($drive->{mbps_rd_max} || 0)*1024*1024,
5155 ($drive->{mbps_wr_max} || 0)*1024*1024,
5156 $drive->{iops_max} || 0,
5157 $drive->{iops_rd_max} || 0,
5158 $drive->{iops_wr_max} || 0,
5159 $drive->{bps_max_length} || 1,
5160 $drive->{bps_rd_max_length} || 1,
5161 $drive->{bps_wr_max_length} || 1,
5162 $drive->{iops_max_length} || 1,
5163 $drive->{iops_rd_max_length} || 1,
5164 $drive->{iops_wr_max_length} || 1);
5173 if ($drive->{file} eq 'none
') {
5174 mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5175 if (drive_is_cloudinit($old_drive)) {
5176 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5179 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5180 mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5181 mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5189 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5191 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5192 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5196 my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5197 $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5199 PVE::QemuConfig->lock_config($vmid, sub {
5200 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5202 die "you can't start a vm
if it
's a template\n" if PVE::QemuConfig->is_template($conf);
5204 my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended
');
5206 PVE::QemuConfig->check_lock($conf)
5207 if !($skiplock || $is_suspended);
5209 die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5211 # clean up leftover reboot request files
5212 eval { clear_reboot_request($vmid); };
5215 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5216 vmconfig_apply_pending($vmid, $conf, $storecfg);
5217 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5220 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5222 my $defaults = load_defaults();
5224 # set environment variable useful inside network script
5225 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5227 my $local_volumes = {};
5229 if ($targetstorage) {
5230 foreach_drive($conf, sub {
5231 my ($ds, $drive) = @_;
5233 return if drive_is_cdrom($drive);
5235 my $volid = $drive->{file};
5239 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5241 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5242 return if $scfg->{shared};
5243 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5248 foreach my $opt (sort keys %$local_volumes) {
5250 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5251 my $drive = parse_drive($opt, $conf->{$opt});
5253 #if remote storage is specified, use default format
5254 if ($targetstorage && $targetstorage ne "1") {
5255 $storeid = $targetstorage;
5256 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5257 $format = $defFormat;
5259 #else we use same format than original
5260 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5261 $format = qemu_img_format($scfg, $volid);
5264 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5265 my $newdrive = $drive;
5266 $newdrive->{format} = $format;
5267 $newdrive->{file} = $newvolid;
5268 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5269 $local_volumes->{$opt} = $drivestr;
5270 #pass drive to conf for command line
5271 $conf->{$opt} = $drivestr;
5275 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
5277 if ($is_suspended) {
5278 # enforce machine type on suspended vm to ensure HW compatibility
5279 $forcemachine = $conf->{runningmachine};
5280 print "Resuming suspended VM\n";
5283 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5286 my $get_migration_ip = sub {
5287 my ($cidr, $nodename) = @_;
5289 return $migration_ip if defined($migration_ip);
5291 if (!defined($cidr)) {
5292 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5293 $cidr = $dc_conf->{migration}->{network};
5296 if (defined($cidr)) {
5297 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5299 die "could not get IP: no address configured on local " .
5300 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5302 die "could not get IP: multiple addresses configured on local " .
5303 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5305 $migration_ip = @$ips[0];
5308 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5309 if !defined($migration_ip);
5311 return $migration_ip;
5316 if ($statefile eq 'tcp
') {
5317 my $localip = "localhost";
5318 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5319 my $nodename = PVE::INotify::nodename();
5321 if (!defined($migration_type)) {
5322 if (defined($datacenterconf->{migration}->{type})) {
5323 $migration_type = $datacenterconf->{migration}->{type};
5325 $migration_type = 'secure
';
5329 if ($migration_type eq 'insecure
') {
5330 $localip = $get_migration_ip->($migration_network, $nodename);
5331 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5334 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5335 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5336 $migrate_uri = "tcp:${localip}:${migrate_port}";
5337 push @$cmd, '-incoming
', $migrate_uri;
5340 } elsif ($statefile eq 'unix
') {
5341 # should be default for secure migrations as a ssh TCP forward
5342 # tunnel is not deterministic reliable ready and fails regurarly
5343 # to set up in time, so use UNIX socket forwards
5344 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5345 unlink $socket_addr;
5347 $migrate_uri = "unix:$socket_addr";
5349 push @$cmd, '-incoming
', $migrate_uri;
5352 } elsif (-e $statefile) {
5353 push @$cmd, '-loadstate
', $statefile;
5355 my $statepath = PVE::Storage::path($storecfg, $statefile);
5356 push @$vollist, $statefile;
5357 push @$cmd, '-loadstate
', $statepath;
5364 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
5365 my $d = parse_hostpci($conf->{"hostpci$i"});
5367 my $pcidevices = $d->{pciid};
5368 foreach my $pcidevice (@$pcidevices) {
5369 my $pciid = $pcidevice->{id};
5370 $pciid = "0000:$pciid" if $pciid !~ m/^[0-9a-f]{4}:/;
5372 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5373 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5374 die "no pci device info for device '$pciid'\n" if !$info;
5377 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5378 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5380 die "can't unbind
/bind pci group to vfio
'$pciid'\n"
5381 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5382 die "can
't reset pci device '$pciid'\n"
5383 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5388 PVE::Storage::activate_volumes($storecfg, $vollist);
5391 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5392 outfunc => sub {}, errfunc => sub {});
5394 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5395 # timeout should be more than enough here...
5396 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5398 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5399 : $defaults->{cpuunits};
5401 my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5402 my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5405 Slice => 'qemu
.slice
',
5407 CPUShares => $cpuunits
5410 if (my $cpulimit = $conf->{cpulimit}) {
5411 $properties{CPUQuota} = int($cpulimit * 100);
5413 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5415 my $run_qemu = sub {
5416 PVE::Tools::run_fork sub {
5417 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5418 run_command($cmd, %run_params);
5422 if ($conf->{hugepages}) {
5425 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5426 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5428 PVE::QemuServer::Memory::hugepages_mount();
5429 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5431 eval { $run_qemu->() };
5433 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5437 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5439 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5442 eval { $run_qemu->() };
5446 # deactivate volumes if start fails
5447 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5448 die "start failed: $err";
5451 print "migration listens on $migrate_uri\n" if $migrate_uri;
5453 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5454 eval { mon_cmd($vmid, "cont"); };
5458 #start nbd server for storage migration
5459 if ($targetstorage) {
5460 my $nodename = PVE::INotify::nodename();
5461 my $localip = $get_migration_ip->($migration_network, $nodename);
5462 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5463 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5465 mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet
', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5467 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5469 foreach my $opt (sort keys %$local_volumes) {
5470 my $volid = $local_volumes->{$opt};
5471 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5472 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5473 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5477 if ($migratedfrom) {
5479 set_migration_caps($vmid);
5484 print "spice listens on port $spice_port\n";
5485 if ($spice_ticket) {
5486 mon_cmd($vmid, "set_password", protocol => 'spice
', password => $spice_ticket);
5487 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5492 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5493 if !$statefile && $conf->{balloon};
5495 foreach my $opt (keys %$conf) {
5496 next if $opt !~ m/^net\d+$/;
5497 my $nicconf = parse_net($conf->{$opt});
5498 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5502 mon_cmd($vmid, 'qom-set
',
5503 path => "machine/peripheral/balloon0",
5504 property => "guest-stats-polling-interval",
5505 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5507 if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5508 print "Resumed VM, removing state\n";
5509 delete $conf->@{qw(lock vmstate runningmachine)};
5510 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5511 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5512 PVE
::QemuConfig-
>write_config($vmid, $conf);
5515 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5519 sub vm_commandline
{
5520 my ($storecfg, $vmid, $snapname) = @_;
5522 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5526 my $snapshot = $conf->{snapshots
}->{$snapname};
5527 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5529 # check for a 'runningmachine' in snapshot
5530 $forcemachine = $snapshot->{runningmachine
} if $snapshot->{runningmachine
};
5532 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5537 my $defaults = load_defaults
();
5539 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine);
5541 return PVE
::Tools
::cmd2string
($cmd);
5545 my ($vmid, $skiplock) = @_;
5547 PVE
::QemuConfig-
>lock_config($vmid, sub {
5549 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5551 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5553 mon_cmd
($vmid, "system_reset");
5557 sub get_vm_volumes
{
5561 foreach_volid
($conf, sub {
5562 my ($volid, $attr) = @_;
5564 return if $volid =~ m
|^/|;
5566 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5569 push @$vollist, $volid;
5575 sub vm_stop_cleanup
{
5576 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5581 my $vollist = get_vm_volumes
($conf);
5582 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5585 foreach my $ext (qw(mon qmp pid vnc qga)) {
5586 unlink "/var/run/qemu-server/${vmid}.$ext";
5589 if ($conf->{ivshmem
}) {
5590 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5591 # just delete it for now, VMs which have this already open do not
5592 # are affected, but new VMs will get a separated one. If this
5593 # becomes an issue we either add some sort of ref-counting or just
5594 # add a "don't delete on stop" flag to the ivshmem format.
5595 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5598 foreach my $key (keys %$conf) {
5599 next if $key !~ m/^hostpci(\d+)$/;
5600 my $hostpciindex = $1;
5601 my $d = parse_hostpci
($conf->{$key});
5602 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5604 foreach my $pci (@{$d->{pciid
}}) {
5605 my $pciid = $pci->{id
};
5606 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5610 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5612 warn $@ if $@; # avoid errors - just warn
5615 # call only in locked context
5617 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5619 my $pid = check_running
($vmid, $nocheck);
5624 $conf = PVE
::QemuConfig-
>load_config($vmid);
5625 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5626 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5627 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5628 $timeout = $opts->{down
} if $opts->{down
};
5630 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5635 if (defined($conf) && parse_guest_agent
($conf)->{enabled
}) {
5636 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5638 mon_cmd
($vmid, "system_powerdown");
5641 mon_cmd
($vmid, "quit");
5647 $timeout = 60 if !defined($timeout);
5650 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5655 if ($count >= $timeout) {
5657 warn "VM still running - terminating now with SIGTERM\n";
5660 die "VM quit/powerdown failed - got timeout\n";
5663 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5668 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5671 die "VM quit/powerdown failed\n";
5679 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5684 if ($count >= $timeout) {
5685 warn "VM still running - terminating now with SIGKILL\n";
5690 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5693 # Note: use $nocheck to skip tests if VM configuration file exists.
5694 # We need that when migration VMs to other nodes (files already moved)
5695 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5697 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5699 $force = 1 if !defined($force) && !$shutdown;
5702 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5703 kill 15, $pid if $pid;
5704 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5705 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5709 PVE
::QemuConfig-
>lock_config($vmid, sub {
5710 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5715 my ($vmid, $timeout) = @_;
5717 PVE
::QemuConfig-
>lock_config($vmid, sub {
5720 # only reboot if running, as qmeventd starts it again on a stop event
5721 return if !check_running
($vmid);
5723 create_reboot_request
($vmid);
5725 my $storecfg = PVE
::Storage
::config
();
5726 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5730 # avoid that the next normal shutdown will be confused for a reboot
5731 clear_reboot_request
($vmid);
5738 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5745 PVE
::QemuConfig-
>lock_config($vmid, sub {
5747 $conf = PVE
::QemuConfig-
>load_config($vmid);
5749 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5750 PVE
::QemuConfig-
>check_lock($conf)
5751 if !($skiplock || $is_backing_up);
5753 die "cannot suspend to disk during backup\n"
5754 if $is_backing_up && $includestate;
5756 if ($includestate) {
5757 $conf->{lock} = 'suspending';
5758 my $date = strftime
("%Y-%m-%d", localtime(time()));
5759 $storecfg = PVE
::Storage
::config
();
5760 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5761 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5762 PVE
::QemuConfig-
>write_config($vmid, $conf);
5764 mon_cmd
($vmid, "stop");
5768 if ($includestate) {
5770 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5773 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5775 my $state = mon_cmd
($vmid, "query-savevm");
5776 if (!$state->{status
}) {
5777 die "savevm not active\n";
5778 } elsif ($state->{status
} eq 'active') {
5781 } elsif ($state->{status
} eq 'completed') {
5782 print "State saved, quitting\n";
5784 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5785 die "query-savevm failed with error '$state->{error}'\n"
5787 die "query-savevm returned status '$state->{status}'\n";
5793 PVE
::QemuConfig-
>lock_config($vmid, sub {
5794 $conf = PVE
::QemuConfig-
>load_config($vmid);
5796 # cleanup, but leave suspending lock, to indicate something went wrong
5798 mon_cmd
($vmid, "savevm-end");
5799 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5800 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5801 delete $conf->@{qw(vmstate runningmachine)};
5802 PVE
::QemuConfig-
>write_config($vmid, $conf);
5808 die "lock changed unexpectedly\n"
5809 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5811 mon_cmd
($vmid, "quit");
5812 $conf->{lock} = 'suspended';
5813 PVE
::QemuConfig-
>write_config($vmid, $conf);
5819 my ($vmid, $skiplock, $nocheck) = @_;
5821 PVE
::QemuConfig-
>lock_config($vmid, sub {
5822 my $res = mon_cmd
($vmid, 'query-status');
5823 my $resume_cmd = 'cont';
5825 if ($res->{status
} && $res->{status
} eq 'suspended') {
5826 $resume_cmd = 'system_wakeup';
5831 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5833 PVE
::QemuConfig-
>check_lock($conf)
5834 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5837 mon_cmd
($vmid, $resume_cmd);
5842 my ($vmid, $skiplock, $key) = @_;
5844 PVE
::QemuConfig-
>lock_config($vmid, sub {
5846 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5848 # there is no qmp command, so we use the human monitor command
5849 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5850 die $res if $res ne '';
5854 # vzdump restore implementaion
5856 sub tar_archive_read_firstfile
{
5857 my $archive = shift;
5859 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5861 # try to detect archive type first
5862 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5863 die "unable to open file '$archive'\n";
5864 my $firstfile = <$fh>;
5868 die "ERROR: archive contaions no data\n" if !$firstfile;
5874 sub tar_restore_cleanup
{
5875 my ($storecfg, $statfile) = @_;
5877 print STDERR
"starting cleanup\n";
5879 if (my $fd = IO
::File-
>new($statfile, "r")) {
5880 while (defined(my $line = <$fd>)) {
5881 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5884 if ($volid =~ m
|^/|) {
5885 unlink $volid || die 'unlink failed\n';
5887 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5889 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5891 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5893 print STDERR
"unable to parse line in statfile - $line";
5900 sub restore_archive
{
5901 my ($archive, $vmid, $user, $opts) = @_;
5903 my $format = $opts->{format
};
5906 if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5907 $format = 'tar' if !$format;
5909 } elsif ($archive =~ m/\.tar$/) {
5910 $format = 'tar' if !$format;
5911 } elsif ($archive =~ m/.tar.lzo$/) {
5912 $format = 'tar' if !$format;
5914 } elsif ($archive =~ m/\.vma$/) {
5915 $format = 'vma' if !$format;
5916 } elsif ($archive =~ m/\.vma\.gz$/) {
5917 $format = 'vma' if !$format;
5919 } elsif ($archive =~ m/\.vma\.lzo$/) {
5920 $format = 'vma' if !$format;
5923 $format = 'vma' if !$format; # default
5926 # try to detect archive format
5927 if ($format eq 'tar') {
5928 return restore_tar_archive
($archive, $vmid, $user, $opts);
5930 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5934 sub restore_update_config_line
{
5935 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5937 return if $line =~ m/^\#qmdump\#/;
5938 return if $line =~ m/^\#vzdump\#/;
5939 return if $line =~ m/^lock:/;
5940 return if $line =~ m/^unused\d+:/;
5941 return if $line =~ m/^parent:/;
5943 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5944 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5945 # try to convert old 1.X settings
5946 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5947 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5948 my ($model, $macaddr) = split(/\=/, $devconfig);
5949 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5952 bridge
=> "vmbr$ind",
5953 macaddr
=> $macaddr,
5955 my $netstr = print_net
($net);
5957 print $outfd "net$cookie->{netcount}: $netstr\n";
5958 $cookie->{netcount
}++;
5960 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5961 my ($id, $netstr) = ($1, $2);
5962 my $net = parse_net
($netstr);
5963 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5964 $netstr = print_net
($net);
5965 print $outfd "$id: $netstr\n";
5966 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5969 my $di = parse_drive
($virtdev, $value);
5970 if (defined($di->{backup
}) && !$di->{backup
}) {
5971 print $outfd "#$line";
5972 } elsif ($map->{$virtdev}) {
5973 delete $di->{format
}; # format can change on restore
5974 $di->{file
} = $map->{$virtdev};
5975 $value = print_drive
($vmid, $di);
5976 print $outfd "$virtdev: $value\n";
5980 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5982 if ($vmgenid ne '0') {
5983 # always generate a new vmgenid if there was a valid one setup
5984 $vmgenid = generate_uuid
();
5986 print $outfd "vmgenid: $vmgenid\n";
5987 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5988 my ($uuid, $uuid_str);
5989 UUID
::generate
($uuid);
5990 UUID
::unparse
($uuid, $uuid_str);
5991 my $smbios1 = parse_smbios1
($2);
5992 $smbios1->{uuid
} = $uuid_str;
5993 print $outfd $1.print_smbios1
($smbios1)."\n";
6000 my ($cfg, $vmid) = @_;
6002 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
6004 my $volid_hash = {};
6005 foreach my $storeid (keys %$info) {
6006 foreach my $item (@{$info->{$storeid}}) {
6007 next if !($item->{volid
} && $item->{size
});
6008 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6009 $volid_hash->{$item->{volid
}} = $item;
6016 sub is_volume_in_use
{
6017 my ($storecfg, $conf, $skip_drive, $volid) = @_;
6019 my $path = PVE
::Storage
::path
($storecfg, $volid);
6021 my $scan_config = sub {
6022 my ($cref, $snapname) = @_;
6024 foreach my $key (keys %$cref) {
6025 my $value = $cref->{$key};
6026 if (is_valid_drivename
($key)) {
6027 next if $skip_drive && $key eq $skip_drive;
6028 my $drive = parse_drive
($key, $value);
6029 next if !$drive || !$drive->{file
} || drive_is_cdrom
($drive);
6030 return 1 if $volid eq $drive->{file
};
6031 if ($drive->{file
} =~ m!^/!) {
6032 return 1 if $drive->{file
} eq $path;
6034 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
}, 1);
6036 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid, 1);
6038 return 1 if $path eq PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
6046 return 1 if &$scan_config($conf);
6050 foreach my $snapname (keys %{$conf->{snapshots
}}) {
6051 return 1 if &$scan_config($conf->{snapshots
}->{$snapname}, $snapname);
6057 sub update_disksize
{
6058 my ($vmid, $conf, $volid_hash) = @_;
6061 my $prefix = "VM $vmid:";
6063 # used and unused disks
6064 my $referenced = {};
6066 # Note: it is allowed to define multiple storages with same path (alias), so
6067 # we need to check both 'volid' and real 'path' (two different volid can point
6068 # to the same path).
6070 my $referencedpath = {};
6073 foreach my $opt (keys %$conf) {
6074 if (is_valid_drivename
($opt)) {
6075 my $drive = parse_drive
($opt, $conf->{$opt});
6076 my $volid = $drive->{file
};
6079 $referenced->{$volid} = 1;
6080 if ($volid_hash->{$volid} &&
6081 (my $path = $volid_hash->{$volid}->{path
})) {
6082 $referencedpath->{$path} = 1;
6085 next if drive_is_cdrom
($drive);
6086 next if !$volid_hash->{$volid};
6088 $drive->{size
} = $volid_hash->{$volid}->{size
};
6089 my $new = print_drive
($vmid, $drive);
6090 if ($new ne $conf->{$opt}) {
6092 $conf->{$opt} = $new;
6093 print "$prefix update disk '$opt' information.\n";
6098 # remove 'unusedX' entry if volume is used
6099 foreach my $opt (keys %$conf) {
6100 next if $opt !~ m/^unused\d+$/;
6101 my $volid = $conf->{$opt};
6102 my $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6103 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6104 print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6106 delete $conf->{$opt};
6109 $referenced->{$volid} = 1;
6110 $referencedpath->{$path} = 1 if $path;
6113 foreach my $volid (sort keys %$volid_hash) {
6114 next if $volid =~ m/vm-$vmid-state-/;
6115 next if $referenced->{$volid};
6116 my $path = $volid_hash->{$volid}->{path
};
6117 next if !$path; # just to be sure
6118 next if $referencedpath->{$path};
6120 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6121 print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6122 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6129 my ($vmid, $nolock, $dryrun) = @_;
6131 my $cfg = PVE
::Storage
::config
();
6133 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6134 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6135 foreach my $stor (keys %{$cfg->{ids
}}) {
6136 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
6139 print "rescan volumes...\n";
6140 my $volid_hash = scan_volids
($cfg, $vmid);
6142 my $updatefn = sub {
6145 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6147 PVE
::QemuConfig-
>check_lock($conf);
6150 foreach my $volid (keys %$volid_hash) {
6151 my $info = $volid_hash->{$volid};
6152 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6155 my $changes = update_disksize
($vmid, $conf, $vm_volids);
6157 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6160 if (defined($vmid)) {
6164 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6167 my $vmlist = config_list
();
6168 foreach my $vmid (keys %$vmlist) {
6172 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6178 sub restore_vma_archive
{
6179 my ($archive, $vmid, $user, $opts, $comp) = @_;
6181 my $readfrom = $archive;
6183 my $cfg = PVE
::Storage
::config
();
6185 my $bwlimit = $opts->{bwlimit
};
6187 my $dbg_cmdstring = '';
6188 my $add_pipe = sub {
6190 push @$commands, $cmd;
6191 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6192 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6197 if ($archive eq '-') {
6200 # If we use a backup from a PVE defined storage we also consider that
6201 # storage's rate limit:
6202 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6203 if (defined($volid)) {
6204 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6205 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6207 print STDERR
"applying read rate limit: $readlimit\n";
6208 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6209 $add_pipe->($cstream);
6216 if ($comp eq 'gzip') {
6217 $cmd = ['zcat', $readfrom];
6218 } elsif ($comp eq 'lzop') {
6219 $cmd = ['lzop', '-d', '-c', $readfrom];
6221 die "unknown compression method '$comp'\n";
6226 my $tmpdir = "/var/tmp/vzdumptmp$$";
6229 # disable interrupts (always do cleanups)
6233 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6235 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6236 POSIX
::mkfifo
($mapfifo, 0600);
6239 my $openfifo = sub {
6240 open($fifofh, '>', $mapfifo) || die $!;
6243 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6250 my $rpcenv = PVE
::RPCEnvironment
::get
();
6252 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6253 my $tmpfn = "$conffile.$$.tmp";
6255 # Note: $oldconf is undef if VM does not exists
6256 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6257 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6261 my $print_devmap = sub {
6262 my $virtdev_hash = {};
6264 my $cfgfn = "$tmpdir/qemu-server.conf";
6266 # we can read the config - that is already extracted
6267 my $fh = IO
::File-
>new($cfgfn, "r") ||
6268 "unable to read qemu-server.conf - $!\n";
6270 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6272 my $pve_firewall_dir = '/etc/pve/firewall';
6273 mkdir $pve_firewall_dir; # make sure the dir exists
6274 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6277 while (defined(my $line = <$fh>)) {
6278 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6279 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6280 die "archive does not contain data for drive '$virtdev'\n"
6281 if !$devinfo->{$devname};
6282 if (defined($opts->{storage
})) {
6283 $storeid = $opts->{storage
} || 'local';
6284 } elsif (!$storeid) {
6287 $format = 'raw' if !$format;
6288 $devinfo->{$devname}->{devname
} = $devname;
6289 $devinfo->{$devname}->{virtdev
} = $virtdev;
6290 $devinfo->{$devname}->{format
} = $format;
6291 $devinfo->{$devname}->{storeid
} = $storeid;
6293 # check permission on storage
6294 my $pool = $opts->{pool
}; # todo: do we need that?
6295 if ($user ne 'root@pam') {
6296 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6299 $storage_limits{$storeid} = $bwlimit;
6301 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6302 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6304 my $drive = parse_drive
($virtdev, $2);
6305 if (drive_is_cloudinit
($drive)) {
6306 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6307 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
6308 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6312 storeid
=> $opts->{storage
} // $storeid,
6313 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6314 file
=> $drive->{file
}, # to make drive_is_cloudinit check possible
6315 name
=> "vm-$vmid-cloudinit",
6318 $virtdev_hash->{$virtdev} = $d;
6323 foreach my $key (keys %storage_limits) {
6324 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6326 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6327 $storage_limits{$key} = $limit * 1024;
6330 foreach my $devname (keys %$devinfo) {
6331 die "found no device mapping information for device '$devname'\n"
6332 if !$devinfo->{$devname}->{virtdev
};
6335 # create empty/temp config
6337 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6338 foreach_drive
($oldconf, sub {
6339 my ($ds, $drive) = @_;
6341 return if drive_is_cdrom
($drive, 1);
6343 my $volid = $drive->{file
};
6344 return if !$volid || $volid =~ m
|^/|;
6346 my ($path, $owner) = PVE
::Storage
::path
($cfg, $volid);
6347 return if !$path || !$owner || ($owner != $vmid);
6349 # Note: only delete disk we want to restore
6350 # other volumes will become unused
6351 if ($virtdev_hash->{$ds}) {
6352 eval { PVE
::Storage
::vdisk_free
($cfg, $volid); };
6359 # delete vmstate files, after the restore we have no snapshots anymore
6360 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
6361 my $snap = $oldconf->{snapshots
}->{$snapname};
6362 if ($snap->{vmstate
}) {
6363 eval { PVE
::Storage
::vdisk_free
($cfg, $snap->{vmstate
}); };
6372 foreach my $virtdev (sort keys %$virtdev_hash) {
6373 my $d = $virtdev_hash->{$virtdev};
6374 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6375 my $storeid = $d->{storeid
};
6376 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
6379 if (my $limit = $storage_limits{$storeid}) {
6380 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6383 # test if requested format is supported
6384 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($cfg, $storeid);
6385 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6386 $d->{format
} = $defFormat if !$supported;
6389 if ($d->{is_cloudinit
}) {
6391 $name .= ".$d->{format}" if $d->{format
} ne 'raw';
6394 my $volid = PVE
::Storage
::vdisk_alloc
($cfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6395 print STDERR
"new volume ID is '$volid'\n";
6396 $d->{volid
} = $volid;
6398 PVE
::Storage
::activate_volumes
($cfg, [$volid]);
6400 my $write_zeros = 1;
6401 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6405 if (!$d->{is_cloudinit
}) {
6406 my $path = PVE
::Storage
::path
($cfg, $volid);
6408 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6410 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6412 $map->{$virtdev} = $volid;
6415 $fh->seek(0, 0) || die "seek failed - $!\n";
6417 my $outfd = new IO
::File
($tmpfn, "w") ||
6418 die "unable to write config for VM $vmid\n";
6420 my $cookie = { netcount
=> 0 };
6421 while (defined(my $line = <$fh>)) {
6422 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6435 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6436 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6438 $oldtimeout = alarm($timeout);
6445 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6446 my ($dev_id, $size, $devname) = ($1, $2, $3);
6447 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6448 } elsif ($line =~ m/^CTIME: /) {
6449 # we correctly received the vma config, so we can disable
6450 # the timeout now for disk allocation (set to 10 minutes, so
6451 # that we always timeout if something goes wrong)
6454 print $fifofh "done\n";
6455 my $tmp = $oldtimeout || 0;
6456 $oldtimeout = undef;
6462 print "restore vma archive: $dbg_cmdstring\n";
6463 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6467 alarm($oldtimeout) if $oldtimeout;
6470 foreach my $devname (keys %$devinfo) {
6471 my $volid = $devinfo->{$devname}->{volid
};
6472 push @$vollist, $volid if $volid;
6475 PVE
::Storage
::deactivate_volumes
($cfg, $vollist);
6483 foreach my $devname (keys %$devinfo) {
6484 my $volid = $devinfo->{$devname}->{volid
};
6487 if ($volid =~ m
|^/|) {
6488 unlink $volid || die 'unlink failed\n';
6490 PVE
::Storage
::vdisk_free
($cfg, $volid);
6492 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6494 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6501 rename($tmpfn, $conffile) ||
6502 die "unable to commit configuration file '$conffile'\n";
6504 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6506 eval { rescan
($vmid, 1); };
6510 sub restore_tar_archive
{
6511 my ($archive, $vmid, $user, $opts) = @_;
6513 if ($archive ne '-') {
6514 my $firstfile = tar_archive_read_firstfile
($archive);
6515 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6516 if $firstfile ne 'qemu-server.conf';
6519 my $storecfg = PVE
::Storage
::config
();
6521 # avoid zombie disks when restoring over an existing VM -> cleanup first
6522 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6523 # skiplock=1 because qmrestore has set the 'create' lock itself already
6524 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6525 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6527 my $tocmd = "/usr/lib/qemu-server/qmextract";
6529 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6530 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6531 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6532 $tocmd .= ' --info' if $opts->{info
};
6534 # tar option "xf" does not autodetect compression when read from STDIN,
6535 # so we pipe to zcat
6536 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6537 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6539 my $tmpdir = "/var/tmp/vzdumptmp$$";
6542 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6543 local $ENV{VZDUMP_VMID
} = $vmid;
6544 local $ENV{VZDUMP_USER
} = $user;
6546 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6547 my $tmpfn = "$conffile.$$.tmp";
6549 # disable interrupts (always do cleanups)
6553 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6561 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6563 if ($archive eq '-') {
6564 print "extracting archive from STDIN\n";
6565 run_command
($cmd, input
=> "<&STDIN");
6567 print "extracting archive '$archive'\n";
6571 return if $opts->{info
};
6575 my $statfile = "$tmpdir/qmrestore.stat";
6576 if (my $fd = IO
::File-
>new($statfile, "r")) {
6577 while (defined (my $line = <$fd>)) {
6578 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6579 $map->{$1} = $2 if $1;
6581 print STDERR
"unable to parse line in statfile - $line\n";
6587 my $confsrc = "$tmpdir/qemu-server.conf";
6589 my $srcfd = new IO
::File
($confsrc, "r") ||
6590 die "unable to open file '$confsrc'\n";
6592 my $outfd = new IO
::File
($tmpfn, "w") ||
6593 die "unable to write config for VM $vmid\n";
6595 my $cookie = { netcount
=> 0 };
6596 while (defined (my $line = <$srcfd>)) {
6597 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6605 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6611 rename $tmpfn, $conffile ||
6612 die "unable to commit configuration file '$conffile'\n";
6614 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6616 eval { rescan
($vmid, 1); };
6620 sub foreach_storage_used_by_vm
{
6621 my ($conf, $func) = @_;
6625 foreach_drive
($conf, sub {
6626 my ($ds, $drive) = @_;
6627 return if drive_is_cdrom
($drive);
6629 my $volid = $drive->{file
};
6631 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6632 $sidhash->{$sid} = $sid if $sid;
6635 foreach my $sid (sort keys %$sidhash) {
6640 my $qemu_snap_storage = {
6643 sub do_snapshots_with_qemu
{
6644 my ($storecfg, $volid) = @_;
6646 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6647 my $scfg = $storecfg->{ids
}->{$storage_name};
6649 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6653 if ($volid =~ m/\.(qcow2|qed)$/){
6660 sub qga_check_running
{
6661 my ($vmid, $nowarn) = @_;
6663 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6665 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6671 sub template_create
{
6672 my ($vmid, $conf, $disk) = @_;
6674 my $storecfg = PVE
::Storage
::config
();
6676 foreach_drive
($conf, sub {
6677 my ($ds, $drive) = @_;
6679 return if drive_is_cdrom
($drive);
6680 return if $disk && $ds ne $disk;
6682 my $volid = $drive->{file
};
6683 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6685 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6686 $drive->{file
} = $voliddst;
6687 $conf->{$ds} = print_drive
($vmid, $drive);
6688 PVE
::QemuConfig-
>write_config($vmid, $conf);
6692 sub convert_iscsi_path
{
6695 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6700 my $initiator_name = get_initiator_name
();
6702 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6703 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6706 die "cannot convert iscsi path '$path', unkown format\n";
6709 sub qemu_img_convert
{
6710 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6712 my $storecfg = PVE
::Storage
::config
();
6713 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6714 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6716 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6720 my $src_is_iscsi = 0;
6721 my $src_format = 'raw';
6724 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6725 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6726 $src_format = qemu_img_format
($src_scfg, $src_volname);
6727 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6728 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6729 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6730 } elsif (-f
$src_volid) {
6731 $src_path = $src_volid;
6732 if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6737 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6739 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6740 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6741 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6742 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6745 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6746 push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6747 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6748 push @$cmd, '-T', $cachemode if defined($cachemode);
6750 if ($src_is_iscsi) {
6751 push @$cmd, '--image-opts';
6752 $src_path = convert_iscsi_path
($src_path);
6754 push @$cmd, '-f', $src_format;
6757 if ($dst_is_iscsi) {
6758 push @$cmd, '--target-image-opts';
6759 $dst_path = convert_iscsi_path
($dst_path);
6761 push @$cmd, '-O', $dst_format;
6764 push @$cmd, $src_path;
6766 if (!$dst_is_iscsi && $is_zero_initialized) {
6767 push @$cmd, "zeroinit:$dst_path";
6769 push @$cmd, $dst_path;
6774 if($line =~ m/\((\S+)\/100\
%\)/){
6776 my $transferred = int($size * $percent / 100);
6777 my $remaining = $size - $transferred;
6779 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6784 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6786 die "copy failed: $err" if $err;
6789 sub qemu_img_format
{
6790 my ($scfg, $volname) = @_;
6792 if ($scfg->{path
} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6799 sub qemu_drive_mirror
{
6800 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6802 $jobs = {} if !$jobs;
6806 $jobs->{"drive-$drive"} = {};
6808 if ($dst_volid =~ /^nbd:/) {
6809 $qemu_target = $dst_volid;
6812 my $storecfg = PVE
::Storage
::config
();
6813 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6815 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6817 $format = qemu_img_format
($dst_scfg, $dst_volname);
6819 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6821 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6824 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6825 $opts->{format
} = $format if $format;
6827 if (defined($bwlimit)) {
6828 $opts->{speed
} = $bwlimit * 1024;
6829 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6831 print "drive mirror is starting for drive-$drive\n";
6834 # if a job already runs for this device we get an error, catch it for cleanup
6835 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
6837 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6839 die "mirroring error: $err\n";
6842 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6845 sub qemu_drive_mirror_monitor
{
6846 my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6849 my $err_complete = 0;
6852 die "storage migration timed out\n" if $err_complete > 300;
6854 my $stats = mon_cmd
($vmid, "query-block-jobs");
6856 my $running_mirror_jobs = {};
6857 foreach my $stat (@$stats) {
6858 next if $stat->{type
} ne 'mirror';
6859 $running_mirror_jobs->{$stat->{device
}} = $stat;
6862 my $readycounter = 0;
6864 foreach my $job (keys %$jobs) {
6866 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6867 print "$job : finished\n";
6868 delete $jobs->{$job};
6872 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6874 my $busy = $running_mirror_jobs->{$job}->{busy
};
6875 my $ready = $running_mirror_jobs->{$job}->{ready
};
6876 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6877 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6878 my $remaining = $total - $transferred;
6879 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6881 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6884 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6887 last if scalar(keys %$jobs) == 0;
6889 if ($readycounter == scalar(keys %$jobs)) {
6890 print "all mirroring jobs are ready \n";
6891 last if $skipcomplete; #do the complete later
6893 if ($vmiddst && $vmiddst != $vmid) {
6894 my $agent_running = $qga && qga_check_running
($vmid);
6895 if ($agent_running) {
6896 print "freeze filesystem\n";
6897 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6899 print "suspend vm\n";
6900 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6903 # if we clone a disk for a new target vm, we don't switch the disk
6904 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6906 if ($agent_running) {
6907 print "unfreeze filesystem\n";
6908 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6910 print "resume vm\n";
6911 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6917 foreach my $job (keys %$jobs) {
6918 # try to switch the disk if source and destination are on the same guest
6919 print "$job: Completing block job...\n";
6921 eval { mon_cmd
($vmid, "block-job-complete", device
=> $job) };
6922 if ($@ =~ m/cannot be completed/) {
6923 print "$job: Block job cannot be completed, try again.\n";
6926 print "$job: Completed successfully.\n";
6927 $jobs->{$job}->{complete
} = 1;
6938 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6939 die "mirroring error: $err";
6944 sub qemu_blockjobs_cancel
{
6945 my ($vmid, $jobs) = @_;
6947 foreach my $job (keys %$jobs) {
6948 print "$job: Cancelling block job\n";
6949 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6950 $jobs->{$job}->{cancel
} = 1;
6954 my $stats = mon_cmd
($vmid, "query-block-jobs");
6956 my $running_jobs = {};
6957 foreach my $stat (@$stats) {
6958 $running_jobs->{$stat->{device
}} = $stat;
6961 foreach my $job (keys %$jobs) {
6963 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6964 print "$job: Done.\n";
6965 delete $jobs->{$job};
6969 last if scalar(keys %$jobs) == 0;
6976 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6977 $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6982 print "create linked clone of drive $drivename ($drive->{file})\n";
6983 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6984 push @$newvollist, $newvolid;
6987 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6988 $storeid = $storage if $storage;
6990 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6991 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
6993 print "create full clone of drive $drivename ($drive->{file})\n";
6995 if (drive_is_cloudinit
($drive)) {
6996 $name = "vm-$newvmid-cloudinit";
6997 $name .= ".$dst_format" if $dst_format ne 'raw';
6999 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
7001 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7002 push @$newvollist, $newvolid;
7004 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
7006 if (drive_is_cloudinit
($drive)) {
7010 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
7011 if (!$running || $snapname) {
7012 # TODO: handle bwlimits
7013 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
7016 my $kvmver = get_running_qemu_version
($vmid);
7017 if (!min_version
($kvmver, 2, 7)) {
7018 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7019 if $drive->{iothread
};
7022 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7027 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
7030 $disk->{format
} = undef;
7031 $disk->{file
} = $newvolid;
7032 $disk->{size
} = $size;
7037 sub get_running_qemu_version
{
7039 my $res = mon_cmd
($vmid, "query-version");
7040 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7043 sub qemu_use_old_bios_files
{
7044 my ($machine_type) = @_;
7046 return if !$machine_type;
7048 my $use_old_bios_files = undef;
7050 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7052 $use_old_bios_files = 1;
7054 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, kvm_user_version
());
7055 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7056 # load new efi bios files on migration. So this hack is required to allow
7057 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7058 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7059 $use_old_bios_files = !min_version
($version, 2, 4);
7062 return ($use_old_bios_files, $machine_type);
7065 sub create_efidisk
($$$$$) {
7066 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7068 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7069 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7071 my $vars_size_b = -s
$ovmf_vars;
7072 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7073 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7074 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7076 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7078 return ($volid, $vars_size);
7081 sub vm_iothreads_list
{
7084 my $res = mon_cmd
($vmid, 'query-iothreads');
7087 foreach my $iothread (@$res) {
7088 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7095 my ($conf, $drive) = @_;
7099 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7101 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7107 my $controller = int($drive->{index} / $maxdev);
7108 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single') ?
"virtioscsi" : "scsihw";
7110 return ($maxdev, $controller, $controller_prefix);
7113 sub add_hyperv_enlightenments
{
7114 my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7116 return if $winversion < 6;
7117 return if $bios && $bios eq 'ovmf' && $winversion < 8;
7119 if ($gpu_passthrough || defined($hv_vendor_id)) {
7120 $hv_vendor_id //= 'proxmox';
7121 push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7124 if (min_version
($machine_version, 2, 3)) {
7125 push @$cpuFlags , 'hv_spinlocks=0x1fff';
7126 push @$cpuFlags , 'hv_vapic';
7127 push @$cpuFlags , 'hv_time';
7129 push @$cpuFlags , 'hv_spinlocks=0xffff';
7132 if (min_version
($machine_version, 2, 6)) {
7133 push @$cpuFlags , 'hv_reset';
7134 push @$cpuFlags , 'hv_vpindex';
7135 push @$cpuFlags , 'hv_runtime';
7138 if ($winversion >= 7) {
7139 push @$cpuFlags , 'hv_relaxed';
7141 if (min_version
($machine_version, 2, 12)) {
7142 push @$cpuFlags , 'hv_synic';
7143 push @$cpuFlags , 'hv_stimer';
7146 if (min_version
($machine_version, 3, 1)) {
7147 push @$cpuFlags , 'hv_ipi';
7152 sub windows_version
{
7155 return 0 if !$ostype;
7159 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7161 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7163 } elsif ($ostype =~ m/^win(\d+)$/) {
7170 sub resolve_dst_disk_format
{
7171 my ($storecfg, $storeid, $src_volname, $format) = @_;
7172 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7175 # if no target format is specified, use the source disk format as hint
7177 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7178 $format = qemu_img_format
($scfg, $src_volname);
7184 # test if requested format is supported - else use default
7185 my $supported = grep { $_ eq $format } @$validFormats;
7186 $format = $defFormat if !$supported;
7190 sub resolve_first_disk
{
7192 my @disks = PVE
::QemuServer
::valid_drive_names
();
7194 foreach my $ds (reverse @disks) {
7195 next if !$conf->{$ds};
7196 my $disk = PVE
::QemuServer
::parse_drive
($ds, $conf->{$ds});
7197 next if PVE
::QemuServer
::drive_is_cdrom
($disk);
7204 my ($uuid, $uuid_str);
7205 UUID
::generate
($uuid);
7206 UUID
::unparse
($uuid, $uuid_str);
7210 sub generate_smbios1_uuid
{
7211 return "uuid=".generate_uuid
();
7217 mon_cmd
($vmid, 'nbd-server-stop');
7220 sub create_reboot_request
{
7222 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7223 or die "failed to create reboot trigger file: $!\n";
7227 sub clear_reboot_request
{
7229 my $path = "/run/qemu-server/$vmid.reboot";
7232 $res = unlink($path);
7233 die "could not remove reboot request for $vmid: $!"
7234 if !$res && $! != POSIX
::ENOENT
;
7239 # bash completion helper
7241 sub complete_backup_archives
{
7242 my ($cmdname, $pname, $cvalue) = @_;
7244 my $cfg = PVE
::Storage
::config
();
7248 if ($cvalue =~ m/^([^:]+):/) {
7252 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7255 foreach my $id (keys %$data) {
7256 foreach my $item (@{$data->{$id}}) {
7257 next if $item->{format
} !~ m/^vma\.(gz|lzo)$/;
7258 push @$res, $item->{volid
} if defined($item->{volid
});
7265 my $complete_vmid_full = sub {
7268 my $idlist = vmstatus
();
7272 foreach my $id (keys %$idlist) {
7273 my $d = $idlist->{$id};
7274 if (defined($running)) {
7275 next if $d->{template
};
7276 next if $running && $d->{status
} ne 'running';
7277 next if !$running && $d->{status
} eq 'running';
7286 return &$complete_vmid_full();
7289 sub complete_vmid_stopped
{
7290 return &$complete_vmid_full(0);
7293 sub complete_vmid_running
{
7294 return &$complete_vmid_full(1);
7297 sub complete_storage
{
7299 my $cfg = PVE
::Storage
::config
();
7300 my $ids = $cfg->{ids
};
7303 foreach my $sid (keys %$ids) {
7304 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7305 next if !$ids->{$sid}->{content
}->{images
};
7312 sub complete_migration_storage
{
7313 my ($cmd, $param, $current_value, $all_args) = @_;
7315 my $targetnode = @$all_args[1];
7317 my $cfg = PVE
::Storage
::config
();
7318 my $ids = $cfg->{ids
};
7321 foreach my $sid (keys %$ids) {
7322 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7323 next if !$ids->{$sid}->{content
}->{images
};