1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
30 use PVE
::DataCenterConfig
;
31 use PVE
::Exception
qw(raise raise_param_exc);
32 use PVE
::GuestHelpers
;
34 use PVE
::JSONSchema
qw(get_standard_option);
36 use PVE
::RPCEnvironment
;
40 use PVE
::Tools
qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
44 use PVE
::QemuServer
::Helpers
qw(min_version);
45 use PVE
::QemuServer
::Cloudinit
;
46 use PVE
::QemuServer
::Machine
;
47 use PVE
::QemuServer
::Memory
;
48 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
49 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port);
50 use PVE
::QemuServer
::USB
qw(parse_usb_device);
52 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
55 "$EDK2_FW_BASE/OVMF_CODE.fd",
56 "$EDK2_FW_BASE/OVMF_VARS.fd"
59 "$EDK2_FW_BASE/AAVMF_CODE.fd",
60 "$EDK2_FW_BASE/AAVMF_VARS.fd"
64 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
66 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
68 # Note about locking: we use flock on the config file protect
69 # against concurent actions.
70 # Aditionaly, we have a 'lock' setting in the config file. This
71 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
72 # allowed when such lock is set. But you can ignore this kind of
73 # lock with the --skiplock flag.
75 cfs_register_file
('/qemu-server/',
79 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
80 description
=> "Some command save/restore state from this location.",
86 PVE
::JSONSchema
::register_standard_option
('pve-qm-image-format', {
88 enum
=> [qw(raw cow qcow qed qcow2 vmdk cloop)],
89 description
=> "The drive's backing file's data format.",
93 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
94 description
=> "Specifies the Qemu machine type.",
96 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
101 #no warnings 'redefine';
104 my ($controller, $vmid, $option, $value) = @_;
106 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
107 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
111 my $nodename = PVE
::INotify
::nodename
();
113 my $cpu_vendor_list = {
115 486 => 'GenuineIntel',
116 pentium
=> 'GenuineIntel',
117 pentium2
=> 'GenuineIntel',
118 pentium3
=> 'GenuineIntel',
119 coreduo
=> 'GenuineIntel',
120 core2duo
=> 'GenuineIntel',
121 Conroe
=> 'GenuineIntel',
122 Penryn
=> 'GenuineIntel',
123 Nehalem
=> 'GenuineIntel',
124 'Nehalem-IBRS' => 'GenuineIntel',
125 Westmere
=> 'GenuineIntel',
126 'Westmere-IBRS' => 'GenuineIntel',
127 SandyBridge
=> 'GenuineIntel',
128 'SandyBridge-IBRS' => 'GenuineIntel',
129 IvyBridge
=> 'GenuineIntel',
130 'IvyBridge-IBRS' => 'GenuineIntel',
131 Haswell
=> 'GenuineIntel',
132 'Haswell-IBRS' => 'GenuineIntel',
133 'Haswell-noTSX' => 'GenuineIntel',
134 'Haswell-noTSX-IBRS' => 'GenuineIntel',
135 Broadwell
=> 'GenuineIntel',
136 'Broadwell-IBRS' => 'GenuineIntel',
137 'Broadwell-noTSX' => 'GenuineIntel',
138 'Broadwell-noTSX-IBRS' => 'GenuineIntel',
139 'Skylake-Client' => 'GenuineIntel',
140 'Skylake-Client-IBRS' => 'GenuineIntel',
141 'Skylake-Server' => 'GenuineIntel',
142 'Skylake-Server-IBRS' => 'GenuineIntel',
143 'Cascadelake-Server' => 'GenuineIntel',
144 KnightsMill
=> 'GenuineIntel',
148 athlon
=> 'AuthenticAMD',
149 phenom
=> 'AuthenticAMD',
150 Opteron_G1
=> 'AuthenticAMD',
151 Opteron_G2
=> 'AuthenticAMD',
152 Opteron_G3
=> 'AuthenticAMD',
153 Opteron_G4
=> 'AuthenticAMD',
154 Opteron_G5
=> 'AuthenticAMD',
155 EPYC
=> 'AuthenticAMD',
156 'EPYC-IBPB' => 'AuthenticAMD',
158 # generic types, use vendor from host node
167 my @supported_cpu_flags = (
181 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
185 description
=> "Emulated CPU type.",
187 enum
=> [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
192 description
=> "Do not identify as a KVM virtual machine.",
199 pattern
=> qr/[a-zA-Z0-9]{1,12}/,
200 format_description
=> 'vendor-id',
201 description
=> 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
205 description
=> "List of additional CPU flags separated by ';'."
206 . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
207 . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
208 format_description
=> '+FLAG[;-FLAG...]',
210 pattern
=> qr/$cpu_flag(;$cpu_flag)*/,
219 enum
=> [qw(i6300esb ib700)],
220 description
=> "Watchdog type to emulate.",
221 default => 'i6300esb',
226 enum
=> [qw(reset shutdown poweroff pause debug none)],
227 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
231 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
235 description
=> "Enable/disable Qemu GuestAgent.",
240 fstrim_cloned_disks
=> {
241 description
=> "Run fstrim after cloning/moving a disk.",
247 description
=> "Select the agent type",
251 enum
=> [qw(virtio isa)],
257 description
=> "Select the VGA type.",
262 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
265 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
277 description
=> "The size of the file in MB.",
281 pattern
=> '[a-zA-Z0-9\-]+',
283 format_description
=> 'string',
284 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
291 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
292 description
=> "Configure an audio device."
299 description
=> "Driver backend for the audio device."
303 my $spice_enhancements_fmt = {
308 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
312 enum
=> ['off', 'all', 'filter'],
315 description
=> "Enable video streaming. Uses compression for detected video streams."
323 description
=> "Specifies whether a VM will be started during system bootup.",
329 description
=> "Automatic restart after crash (currently ignored).",
334 type
=> 'string', format
=> 'pve-hotplug-features',
335 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
336 default => 'network,disk,usb',
341 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
347 description
=> "Lock/unlock the VM.",
348 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
353 description
=> "Limit of CPU usage.",
354 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
362 description
=> "CPU weight for a VM.",
363 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
371 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
378 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
384 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
392 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
393 "It should not be necessary to set it.",
394 enum
=> PVE
::Tools
::kvmkeymaplist
(),
399 type
=> 'string', format
=> 'dns-name',
400 description
=> "Set a name for the VM. Only used on the configuration web interface.",
405 description
=> "SCSI controller model",
406 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
412 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
417 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
418 description
=> "Specify guest operating system.",
419 verbose_description
=> <<EODESC,
420 Specify guest operating system. This is used to enable special
421 optimization/features for specific operating systems:
424 other;; unspecified OS
425 wxp;; Microsoft Windows XP
426 w2k;; Microsoft Windows 2000
427 w2k3;; Microsoft Windows 2003
428 w2k8;; Microsoft Windows 2008
429 wvista;; Microsoft Windows Vista
430 win7;; Microsoft Windows 7
431 win8;; Microsoft Windows 8/2012/2012r2
432 win10;; Microsoft Windows 10/2016
433 l24;; Linux 2.4 Kernel
434 l26;; Linux 2.6 - 5.X Kernel
435 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
441 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
442 pattern
=> '[acdn]{1,4}',
447 type
=> 'string', format
=> 'pve-qm-bootdisk',
448 description
=> "Enable booting from specified disk.",
449 pattern
=> '(ide|sata|scsi|virtio)\d+',
454 description
=> "The number of CPUs. Please use option -sockets instead.",
461 description
=> "The number of CPU sockets.",
468 description
=> "The number of cores per socket.",
475 description
=> "Enable/disable NUMA.",
481 description
=> "Enable/disable hugepages memory.",
482 enum
=> [qw(any 2 1024)],
487 description
=> "Number of hotplugged vcpus.",
494 description
=> "Enable/disable ACPI.",
499 description
=> "Enable/disable Qemu GuestAgent and its properties.",
501 format
=> $agent_fmt,
506 description
=> "Enable/disable KVM hardware virtualization.",
512 description
=> "Enable/disable time drift fix.",
518 description
=> "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
523 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
527 type
=> 'string', format
=> $vga_fmt,
528 description
=> "Configure the VGA hardware.",
529 verbose_description
=> "Configure the VGA Hardware. If you want to use ".
530 "high resolution modes (>= 1280x1024x16) you may need to increase " .
531 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
532 "is 'std' for all OS types besides some Windows versions (XP and " .
533 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
534 "display server. For win* OS you can select how many independent " .
535 "displays you want, Linux guests can add displays them self.\n".
536 "You can also run without any graphic card, using a serial device as terminal.",
540 type
=> 'string', format
=> 'pve-qm-watchdog',
541 description
=> "Create a virtual hardware watchdog device.",
542 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled" .
543 " (by a guest action), the watchdog must be periodically polled " .
544 "by an agent inside the guest or else the watchdog will reset " .
545 "the guest (or execute the respective action specified)",
550 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
551 description
=> "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
552 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
555 startup
=> get_standard_option
('pve-startup-order'),
559 description
=> "Enable/disable Template.",
565 description
=> "Arbitrary arguments passed to kvm.",
566 verbose_description
=> <<EODESCR,
567 Arbitrary arguments passed to kvm, for example:
569 args: -no-reboot -no-hpet
571 NOTE: this option is for experts only.
578 description
=> "Enable/disable the USB tablet device.",
579 verbose_description
=> "Enable/disable the USB tablet device. This device is " .
580 "usually needed to allow absolute mouse positioning with VNC. " .
581 "Else the mouse runs out of sync with normal VNC clients. " .
582 "If you're running lots of console-only guests on one host, " .
583 "you may consider disabling this to save some context switches. " .
584 "This is turned off by default if you use spice (-vga=qxl).",
589 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
593 migrate_downtime
=> {
596 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
602 type
=> 'string', format
=> 'pve-qm-ide',
603 typetext
=> '<volume>',
604 description
=> "This is an alias for option -ide2",
608 description
=> "Emulated CPU type.",
612 parent
=> get_standard_option
('pve-snapshot-name', {
614 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
618 description
=> "Timestamp for snapshots.",
624 type
=> 'string', format
=> 'pve-volume-id',
625 description
=> "Reference to a volume which stores the VM state. This is used internally for snapshots.",
627 vmstatestorage
=> get_standard_option
('pve-storage-id', {
628 description
=> "Default storage for VM state volumes/files.",
631 runningmachine
=> get_standard_option
('pve-qemu-machine', {
632 description
=> "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
634 machine
=> get_standard_option
('pve-qemu-machine'),
636 description
=> "Virtual processor architecture. Defaults to the host.",
639 enum
=> [qw(x86_64 aarch64)],
642 description
=> "Specify SMBIOS type 1 fields.",
643 type
=> 'string', format
=> 'pve-qm-smbios1',
650 description
=> "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
656 enum
=> [ qw(seabios ovmf) ],
657 description
=> "Select BIOS implementation.",
658 default => 'seabios',
662 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
663 format_description
=> 'UUID',
664 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
665 verbose_description
=> "The VM generation ID (vmgenid) device exposes a".
666 " 128-bit integer value identifier to the guest OS. This allows to".
667 " notify the guest operating system when the virtual machine is".
668 " executed with a different configuration (e.g. snapshot execution".
669 " or creation from a template). The guest operating system notices".
670 " the change, and is then able to react as appropriate by marking".
671 " its copies of distributed databases as dirty, re-initializing its".
672 " random number generator, etc.\n".
673 "Note that auto-creation only works when done throug API/CLI create".
674 " or update methods, but not when manually editing the config file.",
675 default => "1 (autogenerated)",
680 format
=> 'pve-volume-id',
682 description
=> "Script that will be executed during various steps in the vms lifetime.",
686 format
=> $ivshmem_fmt,
687 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
692 format
=> $audio_fmt,
693 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
696 spice_enhancements
=> {
698 format
=> $spice_enhancements_fmt,
699 description
=> "Configure additional enhancements for SPICE.",
703 type
=> 'string', format
=> 'pve-tag-list',
704 description
=> 'Tags of the VM. This is only meta information.',
713 description
=> 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
714 format
=> 'pve-volume-id',
715 format_description
=> 'volume',
720 description
=> 'Specify a custom file containing all network data passed to the VM via cloud-init.',
721 format
=> 'pve-volume-id',
722 format_description
=> 'volume',
727 description
=> 'Specify a custom file containing all user data passed to the VM via cloud-init.',
728 format
=> 'pve-volume-id',
729 format_description
=> 'volume',
732 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
734 my $confdesc_cloudinit = {
738 description
=> 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
739 enum
=> ['configdrive2', 'nocloud'],
744 description
=> "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
749 description
=> 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
754 description
=> 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
755 format
=> 'pve-qm-cicustom',
760 description
=> "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
764 type
=> 'string', format
=> 'address-list',
765 description
=> "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
770 format
=> 'urlencoded',
771 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
775 # what about other qemu settings ?
777 #machine => 'string',
790 ##soundhw => 'string',
792 while (my ($k, $v) = each %$confdesc) {
793 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
796 my $MAX_IDE_DISKS = 4;
797 my $MAX_SCSI_DISKS = 14;
798 my $MAX_VIRTIO_DISKS = 16;
799 my $MAX_SATA_DISKS = 6;
800 my $MAX_USB_DEVICES = 5;
802 my $MAX_UNUSED_DISKS = 256;
803 my $MAX_HOSTPCI_DEVICES = 16;
804 my $MAX_SERIAL_PORTS = 4;
805 my $MAX_PARALLEL_PORTS = 3;
811 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
812 description
=> "CPUs accessing this NUMA node.",
813 format_description
=> "id[-id];...",
817 description
=> "Amount of memory this NUMA node provides.",
822 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
823 description
=> "Host NUMA nodes to use.",
824 format_description
=> "id[-id];...",
829 enum
=> [qw(preferred bind interleave)],
830 description
=> "NUMA allocation policy.",
834 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
837 type
=> 'string', format
=> $numa_fmt,
838 description
=> "NUMA topology.",
840 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
842 for (my $i = 0; $i < $MAX_NUMA; $i++) {
843 $confdesc->{"numa$i"} = $numadesc;
846 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
847 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
848 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
849 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
851 my $net_fmt_bridge_descr = <<__EOD__;
852 Bridge to attach the network device to. The Proxmox VE standard bridge
855 If you do not specify a bridge, we create a kvm user (NATed) network
856 device, which provides DHCP and DNS services. The following addresses
863 The DHCP server assign addresses to the guest starting from 10.0.2.15.
867 macaddr
=> get_standard_option
('mac-addr', {
868 description
=> "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
872 description
=> "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
873 enum
=> $nic_model_list,
876 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
879 description
=> $net_fmt_bridge_descr,
880 format_description
=> 'bridge',
885 minimum
=> 0, maximum
=> 16,
886 description
=> 'Number of packet queues to be used on the device.',
892 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
897 minimum
=> 1, maximum
=> 4094,
898 description
=> 'VLAN tag to apply to packets on this interface.',
903 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
904 description
=> 'VLAN trunks to pass through this interface.',
905 format_description
=> 'vlanid[;vlanid...]',
910 description
=> 'Whether this interface should be protected by the firewall.',
915 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
922 type
=> 'string', format
=> $net_fmt,
923 description
=> "Specify network devices.",
926 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
931 format
=> 'pve-ipv4-config',
932 format_description
=> 'IPv4Format/CIDR',
933 description
=> 'IPv4 address in CIDR format.',
940 format_description
=> 'GatewayIPv4',
941 description
=> 'Default gateway for IPv4 traffic.',
947 format
=> 'pve-ipv6-config',
948 format_description
=> 'IPv6Format/CIDR',
949 description
=> 'IPv6 address in CIDR format.',
956 format_description
=> 'GatewayIPv6',
957 description
=> 'Default gateway for IPv6 traffic.',
962 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
965 type
=> 'string', format
=> 'pve-qm-ipconfig',
966 description
=> <<'EODESCR',
967 cloud-init: Specify IP addresses and gateways for the corresponding interface.
969 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
971 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
972 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
974 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
977 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
979 for (my $i = 0; $i < $MAX_NETS; $i++) {
980 $confdesc->{"net$i"} = $netdesc;
981 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
984 foreach my $key (keys %$confdesc_cloudinit) {
985 $confdesc->{$key} = $confdesc_cloudinit->{$key};
988 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
989 sub verify_volume_id_or_qm_path
{
990 my ($volid, $noerr) = @_;
992 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
996 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
997 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
999 return undef if $noerr;
1007 my %drivedesc_base = (
1008 volume
=> { alias
=> 'file' },
1011 format
=> 'pve-volume-id-or-qm-path',
1013 format_description
=> 'volume',
1014 description
=> "The drive's backing volume.",
1018 enum
=> [qw(cdrom disk)],
1019 description
=> "The drive's media type.",
1025 description
=> "Force the drive's physical geometry to have a specific cylinder count.",
1030 description
=> "Force the drive's physical geometry to have a specific head count.",
1035 description
=> "Force the drive's physical geometry to have a specific sector count.",
1040 enum
=> [qw(none lba auto)],
1041 description
=> "Force disk geometry bios translation mode.",
1046 description
=> "Controls qemu's snapshot mode feature."
1047 . " If activated, changes made to the disk are temporary and will"
1048 . " be discarded when the VM is shutdown.",
1053 enum
=> [qw(none writethrough writeback unsafe directsync)],
1054 description
=> "The drive's cache mode",
1057 format
=> get_standard_option
('pve-qm-image-format'),
1060 format
=> 'disk-size',
1061 format_description
=> 'DiskSize',
1062 description
=> "Disk size. This is purely informational and has no effect.",
1067 description
=> "Whether the drive should be included when making backups.",
1072 description
=> 'Whether the drive should considered for replication jobs.',
1078 enum
=> [qw(ignore report stop)],
1079 description
=> 'Read error action.',
1084 enum
=> [qw(enospc ignore report stop)],
1085 description
=> 'Write error action.',
1090 enum
=> [qw(native threads)],
1091 description
=> 'AIO type to use.',
1096 enum
=> [qw(ignore on)],
1097 description
=> 'Controls whether to pass discard/trim requests to the underlying storage.',
1102 description
=> 'Controls whether to detect and try to optimize writes of zeroes.',
1107 format
=> 'urlencoded',
1108 format_description
=> 'serial',
1109 maxLength
=> 20*3, # *3 since it's %xx url enoded
1110 description
=> "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1115 description
=> 'Mark this locally-managed volume as available on all nodes',
1116 verbose_description
=> "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1122 my %iothread_fmt = ( iothread
=> {
1124 description
=> "Whether to use iothreads for this drive",
1131 format
=> 'urlencoded',
1132 format_description
=> 'model',
1133 maxLength
=> 40*3, # *3 since it's %xx url enoded
1134 description
=> "The drive's reported model name, url-encoded, up to 40 bytes long.",
1142 description
=> "Number of queues.",
1148 my %scsiblock_fmt = (
1151 description
=> "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1160 description
=> "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1168 pattern
=> qr/^(0x)[0-9a-fA-F]{16}/,
1169 format_description
=> 'wwn',
1170 description
=> "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1175 my $add_throttle_desc = sub {
1176 my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1179 format_description
=> $unit,
1180 description
=> "Maximum $what in $longunit.",
1183 $d->{minimum
} = $minimum if defined($minimum);
1184 $drivedesc_base{$key} = $d;
1186 # throughput: (leaky bucket)
1187 $add_throttle_desc->('bps', 'integer', 'r/w speed', 'bps', 'bytes per second');
1188 $add_throttle_desc->('bps_rd', 'integer', 'read speed', 'bps', 'bytes per second');
1189 $add_throttle_desc->('bps_wr', 'integer', 'write speed', 'bps', 'bytes per second');
1190 $add_throttle_desc->('mbps', 'number', 'r/w speed', 'mbps', 'megabytes per second');
1191 $add_throttle_desc->('mbps_rd', 'number', 'read speed', 'mbps', 'megabytes per second');
1192 $add_throttle_desc->('mbps_wr', 'number', 'write speed', 'mbps', 'megabytes per second');
1193 $add_throttle_desc->('iops', 'integer', 'r/w I/O', 'iops', 'operations per second');
1194 $add_throttle_desc->('iops_rd', 'integer', 'read I/O', 'iops', 'operations per second');
1195 $add_throttle_desc->('iops_wr', 'integer', 'write I/O', 'iops', 'operations per second');
1197 # pools: (pool of IO before throttling starts taking effect)
1198 $add_throttle_desc->('mbps_max', 'number', 'unthrottled r/w pool', 'mbps', 'megabytes per second');
1199 $add_throttle_desc->('mbps_rd_max', 'number', 'unthrottled read pool', 'mbps', 'megabytes per second');
1200 $add_throttle_desc->('mbps_wr_max', 'number', 'unthrottled write pool', 'mbps', 'megabytes per second');
1201 $add_throttle_desc->('iops_max', 'integer', 'unthrottled r/w I/O pool', 'iops', 'operations per second');
1202 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool', 'iops', 'operations per second');
1203 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1206 $add_throttle_desc->('bps_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1207 $add_throttle_desc->('bps_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1208 $add_throttle_desc->('bps_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1210 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1211 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1214 $drivedesc_base{'bps_rd_length'} = { alias
=> 'bps_rd_max_length' };
1215 $drivedesc_base{'bps_wr_length'} = { alias
=> 'bps_wr_max_length' };
1216 $drivedesc_base{'iops_rd_length'} = { alias
=> 'iops_rd_max_length' };
1217 $drivedesc_base{'iops_wr_length'} = { alias
=> 'iops_wr_max_length' };
1225 PVE
::JSONSchema
::register_format
("pve-qm-ide", $ide_fmt);
1229 type
=> 'string', format
=> $ide_fmt,
1230 description
=> "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1232 PVE
::JSONSchema
::register_standard_option
("pve-qm-ide", $idedesc);
1244 type
=> 'string', format
=> $scsi_fmt,
1245 description
=> "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1247 PVE
::JSONSchema
::register_standard_option
("pve-qm-scsi", $scsidesc);
1256 type
=> 'string', format
=> $sata_fmt,
1257 description
=> "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1259 PVE
::JSONSchema
::register_standard_option
("pve-qm-sata", $satadesc);
1267 type
=> 'string', format
=> $virtio_fmt,
1268 description
=> "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1270 PVE
::JSONSchema
::register_standard_option
("pve-qm-virtio", $virtiodesc);
1272 my $alldrive_fmt = {
1283 volume
=> { alias
=> 'file' },
1286 format
=> 'pve-volume-id-or-qm-path',
1288 format_description
=> 'volume',
1289 description
=> "The drive's backing volume.",
1291 format
=> get_standard_option
('pve-qm-image-format'),
1294 format
=> 'disk-size',
1295 format_description
=> 'DiskSize',
1296 description
=> "Disk size. This is purely informational and has no effect.",
1301 my $efidisk_desc = {
1303 type
=> 'string', format
=> $efidisk_fmt,
1304 description
=> "Configure a Disk for storing EFI vars",
1307 PVE
::JSONSchema
::register_standard_option
("pve-qm-efidisk", $efidisk_desc);
1312 type
=> 'string', format
=> 'pve-qm-usb-device',
1313 format_description
=> 'HOSTUSBDEVICE|spice',
1314 description
=> <<EODESCR,
1315 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1317 'bus-port(.port)*' (decimal numbers) or
1318 'vendor_id:product_id' (hexadeciaml numbers) or
1321 You can use the 'lsusb -t' command to list existing usb devices.
1323 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1325 The value 'spice' can be used to add a usb redirection devices for spice.
1331 description
=> "Specifies whether if given host option is a USB3 device or port.",
1338 type
=> 'string', format
=> $usb_fmt,
1339 description
=> "Configure an USB device (n is 0 to 4).",
1341 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1343 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1348 pattern
=> qr/$PCIRE(;$PCIRE)*/,
1349 format_description
=> 'HOSTPCIID[;HOSTPCIID2...]',
1350 description
=> <<EODESCR,
1351 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1352 of PCI virtual functions of the host. HOSTPCIID syntax is:
1354 'bus:dev.func' (hexadecimal numbers)
1356 You can us the 'lspci' command to list existing PCI devices.
1361 description
=> "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1367 pattern
=> '[^,;]+',
1368 format_description
=> 'string',
1369 description
=> "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1374 description
=> "Choose the PCI-express bus (needs the 'q35' machine model).",
1380 description
=> "Enable vfio-vga device support.",
1386 format_description
=> 'string',
1387 pattern
=> '[^/\.:]+',
1389 description
=> <<EODESCR
1390 The type of mediated device to use.
1391 An instance of this type will be created on startup of the VM and
1392 will be cleaned up when the VM stops.
1396 PVE
::JSONSchema
::register_format
('pve-qm-hostpci', $hostpci_fmt);
1400 type
=> 'string', format
=> 'pve-qm-hostpci',
1401 description
=> "Map host PCI devices into guest.",
1402 verbose_description
=> <<EODESCR,
1403 Map host PCI devices into guest.
1405 NOTE: This option allows direct access to host hardware. So it is no longer
1406 possible to migrate such machines - use with special care.
1408 CAUTION: Experimental! User reported problems with this option.
1411 PVE
::JSONSchema
::register_standard_option
("pve-qm-hostpci", $hostpcidesc);
1416 pattern
=> '(/dev/.+|socket)',
1417 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1418 verbose_description
=> <<EODESCR,
1419 Create a serial device inside the VM (n is 0 to 3), and pass through a
1420 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1421 host side (use 'qm terminal' to open a terminal connection).
1423 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1425 CAUTION: Experimental! User reported problems with this option.
1432 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1433 description
=> "Map host parallel devices (n is 0 to 2).",
1434 verbose_description
=> <<EODESCR,
1435 Map host parallel devices (n is 0 to 2).
1437 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1439 CAUTION: Experimental! User reported problems with this option.
1443 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1444 $confdesc->{"parallel$i"} = $paralleldesc;
1447 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1448 $confdesc->{"serial$i"} = $serialdesc;
1451 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
1452 $confdesc->{"hostpci$i"} = $hostpcidesc;
1455 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) {
1456 $drivename_hash->{"ide$i"} = 1;
1457 $confdesc->{"ide$i"} = $idedesc;
1460 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) {
1461 $drivename_hash->{"sata$i"} = 1;
1462 $confdesc->{"sata$i"} = $satadesc;
1465 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) {
1466 $drivename_hash->{"scsi$i"} = 1;
1467 $confdesc->{"scsi$i"} = $scsidesc ;
1470 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) {
1471 $drivename_hash->{"virtio$i"} = 1;
1472 $confdesc->{"virtio$i"} = $virtiodesc;
1475 $drivename_hash->{efidisk0
} = 1;
1476 $confdesc->{efidisk0
} = $efidisk_desc;
1478 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1479 $confdesc->{"usb$i"} = $usbdesc;
1484 type
=> 'string', format
=> 'pve-volume-id',
1485 description
=> "Reference to unused volumes. This is used internally, and should not be modified manually.",
1488 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1489 $confdesc->{"unused$i"} = $unuseddesc;
1492 my $kvm_api_version = 0;
1495 return $kvm_api_version if $kvm_api_version;
1497 open my $fh, '<', '/dev/kvm'
1500 # 0xae00 => KVM_GET_API_VERSION
1501 $kvm_api_version = ioctl($fh, 0xae00, 0);
1503 return $kvm_api_version;
1506 my $kvm_user_version = {};
1509 sub kvm_user_version
{
1512 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1513 my $st = stat($binary);
1515 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1516 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1517 $cachedmtime == $st->mtime;
1519 $kvm_user_version->{$binary} = 'unknown';
1520 $kvm_mtime->{$binary} = $st->mtime;
1524 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1525 $kvm_user_version->{$binary} = $2;
1529 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1532 return $kvm_user_version->{$binary};
1536 sub kernel_has_vhost_net
{
1537 return -c
'/dev/vhost-net';
1540 sub valid_drive_names
{
1541 # order is important - used to autoselect boot disk
1542 return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1543 (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1544 (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1545 (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1549 sub is_valid_drivename
{
1552 return defined($drivename_hash->{$dev});
1557 return defined($confdesc->{$key});
1561 sub get_cdrom_path
{
1563 return $cdrom_path if $cdrom_path;
1565 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1566 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1567 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1571 my ($storecfg, $vmid, $cdrom) = @_;
1573 if ($cdrom eq 'cdrom') {
1574 return get_cdrom_path
();
1575 } elsif ($cdrom eq 'none') {
1577 } elsif ($cdrom =~ m
|^/|) {
1580 return PVE
::Storage
::path
($storecfg, $cdrom);
1584 # try to convert old style file names to volume IDs
1585 sub filename_to_volume_id
{
1586 my ($vmid, $file, $media) = @_;
1588 if (!($file eq 'none' || $file eq 'cdrom' ||
1589 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1591 return undef if $file =~ m
|/|;
1593 if ($media && $media eq 'cdrom') {
1594 $file = "local:iso/$file";
1596 $file = "local:$vmid/$file";
1603 sub verify_media_type
{
1604 my ($opt, $vtype, $media) = @_;
1609 if ($media eq 'disk') {
1611 } elsif ($media eq 'cdrom') {
1614 die "internal error";
1617 return if ($vtype eq $etype);
1619 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1622 sub cleanup_drive_path
{
1623 my ($opt, $storecfg, $drive) = @_;
1625 # try to convert filesystem paths to volume IDs
1627 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1628 ($drive->{file
} !~ m
|^/dev/.+|) &&
1629 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1630 ($drive->{file
} !~ m/^\d+$/)) {
1631 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1632 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1633 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1634 verify_media_type
($opt, $vtype, $drive->{media
});
1635 $drive->{file
} = $volid;
1638 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1641 sub parse_hotplug_features
{
1646 return $res if $data eq '0';
1648 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1650 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1651 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1654 die "invalid hotplug feature '$feature'\n";
1660 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1661 sub pve_verify_hotplug_features
{
1662 my ($value, $noerr) = @_;
1664 return $value if parse_hotplug_features
($value);
1666 return undef if $noerr;
1668 die "unable to parse hotplug option\n";
1671 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1672 # [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1673 # [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1674 # [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1675 # [,iothread=on][,serial=serial][,model=model]
1678 my ($key, $data) = @_;
1680 my ($interface, $index);
1682 if ($key =~ m/^([^\d]+)(\d+)$/) {
1689 my $desc = $key =~ /^unused\d+$/ ?
$alldrive_fmt
1690 : $confdesc->{$key}->{format
};
1692 warn "invalid drive key: $key\n";
1695 my $res = eval { PVE
::JSONSchema
::parse_property_string
($desc, $data) };
1696 return undef if !$res;
1697 $res->{interface
} = $interface;
1698 $res->{index} = $index;
1701 foreach my $opt (qw(bps bps_rd bps_wr)) {
1702 if (my $bps = defined(delete $res->{$opt})) {
1703 if (defined($res->{"m$opt"})) {
1704 warn "both $opt and m$opt specified\n";
1708 $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1712 # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1713 for my $requirement (
1714 [mbps_max
=> 'mbps'],
1715 [mbps_rd_max
=> 'mbps_rd'],
1716 [mbps_wr_max
=> 'mbps_wr'],
1717 [miops_max
=> 'miops'],
1718 [miops_rd_max
=> 'miops_rd'],
1719 [miops_wr_max
=> 'miops_wr'],
1720 [bps_max_length
=> 'mbps_max'],
1721 [bps_rd_max_length
=> 'mbps_rd_max'],
1722 [bps_wr_max_length
=> 'mbps_wr_max'],
1723 [iops_max_length
=> 'iops_max'],
1724 [iops_rd_max_length
=> 'iops_rd_max'],
1725 [iops_wr_max_length
=> 'iops_wr_max']) {
1726 my ($option, $requires) = @$requirement;
1727 if ($res->{$option} && !$res->{$requires}) {
1728 warn "$option requires $requires\n";
1733 return undef if $error;
1735 return undef if $res->{mbps_rd
} && $res->{mbps
};
1736 return undef if $res->{mbps_wr
} && $res->{mbps
};
1737 return undef if $res->{iops_rd
} && $res->{iops
};
1738 return undef if $res->{iops_wr
} && $res->{iops
};
1740 if ($res->{media
} && ($res->{media
} eq 'cdrom')) {
1741 return undef if $res->{snapshot
} || $res->{trans
} || $res->{format
};
1742 return undef if $res->{heads
} || $res->{secs
} || $res->{cyls
};
1743 return undef if $res->{interface
} eq 'virtio';
1746 if (my $size = $res->{size
}) {
1747 return undef if !defined($res->{size
} = PVE
::JSONSchema
::parse_size
($size));
1755 my $data = { %$drive };
1756 delete $data->{$_} for qw(index interface);
1757 return PVE
::JSONSchema
::print_property_string
($data, $alldrive_fmt);
1761 my($fh, $noerr) = @_;
1764 my $SG_GET_VERSION_NUM = 0x2282;
1766 my $versionbuf = "\x00" x
8;
1767 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1769 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1772 my $version = unpack("I", $versionbuf);
1773 if ($version < 30000) {
1774 die "scsi generic interface too old\n" if !$noerr;
1778 my $buf = "\x00" x
36;
1779 my $sensebuf = "\x00" x
8;
1780 my $cmd = pack("C x3 C x1", 0x12, 36);
1782 # see /usr/include/scsi/sg.h
1783 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1785 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1786 length($sensebuf), 0, length($buf), $buf,
1787 $cmd, $sensebuf, 6000);
1789 $ret = ioctl($fh, $SG_IO, $packet);
1791 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1795 my @res = unpack($sg_io_hdr_t, $packet);
1796 if ($res[17] || $res[18]) {
1797 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1802 (my $byte0, my $byte1, $res->{vendor
},
1803 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1805 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1806 $res->{type
} = $byte0 & 31;
1814 my $fh = IO
::File-
>new("+<$path") || return undef;
1815 my $res = scsi_inquiry
($fh, 1);
1821 sub print_tabletdevice_full
{
1822 my ($conf, $arch) = @_;
1824 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1826 # we use uhci for old VMs because tablet driver was buggy in older qemu
1828 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1834 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1837 sub print_keyboarddevice_full
{
1838 my ($conf, $arch, $machine) = @_;
1840 return undef if $arch ne 'aarch64';
1842 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1845 sub print_drivedevice_full
{
1846 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1851 if ($drive->{interface
} eq 'virtio') {
1852 my $pciaddr = print_pci_addr
("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1853 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1854 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread
};
1855 } elsif ($drive->{interface
} eq 'scsi') {
1857 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1858 my $unit = $drive->{index} % $maxdev;
1859 my $devicetype = 'hd';
1861 if (drive_is_cdrom
($drive)) {
1864 if ($drive->{file
} =~ m
|^/|) {
1865 $path = $drive->{file
};
1866 if (my $info = path_is_scsi
($path)) {
1867 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1868 $devicetype = 'block';
1869 } elsif ($info->{type
} == 1) { # tape
1870 $devicetype = 'generic';
1874 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1877 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1878 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, kvm_user_version
());
1879 if ($path =~ m/^iscsi\:\/\
// &&
1880 !min_version
($version, 4, 1)) {
1881 $devicetype = 'generic';
1885 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1886 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1888 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1891 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1892 $device .= ",rotation_rate=1";
1894 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1896 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1897 my $maxdev = ($drive->{interface
} eq 'sata') ?
$MAX_SATA_DISKS : 2;
1898 my $controller = int($drive->{index} / $maxdev);
1899 my $unit = $drive->{index} % $maxdev;
1900 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1902 $device = "ide-$devicetype";
1903 if ($drive->{interface
} eq 'ide') {
1904 $device .= ",bus=ide.$controller,unit=$unit";
1906 $device .= ",bus=ahci$controller.$unit";
1908 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1910 if ($devicetype eq 'hd') {
1911 if (my $model = $drive->{model
}) {
1912 $model = URI
::Escape
::uri_unescape
($model);
1913 $device .= ",model=$model";
1915 if ($drive->{ssd
}) {
1916 $device .= ",rotation_rate=1";
1919 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1920 } elsif ($drive->{interface
} eq 'usb') {
1922 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1924 die "unsupported interface type";
1927 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1929 if (my $serial = $drive->{serial
}) {
1930 $serial = URI
::Escape
::uri_unescape
($serial);
1931 $device .= ",serial=$serial";
1938 sub get_initiator_name
{
1941 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return undef;
1942 while (defined(my $line = <$fh>)) {
1943 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1952 sub print_drive_full
{
1953 my ($storecfg, $vmid, $drive) = @_;
1956 my $volid = $drive->{file
};
1959 if (drive_is_cdrom
($drive)) {
1960 $path = get_iso_path
($storecfg, $vmid, $volid);
1962 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1964 $path = PVE
::Storage
::path
($storecfg, $volid);
1965 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1966 $format = qemu_img_format
($scfg, $volname);
1974 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1975 foreach my $o (@qemu_drive_options) {
1976 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1979 # snapshot only accepts on|off
1980 if (defined($drive->{snapshot
})) {
1981 my $v = $drive->{snapshot
} ?
'on' : 'off';
1982 $opts .= ",snapshot=$v";
1985 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1986 my ($dir, $qmpname) = @$type;
1987 if (my $v = $drive->{"mbps$dir"}) {
1988 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1990 if (my $v = $drive->{"mbps${dir}_max"}) {
1991 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1993 if (my $v = $drive->{"bps${dir}_max_length"}) {
1994 $opts .= ",throttling.bps$qmpname-max-length=$v";
1996 if (my $v = $drive->{"iops${dir}"}) {
1997 $opts .= ",throttling.iops$qmpname=$v";
1999 if (my $v = $drive->{"iops${dir}_max"}) {
2000 $opts .= ",throttling.iops$qmpname-max=$v";
2002 if (my $v = $drive->{"iops${dir}_max_length"}) {
2003 $opts .= ",throttling.iops$qmpname-max-length=$v";
2007 $opts .= ",format=$format" if $format && !$drive->{format
};
2009 my $cache_direct = 0;
2011 if (my $cache = $drive->{cache
}) {
2012 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2013 } elsif (!drive_is_cdrom
($drive)) {
2014 $opts .= ",cache=none";
2018 # aio native works only with O_DIRECT
2019 if (!$drive->{aio
}) {
2021 $opts .= ",aio=native";
2023 $opts .= ",aio=threads";
2027 if (!drive_is_cdrom
($drive)) {
2029 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
2030 $detectzeroes = 'off';
2031 } elsif ($drive->{discard
}) {
2032 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
2034 # This used to be our default with discard not being specified:
2035 $detectzeroes = 'on';
2037 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2040 my $pathinfo = $path ?
"file=$path," : '';
2042 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2045 sub print_netdevice_full
{
2046 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2048 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
2050 my $device = $net->{model
};
2051 if ($net->{model
} eq 'virtio') {
2052 $device = 'virtio-net-pci';
2055 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
2056 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2057 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
2058 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2059 my $vectors = $net->{queues
} * 2 + 2;
2060 $tmpstr .= ",vectors=$vectors,mq=on";
2062 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
2064 if ($use_old_bios_files) {
2066 if ($device eq 'virtio-net-pci') {
2067 $romfile = 'pxe-virtio.rom';
2068 } elsif ($device eq 'e1000') {
2069 $romfile = 'pxe-e1000.rom';
2070 } elsif ($device eq 'ne2k') {
2071 $romfile = 'pxe-ne2k_pci.rom';
2072 } elsif ($device eq 'pcnet') {
2073 $romfile = 'pxe-pcnet.rom';
2074 } elsif ($device eq 'rtl8139') {
2075 $romfile = 'pxe-rtl8139.rom';
2077 $tmpstr .= ",romfile=$romfile" if $romfile;
2083 sub print_netdev_full
{
2084 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2087 if ($netid =~ m/^net(\d+)$/) {
2091 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
2093 my $ifname = "tap${vmid}i$i";
2095 # kvm uses TUNSETIFF ioctl, and that limits ifname length
2096 die "interface name '$ifname' is too long (max 15 character)\n"
2097 if length($ifname) >= 16;
2099 my $vhostparam = '';
2100 if (is_native
($arch)) {
2101 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
2104 my $vmname = $conf->{name
} || "vm$vmid";
2107 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
2109 if ($net->{bridge
}) {
2110 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2112 $netdev = "type=user,id=$netid,hostname=$vmname";
2115 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
2121 sub print_cpu_device
{
2122 my ($conf, $id) = @_;
2124 my $kvm = $conf->{kvm
} // 1;
2125 my $cpu = $kvm ?
"kvm64" : "qemu64";
2126 if (my $cputype = $conf->{cpu
}) {
2127 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
2128 or die "Cannot parse cpu description: $cputype\n";
2129 $cpu = $cpuconf->{cputype
};
2132 my $cores = $conf->{cores
} || 1;
2134 my $current_core = ($id - 1) % $cores;
2135 my $current_socket = int(($id - 1 - $current_core)/$cores);
2137 return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2141 'cirrus' => 'cirrus-vga',
2143 'vmware' => 'vmware-svga',
2144 'virtio' => 'virtio-vga',
2147 sub print_vga_device
{
2148 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
2150 my $type = $vga_map->{$vga->{type
}};
2151 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2152 $type = 'virtio-gpu';
2154 my $vgamem_mb = $vga->{memory
};
2156 my $max_outputs = '';
2158 $type = $id ?
'qxl' : 'qxl-vga';
2160 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
2161 # set max outputs so linux can have up to 4 qxl displays with one device
2162 if (min_version
($machine_version, 4, 1)) {
2163 $max_outputs = ",max_outputs=4";
2168 die "no devicetype for $vga->{type}\n" if !$type;
2172 if ($vga->{type
} eq 'virtio') {
2173 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
2174 $memory = ",max_hostmem=$bytes";
2176 # from https://www.spice-space.org/multiple-monitors.html
2177 $memory = ",vgamem_mb=$vga->{memory}";
2178 my $ram = $vgamem_mb * 4;
2179 my $vram = $vgamem_mb * 2;
2180 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2182 $memory = ",vgamem_mb=$vga->{memory}";
2184 } elsif ($qxlnum && $id) {
2185 $memory = ",ram_size=67108864,vram_size=33554432";
2188 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2189 my $vgaid = "vga" . ($id // '');
2192 if ($q35 && $vgaid eq 'vga') {
2193 # the first display uses pcie.0 bus on q35 machines
2194 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
2196 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
2199 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
2202 sub drive_is_cloudinit
{
2204 return $drive->{file
} =~ m
@[:/]vm-\d
+-cloudinit
(?
:\
.$QEMU_FORMAT_RE)?
$@;
2207 sub drive_is_cdrom
{
2208 my ($drive, $exclude_cloudinit) = @_;
2210 return 0 if $exclude_cloudinit && drive_is_cloudinit
($drive);
2212 return $drive && $drive->{media
} && ($drive->{media
} eq 'cdrom');
2216 sub parse_number_sets
{
2219 foreach my $part (split(/;/, $set)) {
2220 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2221 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2222 push @$res, [ $1, $2 ];
2224 die "invalid range: $part\n";
2233 my $res = PVE
::JSONSchema
::parse_property_string
($numa_fmt, $data);
2234 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
2235 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
2242 return undef if !$value;
2244 my $res = PVE
::JSONSchema
::parse_property_string
($hostpci_fmt, $value);
2246 my @idlist = split(/;/, $res->{host
});
2247 delete $res->{host
};
2248 foreach my $id (@idlist) {
2249 my $devs = PVE
::SysFSTools
::lspci
($id);
2250 die "no PCI device found for '$id'\n" if !scalar(@$devs);
2251 push @{$res->{pciid
}}, @$devs;
2256 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2260 my $res = eval { PVE
::JSONSchema
::parse_property_string
($net_fmt, $data) };
2265 if (!defined($res->{macaddr
})) {
2266 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
2267 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
2272 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2273 sub parse_ipconfig
{
2276 my $res = eval { PVE
::JSONSchema
::parse_property_string
($ipconfig_fmt, $data) };
2282 if ($res->{gw
} && !$res->{ip
}) {
2283 warn 'gateway specified without specifying an IP address';
2286 if ($res->{gw6
} && !$res->{ip6
}) {
2287 warn 'IPv6 gateway specified without specifying an IPv6 address';
2290 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
2291 warn 'gateway specified together with DHCP';
2294 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
2296 warn "IPv6 gateway specified together with $res->{ip6} address";
2300 if (!$res->{ip
} && !$res->{ip6
}) {
2301 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
2310 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
2313 sub add_random_macs
{
2314 my ($settings) = @_;
2316 foreach my $opt (keys %$settings) {
2317 next if $opt !~ m/^net(\d+)$/;
2318 my $net = parse_net
($settings->{$opt});
2320 $settings->{$opt} = print_net
($net);
2324 sub vm_is_volid_owner
{
2325 my ($storecfg, $vmid, $volid) = @_;
2327 if ($volid !~ m
|^/|) {
2329 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
2330 if ($owner && ($owner == $vmid)) {
2338 sub vmconfig_register_unused_drive
{
2339 my ($storecfg, $vmid, $conf, $drive) = @_;
2341 if (drive_is_cloudinit
($drive)) {
2342 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2344 } elsif (!drive_is_cdrom
($drive)) {
2345 my $volid = $drive->{file
};
2346 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2347 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2352 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2356 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2357 format_description
=> 'UUID',
2358 description
=> "Set SMBIOS1 UUID.",
2363 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2364 format_description
=> 'Base64 encoded string',
2365 description
=> "Set SMBIOS1 version.",
2370 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2371 format_description
=> 'Base64 encoded string',
2372 description
=> "Set SMBIOS1 serial number.",
2377 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2378 format_description
=> 'Base64 encoded string',
2379 description
=> "Set SMBIOS1 manufacturer.",
2384 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2385 format_description
=> 'Base64 encoded string',
2386 description
=> "Set SMBIOS1 product ID.",
2391 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2392 format_description
=> 'Base64 encoded string',
2393 description
=> "Set SMBIOS1 SKU string.",
2398 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2399 format_description
=> 'Base64 encoded string',
2400 description
=> "Set SMBIOS1 family string.",
2405 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2413 my $res = eval { PVE
::JSONSchema
::parse_property_string
($smbios1_fmt, $data) };
2420 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2423 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2425 PVE
::JSONSchema
::register_format
('pve-qm-bootdisk', \
&verify_bootdisk
);
2426 sub verify_bootdisk
{
2427 my ($value, $noerr) = @_;
2429 return $value if is_valid_drivename
($value);
2431 return undef if $noerr;
2433 die "invalid boot disk '$value'\n";
2436 sub parse_watchdog
{
2439 return undef if !$value;
2441 my $res = eval { PVE
::JSONSchema
::parse_property_string
($watchdog_fmt, $value) };
2446 sub parse_guest_agent
{
2449 return {} if !defined($value->{agent
});
2451 my $res = eval { PVE
::JSONSchema
::parse_property_string
($agent_fmt, $value->{agent
}) };
2454 # if the agent is disabled ignore the other potentially set properties
2455 return {} if !$res->{enabled
};
2462 return {} if !$value;
2463 my $res = eval { PVE
::JSONSchema
::parse_property_string
($vga_fmt, $value) };
2468 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2469 sub verify_usb_device
{
2470 my ($value, $noerr) = @_;
2472 return $value if parse_usb_device
($value);
2474 return undef if $noerr;
2476 die "unable to parse usb device\n";
2479 # add JSON properties for create and set function
2480 sub json_config_properties
{
2483 foreach my $opt (keys %$confdesc) {
2484 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2485 $prop->{$opt} = $confdesc->{$opt};
2491 # return copy of $confdesc_cloudinit to generate documentation
2492 sub cloudinit_config_properties
{
2494 return dclone
($confdesc_cloudinit);
2498 my ($key, $value) = @_;
2500 die "unknown setting '$key'\n" if !$confdesc->{$key};
2502 my $type = $confdesc->{$key}->{type
};
2504 if (!defined($value)) {
2505 die "got undefined value\n";
2508 if ($value =~ m/[\n\r]/) {
2509 die "property contains a line feed\n";
2512 if ($type eq 'boolean') {
2513 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2514 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2515 die "type check ('boolean') failed - got '$value'\n";
2516 } elsif ($type eq 'integer') {
2517 return int($1) if $value =~ m/^(\d+)$/;
2518 die "type check ('integer') failed - got '$value'\n";
2519 } elsif ($type eq 'number') {
2520 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2521 die "type check ('number') failed - got '$value'\n";
2522 } elsif ($type eq 'string') {
2523 if (my $fmt = $confdesc->{$key}->{format
}) {
2524 PVE
::JSONSchema
::check_format
($fmt, $value);
2527 $value =~ s/^\"(.*)\"$/$1/;
2530 die "internal error"
2535 my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2537 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2539 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2541 if ($conf->{template
}) {
2542 # check if any base image is still used by a linked clone
2543 foreach_drive
($conf, sub {
2544 my ($ds, $drive) = @_;
2545 return if drive_is_cdrom
($drive);
2547 my $volid = $drive->{file
};
2548 return if !$volid || $volid =~ m
|^/|;
2550 die "base volume '$volid' is still in use by linked cloned\n"
2551 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2556 # only remove disks owned by this VM
2557 foreach_drive
($conf, sub {
2558 my ($ds, $drive) = @_;
2559 return if drive_is_cdrom
($drive, 1);
2561 my $volid = $drive->{file
};
2562 return if !$volid || $volid =~ m
|^/|;
2564 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2565 return if !$path || !$owner || ($owner != $vmid);
2567 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2568 warn "Could not remove disk '$volid', check manually: $@" if $@;
2571 # also remove unused disk
2572 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2573 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2574 my ($volid, $sid, $volname, $d) = @_;
2575 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2579 if (defined $replacement_conf) {
2580 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2582 PVE
::QemuConfig-
>destroy_config($vmid);
2586 sub parse_vm_config
{
2587 my ($filename, $raw) = @_;
2589 return undef if !defined($raw);
2592 digest
=> Digest
::SHA
::sha1_hex
($raw),
2597 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2598 || die "got strange filename '$filename'";
2606 my @lines = split(/\n/, $raw);
2607 foreach my $line (@lines) {
2608 next if $line =~ m/^\s*$/;
2610 if ($line =~ m/^\[PENDING\]\s*$/i) {
2611 $section = 'pending';
2612 if (defined($descr)) {
2614 $conf->{description
} = $descr;
2617 $conf = $res->{$section} = {};
2620 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2622 if (defined($descr)) {
2624 $conf->{description
} = $descr;
2627 $conf = $res->{snapshots
}->{$section} = {};
2631 if ($line =~ m/^\#(.*)\s*$/) {
2632 $descr = '' if !defined($descr);
2633 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2637 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2638 $descr = '' if !defined($descr);
2639 $descr .= PVE
::Tools
::decode_text
($2);
2640 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2641 $conf->{snapstate
} = $1;
2642 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2645 $conf->{$key} = $value;
2646 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2648 if ($section eq 'pending') {
2649 $conf->{delete} = $value; # we parse this later
2651 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2653 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2656 eval { $value = check_type
($key, $value); };
2658 warn "vm $vmid - unable to parse value of '$key' - $@";
2660 $key = 'ide2' if $key eq 'cdrom';
2661 my $fmt = $confdesc->{$key}->{format
};
2662 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2663 my $v = parse_drive
($key, $value);
2664 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2665 $v->{file
} = $volid;
2666 $value = print_drive
($v);
2668 warn "vm $vmid - unable to parse value of '$key'\n";
2673 $conf->{$key} = $value;
2678 if (defined($descr)) {
2680 $conf->{description
} = $descr;
2682 delete $res->{snapstate
}; # just to be sure
2687 sub write_vm_config
{
2688 my ($filename, $conf) = @_;
2690 delete $conf->{snapstate
}; # just to be sure
2692 if ($conf->{cdrom
}) {
2693 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2694 $conf->{ide2
} = $conf->{cdrom
};
2695 delete $conf->{cdrom
};
2698 # we do not use 'smp' any longer
2699 if ($conf->{sockets
}) {
2700 delete $conf->{smp
};
2701 } elsif ($conf->{smp
}) {
2702 $conf->{sockets
} = $conf->{smp
};
2703 delete $conf->{cores
};
2704 delete $conf->{smp
};
2707 my $used_volids = {};
2709 my $cleanup_config = sub {
2710 my ($cref, $pending, $snapname) = @_;
2712 foreach my $key (keys %$cref) {
2713 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2714 $key eq 'snapstate' || $key eq 'pending';
2715 my $value = $cref->{$key};
2716 if ($key eq 'delete') {
2717 die "propertry 'delete' is only allowed in [PENDING]\n"
2719 # fixme: check syntax?
2722 eval { $value = check_type
($key, $value); };
2723 die "unable to parse value of '$key' - $@" if $@;
2725 $cref->{$key} = $value;
2727 if (!$snapname && is_valid_drivename
($key)) {
2728 my $drive = parse_drive
($key, $value);
2729 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2734 &$cleanup_config($conf);
2736 &$cleanup_config($conf->{pending
}, 1);
2738 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2739 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2740 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2743 # remove 'unusedX' settings if we re-add a volume
2744 foreach my $key (keys %$conf) {
2745 my $value = $conf->{$key};
2746 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2747 delete $conf->{$key};
2751 my $generate_raw_config = sub {
2752 my ($conf, $pending) = @_;
2756 # add description as comment to top of file
2757 if (defined(my $descr = $conf->{description
})) {
2759 foreach my $cl (split(/\n/, $descr)) {
2760 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2763 $raw .= "#\n" if $pending;
2767 foreach my $key (sort keys %$conf) {
2768 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2769 $raw .= "$key: $conf->{$key}\n";
2774 my $raw = &$generate_raw_config($conf);
2776 if (scalar(keys %{$conf->{pending
}})){
2777 $raw .= "\n[PENDING]\n";
2778 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2781 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2782 $raw .= "\n[$snapname]\n";
2783 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2793 # we use static defaults from our JSON schema configuration
2794 foreach my $key (keys %$confdesc) {
2795 if (defined(my $default = $confdesc->{$key}->{default})) {
2796 $res->{$key} = $default;
2804 my $vmlist = PVE
::Cluster
::get_vmlist
();
2806 return $res if !$vmlist || !$vmlist->{ids
};
2807 my $ids = $vmlist->{ids
};
2809 foreach my $vmid (keys %$ids) {
2810 my $d = $ids->{$vmid};
2811 next if !$d->{node
} || $d->{node
} ne $nodename;
2812 next if !$d->{type
} || $d->{type
} ne 'qemu';
2813 $res->{$vmid}->{exists} = 1;
2818 # test if VM uses local resources (to prevent migration)
2819 sub check_local_resources
{
2820 my ($conf, $noerr) = @_;
2824 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2825 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2827 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2829 foreach my $k (keys %$conf) {
2830 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2831 # sockets are safe: they will recreated be on the target side post-migrate
2832 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2833 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2836 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2841 # check if used storages are available on all nodes (use by migrate)
2842 sub check_storage_availability
{
2843 my ($storecfg, $conf, $node) = @_;
2845 foreach_drive
($conf, sub {
2846 my ($ds, $drive) = @_;
2848 my $volid = $drive->{file
};
2851 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2854 # check if storage is available on both nodes
2855 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2856 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2860 # list nodes where all VM images are available (used by has_feature API)
2862 my ($conf, $storecfg) = @_;
2864 my $nodelist = PVE
::Cluster
::get_nodelist
();
2865 my $nodehash = { map { $_ => 1 } @$nodelist };
2866 my $nodename = PVE
::INotify
::nodename
();
2868 foreach_drive
($conf, sub {
2869 my ($ds, $drive) = @_;
2871 my $volid = $drive->{file
};
2874 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2876 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2877 if ($scfg->{disable
}) {
2879 } elsif (my $avail = $scfg->{nodes
}) {
2880 foreach my $node (keys %$nodehash) {
2881 delete $nodehash->{$node} if !$avail->{$node};
2883 } elsif (!$scfg->{shared
}) {
2884 foreach my $node (keys %$nodehash) {
2885 delete $nodehash->{$node} if $node ne $nodename
2894 sub check_local_storage_availability
{
2895 my ($conf, $storecfg) = @_;
2897 my $nodelist = PVE
::Cluster
::get_nodelist
();
2898 my $nodehash = { map { $_ => {} } @$nodelist };
2900 foreach_drive
($conf, sub {
2901 my ($ds, $drive) = @_;
2903 my $volid = $drive->{file
};
2906 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2908 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2910 if ($scfg->{disable
}) {
2911 foreach my $node (keys %$nodehash) {
2912 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2914 } elsif (my $avail = $scfg->{nodes
}) {
2915 foreach my $node (keys %$nodehash) {
2916 if (!$avail->{$node}) {
2917 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2924 foreach my $node (values %$nodehash) {
2925 if (my $unavail = $node->{unavailable_storages
}) {
2926 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2933 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2935 my ($vmid, $nocheck, $node) = @_;
2937 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2938 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2943 my $vzlist = config_list
();
2945 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2947 while (defined(my $de = $fd->read)) {
2948 next if $de !~ m/^(\d+)\.pid$/;
2950 next if !defined($vzlist->{$vmid});
2951 if (my $pid = check_running
($vmid)) {
2952 $vzlist->{$vmid}->{pid
} = $pid;
2960 my ($storecfg, $conf) = @_;
2962 my $bootdisk = $conf->{bootdisk
};
2963 return undef if !$bootdisk;
2964 return undef if !is_valid_drivename
($bootdisk);
2966 return undef if !$conf->{$bootdisk};
2968 my $drive = parse_drive
($bootdisk, $conf->{$bootdisk});
2969 return undef if !defined($drive);
2971 return undef if drive_is_cdrom
($drive);
2973 my $volid = $drive->{file
};
2974 return undef if !$volid;
2976 return $drive->{size
};
2979 our $vmstatus_return_properties = {
2980 vmid
=> get_standard_option
('pve-vmid'),
2982 description
=> "Qemu process status.",
2984 enum
=> ['stopped', 'running'],
2987 description
=> "Maximum memory in bytes.",
2990 renderer
=> 'bytes',
2993 description
=> "Root disk size in bytes.",
2996 renderer
=> 'bytes',
2999 description
=> "VM name.",
3004 description
=> "Qemu QMP agent status.",
3009 description
=> "PID of running qemu process.",
3014 description
=> "Uptime.",
3017 renderer
=> 'duration',
3020 description
=> "Maximum usable CPUs.",
3025 description
=> "The current config lock, if any.",
3030 description
=> "The current configured tags, if any",
3036 my $last_proc_pid_stat;
3038 # get VM status information
3039 # This must be fast and should not block ($full == false)
3040 # We only query KVM using QMP if $full == true (this can be slow)
3042 my ($opt_vmid, $full) = @_;
3046 my $storecfg = PVE
::Storage
::config
();
3048 my $list = vzlist
();
3049 my $defaults = load_defaults
();
3051 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
3053 my $cpucount = $cpuinfo->{cpus
} || 1;
3055 foreach my $vmid (keys %$list) {
3056 next if $opt_vmid && ($vmid ne $opt_vmid);
3058 my $conf = PVE
::QemuConfig-
>load_config($vmid);
3060 my $d = { vmid
=> $vmid };
3061 $d->{pid
} = $list->{$vmid}->{pid
};
3063 # fixme: better status?
3064 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
3066 my $size = disksize
($storecfg, $conf);
3067 if (defined($size)) {
3068 $d->{disk
} = 0; # no info available
3069 $d->{maxdisk
} = $size;
3075 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
3076 * ($conf->{cores
} || $defaults->{cores
});
3077 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
3078 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
3080 $d->{name
} = $conf->{name
} || "VM $vmid";
3081 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
3082 : $defaults->{memory
}*(1024*1024);
3084 if ($conf->{balloon
}) {
3085 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
3086 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
3087 : $defaults->{shares
};
3098 $d->{diskwrite
} = 0;
3100 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
3102 $d->{serial
} = 1 if conf_has_serial
($conf);
3103 $d->{lock} = $conf->{lock} if $conf->{lock};
3104 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
3109 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
3110 foreach my $dev (keys %$netdev) {
3111 next if $dev !~ m/^tap([1-9]\d*)i/;
3113 my $d = $res->{$vmid};
3116 $d->{netout
} += $netdev->{$dev}->{receive
};
3117 $d->{netin
} += $netdev->{$dev}->{transmit
};
3120 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
3121 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
3126 my $ctime = gettimeofday
;
3128 foreach my $vmid (keys %$list) {
3130 my $d = $res->{$vmid};
3131 my $pid = $d->{pid
};
3134 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
3135 next if !$pstat; # not running
3137 my $used = $pstat->{utime} + $pstat->{stime
};
3139 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
3141 if ($pstat->{vsize
}) {
3142 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
3145 my $old = $last_proc_pid_stat->{$pid};
3147 $last_proc_pid_stat->{$pid} = {
3155 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
3157 if ($dtime > 1000) {
3158 my $dutime = $used - $old->{used
};
3160 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
3161 $last_proc_pid_stat->{$pid} = {
3167 $d->{cpu
} = $old->{cpu
};
3171 return $res if !$full;
3173 my $qmpclient = PVE
::QMPClient-
>new();
3175 my $ballooncb = sub {
3176 my ($vmid, $resp) = @_;
3178 my $info = $resp->{'return'};
3179 return if !$info->{max_mem
};
3181 my $d = $res->{$vmid};
3183 # use memory assigned to VM
3184 $d->{maxmem
} = $info->{max_mem
};
3185 $d->{balloon
} = $info->{actual
};
3187 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
3188 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
3189 $d->{freemem
} = $info->{free_mem
};
3192 $d->{ballooninfo
} = $info;
3195 my $blockstatscb = sub {
3196 my ($vmid, $resp) = @_;
3197 my $data = $resp->{'return'} || [];
3198 my $totalrdbytes = 0;
3199 my $totalwrbytes = 0;
3201 for my $blockstat (@$data) {
3202 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
3203 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
3205 $blockstat->{device
} =~ s/drive-//;
3206 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
3208 $res->{$vmid}->{diskread
} = $totalrdbytes;
3209 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
3212 my $statuscb = sub {
3213 my ($vmid, $resp) = @_;
3215 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3216 # this fails if ballon driver is not loaded, so this must be
3217 # the last commnand (following command are aborted if this fails).
3218 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3220 my $status = 'unknown';
3221 if (!defined($status = $resp->{'return'}->{status
})) {
3222 warn "unable to get VM status\n";
3226 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3229 foreach my $vmid (keys %$list) {
3230 next if $opt_vmid && ($vmid ne $opt_vmid);
3231 next if !$res->{$vmid}->{pid
}; # not running
3232 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3235 $qmpclient->queue_execute(undef, 2);
3237 foreach my $vmid (keys %$list) {
3238 next if $opt_vmid && ($vmid ne $opt_vmid);
3239 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3246 my ($conf, $func, @param) = @_;
3248 foreach my $ds (valid_drive_names
()) {
3249 next if !defined($conf->{$ds});
3251 my $drive = parse_drive
($ds, $conf->{$ds});
3254 &$func($ds, $drive, @param);
3259 my ($conf, $func, @param) = @_;
3263 my $test_volid = sub {
3264 my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3268 $volhash->{$volid}->{cdrom
} //= 1;
3269 $volhash->{$volid}->{cdrom
} = 0 if !$is_cdrom;
3271 $volhash->{$volid}->{replicate
} //= 0;
3272 $volhash->{$volid}->{replicate
} = 1 if $replicate;
3274 $volhash->{$volid}->{shared
} //= 0;
3275 $volhash->{$volid}->{shared
} = 1 if $shared;
3277 $volhash->{$volid}->{referenced_in_config
} //= 0;
3278 $volhash->{$volid}->{referenced_in_config
} = 1 if !defined($snapname);
3280 $volhash->{$volid}->{referenced_in_snapshot
}->{$snapname} = 1
3281 if defined($snapname);
3282 $volhash->{$volid}->{size
} = $size if $size;
3285 foreach_drive
($conf, sub {
3286 my ($ds, $drive) = @_;
3287 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, undef, $drive->{size
});
3290 foreach my $snapname (keys %{$conf->{snapshots
}}) {
3291 my $snap = $conf->{snapshots
}->{$snapname};
3292 $test_volid->($snap->{vmstate
}, 0, 1, $snapname);
3293 foreach_drive
($snap, sub {
3294 my ($ds, $drive) = @_;
3295 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, $snapname);
3299 foreach my $volid (keys %$volhash) {
3300 &$func($volid, $volhash->{$volid}, @param);
3304 sub conf_has_serial
{
3307 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3308 if ($conf->{"serial$i"}) {
3316 sub conf_has_audio
{
3317 my ($conf, $id) = @_;
3320 my $audio = $conf->{"audio$id"};
3321 return undef if !defined($audio);
3323 my $audioproperties = PVE
::JSONSchema
::parse_property_string
($audio_fmt, $audio);
3324 my $audiodriver = $audioproperties->{driver
} // 'spice';
3327 dev
=> $audioproperties->{device
},
3328 dev_id
=> "audiodev$id",
3329 backend
=> $audiodriver,
3330 backend_id
=> "$audiodriver-backend${id}",
3334 sub vga_conf_has_spice
{
3337 my $vgaconf = parse_vga
($vga);
3338 my $vgatype = $vgaconf->{type
};
3339 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3346 return get_host_arch
() eq $arch;
3351 return $conf->{arch
} // get_host_arch
();
3354 my $default_machines = {
3359 sub get_vm_machine
{
3360 my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
3362 my $machine = $forcemachine || $conf->{machine
};
3364 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3366 $machine ||= $default_machines->{$arch};
3367 $machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
3373 sub get_ovmf_files
($) {
3376 my $ovmf = $OVMF->{$arch}
3377 or die "no OVMF images known for architecture '$arch'\n";
3383 aarch64
=> '/usr/bin/qemu-system-aarch64',
3384 x86_64
=> '/usr/bin/qemu-system-x86_64',
3386 sub get_command_for_arch
($) {
3388 return '/usr/bin/kvm' if is_native
($arch);
3390 my $cmd = $Arch2Qemu->{$arch}
3391 or die "don't know how to emulate architecture '$arch'\n";
3395 sub get_cpu_options
{
3396 my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
3399 my $ostype = $conf->{ostype
};
3401 my $cpu = $kvm ?
"kvm64" : "qemu64";
3402 if ($arch eq 'aarch64') {
3403 $cpu = 'cortex-a57';
3406 if (my $cputype = $conf->{cpu
}) {
3407 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
3408 or die "Cannot parse cpu description: $cputype\n";
3409 $cpu = $cpuconf->{cputype
};
3410 $kvm_off = 1 if $cpuconf->{hidden
};
3411 $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3413 if (defined(my $flags = $cpuconf->{flags
})) {
3414 push @$cpuFlags, split(";", $flags);
3418 push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3420 push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
3422 push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3424 push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3426 if (min_version
($machine_version, 2, 3) && $arch eq 'x86_64') {
3428 push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3429 push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3432 add_hyperv_enlightenments
($cpuFlags, $winversion, $machine_version, $conf->{bios
}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3434 push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3436 push @$cpuFlags, 'kvm=off' if $kvm_off;
3438 if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3439 push @$cpuFlags, "vendor=${cpu_vendor}"
3440 if $cpu_vendor ne 'default';
3441 } elsif ($arch ne 'aarch64') {
3442 die "internal error"; # should not happen
3445 $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3447 return ('-cpu', $cpu);
3450 sub config_to_command
{
3451 my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3454 my $globalFlags = [];
3455 my $machineFlags = [];
3460 my $vernum = 0; # unknown
3461 my $ostype = $conf->{ostype
};
3462 my $winversion = windows_version
($ostype);
3463 my $kvm = $conf->{kvm
};
3465 my $arch = get_vm_arch
($conf);
3466 my $kvm_binary = get_command_for_arch
($arch);
3467 my $kvmver = kvm_user_version
($kvm_binary);
3469 my $add_pve_version = min_version
($kvmver, 4, 1);
3471 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3472 my $machine_version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, $kvmver);
3473 $kvm //= 1 if is_native
($arch);
3475 $machine_version =~ m/(\d+)\.(\d+)/;
3476 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
3477 if !PVE
::QemuServer
::min_version
($kvmver, $1, $2);
3480 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3481 if !defined kvm_version
();
3484 if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3485 $vernum = $1*1000000+$2*1000;
3486 } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3487 $vernum = $1*1000000+$2*1000+$3;
3490 die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3492 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3493 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3494 my $use_old_bios_files = undef;
3495 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3497 my $cpuunits = defined($conf->{cpuunits
}) ?
3498 $conf->{cpuunits
} : $defaults->{cpuunits
};
3500 push @$cmd, $kvm_binary;
3502 push @$cmd, '-id', $vmid;
3504 my $vmname = $conf->{name
} || "vm$vmid";
3506 push @$cmd, '-name', $vmname;
3510 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3511 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3512 push @$cmd, '-mon', "chardev=qmp,mode=control";
3514 if (min_version
($machine_version, 2, 12)) {
3515 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3516 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3519 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3521 push @$cmd, '-daemonize';
3523 if ($conf->{smbios1
}) {
3524 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3525 if ($smbios_conf->{base64
}) {
3526 # Do not pass base64 flag to qemu
3527 delete $smbios_conf->{base64
};
3528 my $smbios_string = "";
3529 foreach my $key (keys %$smbios_conf) {
3531 if ($key eq "uuid") {
3532 $value = $smbios_conf->{uuid
}
3534 $value = decode_base64
($smbios_conf->{$key});
3536 # qemu accepts any binary data, only commas need escaping by double comma
3538 $smbios_string .= "," . $key . "=" . $value if $value;
3540 push @$cmd, '-smbios', "type=1" . $smbios_string;
3542 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3546 if ($conf->{vmgenid
}) {
3547 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3550 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3551 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3552 die "uefi base image not found\n" if ! -f
$ovmf_code;
3556 if (my $efidisk = $conf->{efidisk0
}) {
3557 my $d = PVE
::JSONSchema
::parse_property_string
($efidisk_fmt, $efidisk);
3558 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3559 $format = $d->{format
};
3561 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3562 if (!defined($format)) {
3563 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3564 $format = qemu_img_format
($scfg, $volname);
3568 die "efidisk format must be specified\n"
3569 if !defined($format);
3572 warn "no efidisk configured! Using temporary efivars disk.\n";
3573 $path = "/tmp/$vmid-ovmf.fd";
3574 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3578 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3579 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3584 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3585 if (min_version
($machine_version, 4, 0)) {
3586 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3588 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3592 # add usb controllers
3593 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
($conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3594 push @$devices, @usbcontrollers if @usbcontrollers;
3595 my $vga = parse_vga
($conf->{vga
});
3597 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3598 $vga->{type
} = 'qxl' if $qxlnum;
3600 if (!$vga->{type
}) {
3601 if ($arch eq 'aarch64') {
3602 $vga->{type
} = 'virtio';
3603 } elsif (min_version
($machine_version, 2, 9)) {
3604 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3606 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3610 # enable absolute mouse coordinates (needed by vnc)
3612 if (defined($conf->{tablet
})) {
3613 $tablet = $conf->{tablet
};
3615 $tablet = $defaults->{tablet
};
3616 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3617 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3621 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3622 my $kbd = print_keyboarddevice_full
($conf, $arch);
3623 push @$devices, '-device', $kbd if defined($kbd);
3627 my $gpu_passthrough;
3630 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
3631 my $id = "hostpci$i";
3632 my $d = parse_hostpci
($conf->{$id});
3635 if (my $pcie = $d->{pcie
}) {
3636 die "q35 machine model is not enabled" if !$q35;
3637 # win7 wants to have the pcie devices directly on the pcie bus
3638 # instead of in the root port
3639 if ($winversion == 7) {
3640 $pciaddr = print_pcie_addr
("${id}bus0");
3642 # add more root ports if needed, 4 are present by default
3643 # by pve-q35 cfgs, rest added here on demand.
3645 push @$devices, '-device', print_pcie_root_port
($i);
3647 $pciaddr = print_pcie_addr
($id);
3650 $pciaddr = print_pci_addr
($id, $bridges, $arch, $machine_type);
3654 if ($d->{'x-vga'}) {
3655 $xvga = ',x-vga=on' if !($conf->{bios
} && $conf->{bios
} eq 'ovmf');
3657 $vga->{type
} = 'none' if !defined($conf->{vga
});
3658 $gpu_passthrough = 1;
3661 my $pcidevices = $d->{pciid
};
3662 my $multifunction = 1 if @$pcidevices > 1;
3665 if ($d->{mdev
} && scalar(@$pcidevices) == 1) {
3666 my $pci_id = $pcidevices->[0]->{id
};
3667 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $i);
3668 $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3669 } elsif ($d->{mdev
}) {
3670 warn "ignoring mediated device '$id' with multifunction device\n";
3674 foreach my $pcidevice (@$pcidevices) {
3675 my $devicestr = "vfio-pci";
3678 $devicestr .= ",sysfsdev=$sysfspath";
3680 $devicestr .= ",host=$pcidevice->{id}";
3683 my $mf_addr = $multifunction ?
".$j" : '';
3684 $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3687 $devicestr .= ',rombar=0' if defined($d->{rombar
}) && !$d->{rombar
};
3688 $devicestr .= "$xvga";
3689 $devicestr .= ",multifunction=on" if $multifunction;
3690 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile
};
3693 push @$devices, '-device', $devicestr;
3699 my $usb_dev_features = {};
3700 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3702 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
($conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features);
3703 push @$devices, @usbdevices if @usbdevices;
3705 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3706 if (my $path = $conf->{"serial$i"}) {
3707 if ($path eq 'socket') {
3708 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3709 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3710 # On aarch64, serial0 is the UART device. Qemu only allows
3711 # connecting UART devices via the '-serial' command line, as
3712 # the device has a fixed slot on the hardware...
3713 if ($arch eq 'aarch64' && $i == 0) {
3714 push @$devices, '-serial', "chardev:serial$i";
3716 push @$devices, '-device', "isa-serial,chardev=serial$i";
3719 die "no such serial device\n" if ! -c
$path;
3720 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3721 push @$devices, '-device', "isa-serial,chardev=serial$i";
3727 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3728 if (my $path = $conf->{"parallel$i"}) {
3729 die "no such parallel device\n" if ! -c
$path;
3730 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3731 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3732 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3736 if (my $audio = conf_has_audio
($conf)) {
3738 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3740 my $id = $audio->{dev_id
};
3741 if ($audio->{dev
} eq 'AC97') {
3742 push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3743 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3744 push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3745 push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3746 push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3748 die "unkown audio device '$audio->{dev}', implement me!";
3751 push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3755 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3756 $sockets = $conf->{sockets
} if $conf->{sockets
};
3758 my $cores = $conf->{cores
} || 1;
3760 my $maxcpus = $sockets * $cores;
3762 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3764 my $allowed_vcpus = $cpuinfo->{cpus
};
3766 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3767 if ($allowed_vcpus < $maxcpus);
3769 if($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3771 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3772 for (my $i = 2; $i <= $vcpus; $i++) {
3773 my $cpustr = print_cpu_device
($conf,$i);
3774 push @$cmd, '-device', $cpustr;
3779 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3781 push @$cmd, '-nodefaults';
3783 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
3785 my $bootindex_hash = {};
3787 foreach my $o (split(//, $bootorder)) {
3788 $bootindex_hash->{$o} = $i*100;
3792 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3794 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3796 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3798 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3799 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3800 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3801 push @$cmd, '-vnc', "unix:$socket,password";
3803 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3804 push @$cmd, '-nographic';
3808 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3810 my $useLocaltime = $conf->{localtime};
3812 if ($winversion >= 5) { # windows
3813 $useLocaltime = 1 if !defined($conf->{localtime});
3815 # use time drift fix when acpi is enabled
3816 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3817 $tdf = 1 if !defined($conf->{tdf
});
3821 if ($winversion >= 6) {
3822 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3823 push @$cmd, '-no-hpet';
3826 push @$rtcFlags, 'driftfix=slew' if $tdf;
3829 push @$machineFlags, 'accel=tcg';
3832 if ($machine_type) {
3833 push @$machineFlags, "type=${machine_type}";
3836 if (($conf->{startdate
}) && ($conf->{startdate
} ne 'now')) {
3837 push @$rtcFlags, "base=$conf->{startdate}";
3838 } elsif ($useLocaltime) {
3839 push @$rtcFlags, 'base=localtime';
3842 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3844 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3846 push @$cmd, '-S' if $conf->{freeze
};
3848 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3850 my $guest_agent = parse_guest_agent
($conf);
3852 if ($guest_agent->{enabled
}) {
3853 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3854 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3856 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3857 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3858 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3859 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3860 } elsif ($guest_agent->{type
} eq 'isa') {
3861 push @$devices, '-device', "isa-serial,chardev=qga0";
3870 for(my $i = 1; $i < $qxlnum; $i++){
3871 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3874 # assume other OS works like Linux
3875 my ($ram, $vram) = ("134217728", "67108864");
3876 if ($vga->{memory
}) {
3877 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3878 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3880 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3881 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3885 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3887 my $nodename = PVE
::INotify
::nodename
();
3888 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3889 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3890 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3892 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3893 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3894 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3896 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3897 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3899 my $spice_enhancement = PVE
::JSONSchema
::parse_property_string
($spice_enhancements_fmt, $conf->{spice_enhancements
} // '');
3900 if ($spice_enhancement->{foldersharing
}) {
3901 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3902 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3905 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3906 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming
};
3907 push @$devices, '-spice', "$spice_opts";
3910 # enable balloon by default, unless explicitly disabled
3911 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3912 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3913 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3916 if ($conf->{watchdog
}) {
3917 my $wdopts = parse_watchdog
($conf->{watchdog
});
3918 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3919 my $watchdog = $wdopts->{model
} || 'i6300esb';
3920 push @$devices, '-device', "$watchdog$pciaddr";
3921 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3925 my $scsicontroller = {};
3926 my $ahcicontroller = {};
3927 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3929 # Add iscsi initiator name if available
3930 if (my $initiator = get_initiator_name
()) {
3931 push @$devices, '-iscsi', "initiator-name=$initiator";
3934 foreach_drive
($conf, sub {
3935 my ($ds, $drive) = @_;
3937 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3938 push @$vollist, $drive->{file
};
3941 # ignore efidisk here, already added in bios/fw handling code above
3942 return if $drive->{interface
} eq 'efidisk';
3944 $use_virtio = 1 if $ds =~ m/^virtio/;
3946 if (drive_is_cdrom
($drive)) {
3947 if ($bootindex_hash->{d
}) {
3948 $drive->{bootindex
} = $bootindex_hash->{d
};
3949 $bootindex_hash->{d
} += 1;
3952 if ($bootindex_hash->{c
}) {
3953 $drive->{bootindex
} = $bootindex_hash->{c
} if $conf->{bootdisk
} && ($conf->{bootdisk
} eq $ds);
3954 $bootindex_hash->{c
} += 1;
3958 if($drive->{interface
} eq 'virtio'){
3959 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3962 if ($drive->{interface
} eq 'scsi') {
3964 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3966 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3967 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3970 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3971 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3972 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3973 } elsif ($drive->{iothread
}) {
3974 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3978 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3979 $queues = ",num_queues=$drive->{queues}";
3982 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3983 $scsicontroller->{$controller}=1;
3986 if ($drive->{interface
} eq 'sata') {
3987 my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3988 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3989 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3990 $ahcicontroller->{$controller}=1;
3993 my $drive_cmd = print_drive_full
($storecfg, $vmid, $drive);
3994 push @$devices, '-drive',$drive_cmd;
3995 push @$devices, '-device', print_drivedevice_full
($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3998 for (my $i = 0; $i < $MAX_NETS; $i++) {
3999 next if !$conf->{"net$i"};
4000 my $d = parse_net
($conf->{"net$i"});
4003 $use_virtio = 1 if $d->{model
} eq 'virtio';
4005 if ($bootindex_hash->{n
}) {
4006 $d->{bootindex
} = $bootindex_hash->{n
};
4007 $bootindex_hash->{n
} += 1;
4010 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, "net$i");
4011 push @$devices, '-netdev', $netdevfull;
4013 my $netdevicefull = print_netdevice_full
($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4014 push @$devices, '-device', $netdevicefull;
4017 if ($conf->{ivshmem
}) {
4018 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4022 $bus = print_pcie_addr
("ivshmem");
4024 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4027 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4028 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4030 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4031 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4036 if (min_version
($machine_version, 2, 3)) {
4041 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4043 for my $k (sort {$b cmp $a} keys %$bridges) {
4044 $pciaddr = print_pci_addr
("pci.$k", undef, $arch, $machine_type);
4045 unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4049 push @$cmd, @$devices;
4050 push @$cmd, '-rtc', join(',', @$rtcFlags)
4051 if scalar(@$rtcFlags);
4052 push @$cmd, '-machine', join(',', @$machineFlags)
4053 if scalar(@$machineFlags);
4054 push @$cmd, '-global', join(',', @$globalFlags)
4055 if scalar(@$globalFlags);
4057 if (my $vmstate = $conf->{vmstate
}) {
4058 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4059 push @$vollist, $vmstate;
4060 push @$cmd, '-loadstate', $statepath;
4061 print "activating and using '$vmstate' as vmstate\n";
4065 if ($conf->{args
}) {
4066 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4070 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
4076 my $res = mon_cmd
($vmid, 'query-spice');
4078 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4081 sub vm_devices_list
{
4084 my $res = mon_cmd
($vmid, 'query-pci');
4085 my $devices_to_check = [];
4087 foreach my $pcibus (@$res) {
4088 push @$devices_to_check, @{$pcibus->{devices
}},
4091 while (@$devices_to_check) {
4093 for my $d (@$devices_to_check) {
4094 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4095 next if !$d->{'pci_bridge'};
4097 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4098 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4100 $devices_to_check = $to_check;
4103 my $resblock = mon_cmd
($vmid, 'query-block');
4104 foreach my $block (@$resblock) {
4105 if($block->{device
} =~ m/^drive-(\S+)/){
4110 my $resmice = mon_cmd
($vmid, 'query-mice');
4111 foreach my $mice (@$resmice) {
4112 if ($mice->{name
} eq 'QEMU HID Tablet') {
4113 $devices->{tablet
} = 1;
4118 # for usb devices there is no query-usb
4119 # but we can iterate over the entries in
4120 # qom-list path=/machine/peripheral
4121 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4122 foreach my $per (@$resperipheral) {
4123 if ($per->{name
} =~ m/^usb\d+$/) {
4124 $devices->{$per->{name
}} = 1;
4132 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4134 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4136 my $devices_list = vm_devices_list
($vmid);
4137 return 1 if defined($devices_list->{$deviceid});
4139 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4141 if ($deviceid eq 'tablet') {
4143 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4145 } elsif ($deviceid eq 'keyboard') {
4147 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4149 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4151 die "usb hotplug currently not reliable\n";
4152 # since we can't reliably hot unplug all added usb devices
4153 # and usb passthrough disables live migration
4154 # we disable usb hotplugging for now
4155 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device));
4157 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4159 qemu_iothread_add
($vmid, $deviceid, $device);
4161 qemu_driveadd
($storecfg, $vmid, $device);
4162 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4164 qemu_deviceadd
($vmid, $devicefull);
4165 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4167 eval { qemu_drivedel
($vmid, $deviceid); };
4172 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4175 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4176 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4177 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4179 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4181 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4182 qemu_iothread_add
($vmid, $deviceid, $device);
4183 $devicefull .= ",iothread=iothread-$deviceid";
4186 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4187 $devicefull .= ",num_queues=$device->{queues}";
4190 qemu_deviceadd
($vmid, $devicefull);
4191 qemu_deviceaddverify
($vmid, $deviceid);
4193 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4195 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4196 qemu_driveadd
($storecfg, $vmid, $device);
4198 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4199 eval { qemu_deviceadd
($vmid, $devicefull); };
4201 eval { qemu_drivedel
($vmid, $deviceid); };
4206 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4208 return undef if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4210 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4211 my $use_old_bios_files = undef;
4212 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4214 my $netdevicefull = print_netdevice_full
($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4215 qemu_deviceadd
($vmid, $netdevicefull);
4217 qemu_deviceaddverify
($vmid, $deviceid);
4218 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4221 eval { qemu_netdevdel
($vmid, $deviceid); };
4226 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4229 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4230 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4232 qemu_deviceadd
($vmid, $devicefull);
4233 qemu_deviceaddverify
($vmid, $deviceid);
4236 die "can't hotplug device '$deviceid'\n";
4242 # fixme: this should raise exceptions on error!
4243 sub vm_deviceunplug
{
4244 my ($vmid, $conf, $deviceid) = @_;
4246 my $devices_list = vm_devices_list
($vmid);
4247 return 1 if !defined($devices_list->{$deviceid});
4249 die "can't unplug bootdisk" if $conf->{bootdisk
} && $conf->{bootdisk
} eq $deviceid;
4251 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4253 qemu_devicedel
($vmid, $deviceid);
4255 } elsif ($deviceid =~ m/^usb\d+$/) {
4257 die "usb hotplug currently not reliable\n";
4258 # when unplugging usb devices this way,
4259 # there may be remaining usb controllers/hubs
4260 # so we disable it for now
4261 qemu_devicedel
($vmid, $deviceid);
4262 qemu_devicedelverify
($vmid, $deviceid);
4264 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4266 qemu_devicedel
($vmid, $deviceid);
4267 qemu_devicedelverify
($vmid, $deviceid);
4268 qemu_drivedel
($vmid, $deviceid);
4269 qemu_iothread_del
($conf, $vmid, $deviceid);
4271 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4273 qemu_devicedel
($vmid, $deviceid);
4274 qemu_devicedelverify
($vmid, $deviceid);
4275 qemu_iothread_del
($conf, $vmid, $deviceid);
4277 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4279 qemu_devicedel
($vmid, $deviceid);
4280 qemu_drivedel
($vmid, $deviceid);
4281 qemu_deletescsihw
($conf, $vmid, $deviceid);
4283 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4285 qemu_devicedel
($vmid, $deviceid);
4286 qemu_devicedelverify
($vmid, $deviceid);
4287 qemu_netdevdel
($vmid, $deviceid);
4290 die "can't unplug device '$deviceid'\n";
4296 sub qemu_deviceadd
{
4297 my ($vmid, $devicefull) = @_;
4299 $devicefull = "driver=".$devicefull;
4300 my %options = split(/[=,]/, $devicefull);
4302 mon_cmd
($vmid, "device_add" , %options);
4305 sub qemu_devicedel
{
4306 my ($vmid, $deviceid) = @_;
4308 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4311 sub qemu_iothread_add
{
4312 my($vmid, $deviceid, $device) = @_;
4314 if ($device->{iothread
}) {
4315 my $iothreads = vm_iothreads_list
($vmid);
4316 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4320 sub qemu_iothread_del
{
4321 my($conf, $vmid, $deviceid) = @_;
4323 my $confid = $deviceid;
4324 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4325 $confid = 'scsi' . $1;
4327 my $device = parse_drive
($confid, $conf->{$confid});
4328 if ($device->{iothread
}) {
4329 my $iothreads = vm_iothreads_list
($vmid);
4330 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4334 sub qemu_objectadd
{
4335 my($vmid, $objectid, $qomtype) = @_;
4337 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4342 sub qemu_objectdel
{
4343 my($vmid, $objectid) = @_;
4345 mon_cmd
($vmid, "object-del", id
=> $objectid);
4351 my ($storecfg, $vmid, $device) = @_;
4353 my $drive = print_drive_full
($storecfg, $vmid, $device);
4354 $drive =~ s/\\/\\\\/g;
4355 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4357 # If the command succeeds qemu prints: "OK
"
4358 return 1 if $ret =~ m/OK/s;
4360 die "adding drive failed
: $ret\n";
4364 my($vmid, $deviceid) = @_;
4366 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4369 return 1 if $ret eq "";
4371 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4372 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4374 die "deleting drive
$deviceid failed
: $ret\n";
4377 sub qemu_deviceaddverify {
4378 my ($vmid, $deviceid) = @_;
4380 for (my $i = 0; $i <= 5; $i++) {
4381 my $devices_list = vm_devices_list($vmid);
4382 return 1 if defined($devices_list->{$deviceid});
4386 die "error on hotplug device
'$deviceid'\n";
4390 sub qemu_devicedelverify {
4391 my ($vmid, $deviceid) = @_;
4393 # need to verify that the device is correctly removed as device_del
4394 # is async and empty return is not reliable
4396 for (my $i = 0; $i <= 5; $i++) {
4397 my $devices_list = vm_devices_list($vmid);
4398 return 1 if !defined($devices_list->{$deviceid});
4402 die "error on hot-unplugging device
'$deviceid'\n";
4405 sub qemu_findorcreatescsihw {
4406 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4408 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4410 my $scsihwid="$controller_prefix$controller";
4411 my $devices_list = vm_devices_list($vmid);
4413 if(!defined($devices_list->{$scsihwid})) {
4414 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4420 sub qemu_deletescsihw {
4421 my ($conf, $vmid, $opt) = @_;
4423 my $device = parse_drive($opt, $conf->{$opt});
4425 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4426 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4430 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4432 my $devices_list = vm_devices_list($vmid);
4433 foreach my $opt (keys %{$devices_list}) {
4434 if (PVE::QemuServer::is_valid_drivename($opt)) {
4435 my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4436 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4442 my $scsihwid="scsihw
$controller";
4444 vm_deviceunplug($vmid, $conf, $scsihwid);
4449 sub qemu_add_pci_bridge {
4450 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4456 print_pci_addr($device, $bridges, $arch, $machine_type);
4458 while (my ($k, $v) = each %$bridges) {
4461 return 1 if !defined($bridgeid) || $bridgeid < 1;
4463 my $bridge = "pci
.$bridgeid";
4464 my $devices_list = vm_devices_list($vmid);
4466 if (!defined($devices_list->{$bridge})) {
4467 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4473 sub qemu_set_link_status {
4474 my ($vmid, $device, $up) = @_;
4476 mon_cmd($vmid, "set_link
", name => $device,
4477 up => $up ? JSON::true : JSON::false);
4480 sub qemu_netdevadd {
4481 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4483 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4484 my %options = split(/[=,]/, $netdev);
4486 mon_cmd($vmid, "netdev_add
", %options);
4490 sub qemu_netdevdel {
4491 my ($vmid, $deviceid) = @_;
4493 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4496 sub qemu_usb_hotplug {
4497 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4501 # remove the old one first
4502 vm_deviceunplug($vmid, $conf, $deviceid);
4504 # check if xhci controller is necessary and available
4505 if ($device->{usb3}) {
4507 my $devicelist = vm_devices_list($vmid);
4509 if (!$devicelist->{xhci}) {
4510 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4511 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4514 my $d = parse_usb_device($device->{host});
4515 $d->{usb3} = $device->{usb3};
4518 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4521 sub qemu_cpu_hotplug {
4522 my ($vmid, $conf, $vcpus) = @_;
4524 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4527 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4528 $sockets = $conf->{sockets} if $conf->{sockets};
4529 my $cores = $conf->{cores} || 1;
4530 my $maxcpus = $sockets * $cores;
4532 $vcpus = $maxcpus if !$vcpus;
4534 die "you can
't add more vcpus than maxcpus\n"
4535 if $vcpus > $maxcpus;
4537 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4539 if ($vcpus < $currentvcpus) {
4541 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4543 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4544 qemu_devicedel($vmid, "cpu$i");
4546 my $currentrunningvcpus = undef;
4548 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4549 last if scalar(@{$currentrunningvcpus}) == $i-1;
4550 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4554 #update conf after each succesfull cpu unplug
4555 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4556 PVE::QemuConfig->write_config($vmid, $conf);
4559 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4565 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4566 die "vcpus in running vm does not match its configuration\n"
4567 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4569 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4571 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4572 my $cpustr = print_cpu_device($conf, $i);
4573 qemu_deviceadd($vmid, $cpustr);
4576 my $currentrunningvcpus = undef;
4578 $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
4579 last if scalar(@{$currentrunningvcpus}) == $i;
4580 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4584 #update conf after each succesfull cpu hotplug
4585 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4586 PVE::QemuConfig->write_config($vmid, $conf);
4590 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4591 mon_cmd($vmid, "cpu-add", id => int($i));
4596 sub qemu_block_set_io_throttle {
4597 my ($vmid, $deviceid,
4598 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4599 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4600 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4601 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4603 return if !check_running($vmid) ;
4605 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4607 bps_rd => int($bps_rd),
4608 bps_wr => int($bps_wr),
4610 iops_rd => int($iops_rd),
4611 iops_wr => int($iops_wr),
4612 bps_max => int($bps_max),
4613 bps_rd_max => int($bps_rd_max),
4614 bps_wr_max => int($bps_wr_max),
4615 iops_max => int($iops_max),
4616 iops_rd_max => int($iops_rd_max),
4617 iops_wr_max => int($iops_wr_max),
4618 bps_max_length => int($bps_max_length),
4619 bps_rd_max_length => int($bps_rd_max_length),
4620 bps_wr_max_length => int($bps_wr_max_length),
4621 iops_max_length => int($iops_max_length),
4622 iops_rd_max_length => int($iops_rd_max_length),
4623 iops_wr_max_length => int($iops_wr_max_length),
4628 # old code, only used to shutdown old VM after update
4630 my ($fh, $timeout) = @_;
4632 my $sel = new IO::Select;
4639 while (scalar (@ready = $sel->can_read($timeout))) {
4641 if ($count = $fh->sysread($buf, 8192)) {
4642 if ($buf =~ /^(.*)\(qemu\) $/s) {
4649 if (!defined($count)) {
4656 die "monitor read timeout\n" if !scalar(@ready);
4661 sub qemu_block_resize {
4662 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4664 my $running = check_running($vmid);
4666 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4668 return if !$running;
4670 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4674 sub qemu_volume_snapshot {
4675 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4677 my $running = check_running($vmid);
4679 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4680 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4682 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4686 sub qemu_volume_snapshot_delete {
4687 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4689 my $running = check_running($vmid);
4694 my $conf = PVE::QemuConfig->load_config($vmid);
4695 foreach_drive($conf, sub {
4696 my ($ds, $drive) = @_;
4697 $running = 1 if $drive->{file} eq $volid;
4701 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4702 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4704 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4708 sub set_migration_caps {
4714 "auto-converge" => 1,
4716 "x-rdma-pin-all" => 0,
4721 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4723 for my $supported_capability (@$supported_capabilities) {
4725 capability => $supported_capability->{capability},
4726 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4730 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4733 my $fast_plug_option = {
4741 'vmstatestorage
' => 1,
4746 # hotplug changes in [PENDING]
4747 # $selection hash can be used to only apply specified options, for
4748 # example: { cores => 1 } (only apply changed 'cores
')
4749 # $errors ref is used to return error messages
4750 sub vmconfig_hotplug_pending {
4751 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4753 my $defaults = load_defaults();
4754 my $arch = get_vm_arch($conf);
4755 my $machine_type = get_vm_machine($conf, undef, $arch);
4757 # commit values which do not have any impact on running VM first
4758 # Note: those option cannot raise errors, we we do not care about
4759 # $selection and always apply them.
4761 my $add_error = sub {
4762 my ($opt, $msg) = @_;
4763 $errors->{$opt} = "hotplug problem - $msg";
4767 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4768 if ($fast_plug_option->{$opt}) {
4769 $conf->{$opt} = $conf->{pending}->{$opt};
4770 delete $conf->{pending}->{$opt};
4776 PVE::QemuConfig->write_config($vmid, $conf);
4777 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4780 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4782 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4783 foreach my $opt (sort keys %$pending_delete_hash) {
4784 next if $selection && !$selection->{$opt};
4785 my $force = $pending_delete_hash->{$opt}->{force};
4787 if ($opt eq 'hotplug
') {
4788 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4789 } elsif ($opt eq 'tablet
') {
4790 die "skip\n" if !$hotplug_features->{usb};
4791 if ($defaults->{tablet}) {
4792 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4793 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4794 if $arch eq 'aarch64
';
4796 vm_deviceunplug($vmid, $conf, 'tablet
');
4797 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4799 } elsif ($opt =~ m/^usb\d+/) {
4801 # since we cannot reliably hot unplug usb devices
4802 # we are disabling it
4803 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4804 vm_deviceunplug($vmid, $conf, $opt);
4805 } elsif ($opt eq 'vcpus
') {
4806 die "skip\n" if !$hotplug_features->{cpu};
4807 qemu_cpu_hotplug($vmid, $conf, undef);
4808 } elsif ($opt eq 'balloon
') {
4809 # enable balloon device is not hotpluggable
4810 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4811 # here we reset the ballooning value to memory
4812 my $balloon = $conf->{memory} || $defaults->{memory};
4813 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4814 } elsif ($fast_plug_option->{$opt}) {
4816 } elsif ($opt =~ m/^net(\d+)$/) {
4817 die "skip\n" if !$hotplug_features->{network};
4818 vm_deviceunplug($vmid, $conf, $opt);
4819 } elsif (is_valid_drivename($opt)) {
4820 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4821 vm_deviceunplug($vmid, $conf, $opt);
4822 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4823 } elsif ($opt =~ m/^memory$/) {
4824 die "skip\n" if !$hotplug_features->{memory};
4825 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4826 } elsif ($opt eq 'cpuunits
') {
4827 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4828 } elsif ($opt eq 'cpulimit
') {
4829 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4835 &$add_error($opt, $err) if $err ne "skip\n";
4837 # save new config if hotplug was successful
4838 delete $conf->{$opt};
4839 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4840 PVE::QemuConfig->write_config($vmid, $conf);
4841 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4845 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4846 $apply_pending_cloudinit = sub {
4847 return if $apply_pending_cloudinit_done; # once is enough
4848 $apply_pending_cloudinit_done = 1; # once is enough
4850 my ($key, $value) = @_;
4852 my @cloudinit_opts = keys %$confdesc_cloudinit;
4853 foreach my $opt (keys %{$conf->{pending}}) {
4854 next if !grep { $_ eq $opt } @cloudinit_opts;
4855 $conf->{$opt} = delete $conf->{pending}->{$opt};
4858 my $new_conf = { %$conf };
4859 $new_conf->{$key} = $value;
4860 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4863 foreach my $opt (keys %{$conf->{pending}}) {
4864 next if $selection && !$selection->{$opt};
4865 my $value = $conf->{pending}->{$opt};
4867 if ($opt eq 'hotplug
') {
4868 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4869 } elsif ($opt eq 'tablet
') {
4870 die "skip\n" if !$hotplug_features->{usb};
4872 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4873 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4874 if $arch eq 'aarch64
';
4875 } elsif ($value == 0) {
4876 vm_deviceunplug($vmid, $conf, 'tablet
');
4877 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4879 } elsif ($opt =~ m/^usb\d+$/) {
4881 # since we cannot reliably hot unplug usb devices
4882 # we are disabling it
4883 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4884 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4885 die "skip\n" if !$d;
4886 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4887 } elsif ($opt eq 'vcpus
') {
4888 die "skip\n" if !$hotplug_features->{cpu};
4889 qemu_cpu_hotplug($vmid, $conf, $value);
4890 } elsif ($opt eq 'balloon
') {
4891 # enable/disable balloning device is not hotpluggable
4892 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4893 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4894 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4896 # allow manual ballooning if shares is set to zero
4897 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4898 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4899 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4901 } elsif ($opt =~ m/^net(\d+)$/) {
4902 # some changes can be done without hotplug
4903 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4904 $vmid, $opt, $value, $arch, $machine_type);
4905 } elsif (is_valid_drivename($opt)) {
4906 die "skip\n" if $opt eq 'efidisk0
';
4907 # some changes can be done without hotplug
4908 my $drive = parse_drive($opt, $value);
4909 if (drive_is_cloudinit($drive)) {
4910 &$apply_pending_cloudinit($opt, $value);
4912 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4913 $vmid, $opt, $value, 1, $arch, $machine_type);
4914 } elsif ($opt =~ m/^memory$/) { #dimms
4915 die "skip\n" if !$hotplug_features->{memory};
4916 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4917 } elsif ($opt eq 'cpuunits
') {
4918 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4919 } elsif ($opt eq 'cpulimit
') {
4920 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4921 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4923 die "skip\n"; # skip non-hot-pluggable options
4927 &$add_error($opt, $err) if $err ne "skip\n";
4929 # save new config if hotplug was successful
4930 $conf->{$opt} = $value;
4931 delete $conf->{pending}->{$opt};
4932 PVE::QemuConfig->write_config($vmid, $conf);
4933 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4938 sub try_deallocate_drive {
4939 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4941 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4942 my $volid = $drive->{file};
4943 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4944 my $sid = PVE::Storage::parse_volume_id($volid);
4945 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4947 # check if the disk is really unused
4948 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4949 if is_volume_in_use($storecfg, $conf, $key, $volid);
4950 PVE::Storage::vdisk_free($storecfg, $volid);
4953 # If vm is not owner of this disk remove from config
4961 sub vmconfig_delete_or_detach_drive {
4962 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4964 my $drive = parse_drive($opt, $conf->{$opt});
4966 my $rpcenv = PVE::RPCEnvironment::get();
4967 my $authuser = $rpcenv->get_user();
4970 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4971 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4973 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4979 sub vmconfig_apply_pending {
4980 my ($vmid, $conf, $storecfg) = @_;
4984 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4985 foreach my $opt (sort keys %$pending_delete_hash) {
4986 die "internal error" if $opt =~ m/^unused/;
4987 my $force = $pending_delete_hash->{$opt}->{force};
4988 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4989 if (!defined($conf->{$opt})) {
4990 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4991 PVE::QemuConfig->write_config($vmid, $conf);
4992 } elsif (is_valid_drivename($opt)) {
4993 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4994 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4995 delete $conf->{$opt};
4996 PVE::QemuConfig->write_config($vmid, $conf);
4998 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4999 delete $conf->{$opt};
5000 PVE::QemuConfig->write_config($vmid, $conf);
5004 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5006 foreach my $opt (keys %{$conf->{pending}}) { # add/change
5007 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5009 if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5010 # skip if nothing changed
5011 } elsif (is_valid_drivename($opt)) {
5012 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5013 if defined($conf->{$opt});
5014 $conf->{$opt} = $conf->{pending}->{$opt};
5016 $conf->{$opt} = $conf->{pending}->{$opt};
5019 delete $conf->{pending}->{$opt};
5020 PVE::QemuConfig->write_config($vmid, $conf);
5024 my $safe_num_ne = sub {
5027 return 0 if !defined($a) && !defined($b);
5028 return 1 if !defined($a);
5029 return 1 if !defined($b);
5034 my $safe_string_ne = sub {
5037 return 0 if !defined($a) && !defined($b);
5038 return 1 if !defined($a);
5039 return 1 if !defined($b);
5044 sub vmconfig_update_net {
5045 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5047 my $newnet = parse_net($value);
5049 if ($conf->{$opt}) {
5050 my $oldnet = parse_net($conf->{$opt});
5052 if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5053 &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5054 &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5055 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5057 # for non online change, we try to hot-unplug
5058 die "skip\n" if !$hotplug;
5059 vm_deviceunplug($vmid, $conf, $opt);
5062 die "internal error" if $opt !~ m/net(\d+)/;
5063 my $iface = "tap${vmid}i$1";
5065 if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5066 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5067 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5068 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5069 PVE::Network::tap_unplug($iface);
5070 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5071 } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5072 # Rate can be applied on its own but any change above needs to
5073 # include the rate in tap_plug since OVS resets everything.
5074 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5077 if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5078 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5086 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5092 sub vmconfig_update_disk {
5093 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5095 # fixme: do we need force?
5097 my $drive = parse_drive($opt, $value);
5099 if ($conf->{$opt}) {
5101 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
5103 my $media = $drive->{media} || 'disk
';
5104 my $oldmedia = $old_drive->{media} || 'disk
';
5105 die "unable to change media type\n" if $media ne $oldmedia;
5107 if (!drive_is_cdrom($old_drive)) {
5109 if ($drive->{file} ne $old_drive->{file}) {
5111 die "skip\n" if !$hotplug;
5113 # unplug and register as unused
5114 vm_deviceunplug($vmid, $conf, $opt);
5115 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5118 # update existing disk
5120 # skip non hotpluggable value
5121 if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5122 &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5123 &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5124 &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5129 if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5130 &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5131 &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5132 &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5133 &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5134 &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5135 &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5136 &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5137 &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5138 &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5139 &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5140 &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5141 &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5142 &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5143 &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5144 &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5145 &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5146 &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5148 qemu_block_set_io_throttle($vmid,"drive-$opt",
5149 ($drive->{mbps} || 0)*1024*1024,
5150 ($drive->{mbps_rd} || 0)*1024*1024,
5151 ($drive->{mbps_wr} || 0)*1024*1024,
5152 $drive->{iops} || 0,
5153 $drive->{iops_rd} || 0,
5154 $drive->{iops_wr} || 0,
5155 ($drive->{mbps_max} || 0)*1024*1024,
5156 ($drive->{mbps_rd_max} || 0)*1024*1024,
5157 ($drive->{mbps_wr_max} || 0)*1024*1024,
5158 $drive->{iops_max} || 0,
5159 $drive->{iops_rd_max} || 0,
5160 $drive->{iops_wr_max} || 0,
5161 $drive->{bps_max_length} || 1,
5162 $drive->{bps_rd_max_length} || 1,
5163 $drive->{bps_wr_max_length} || 1,
5164 $drive->{iops_max_length} || 1,
5165 $drive->{iops_rd_max_length} || 1,
5166 $drive->{iops_wr_max_length} || 1);
5175 if ($drive->{file} eq 'none
') {
5176 mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5177 if (drive_is_cloudinit($old_drive)) {
5178 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5181 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5182 mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5183 mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5191 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5193 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5194 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5198 my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5199 $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5201 PVE::QemuConfig->lock_config($vmid, sub {
5202 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5204 die "you can't start a vm
if it
's a template\n" if PVE::QemuConfig->is_template($conf);
5206 my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended
');
5208 PVE::QemuConfig->check_lock($conf)
5209 if !($skiplock || $is_suspended);
5211 die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5213 # clean up leftover reboot request files
5214 eval { clear_reboot_request($vmid); };
5217 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5218 vmconfig_apply_pending($vmid, $conf, $storecfg);
5219 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5222 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5224 my $defaults = load_defaults();
5226 # set environment variable useful inside network script
5227 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5229 my $local_volumes = {};
5231 if ($targetstorage) {
5232 foreach_drive($conf, sub {
5233 my ($ds, $drive) = @_;
5235 return if drive_is_cdrom($drive);
5237 my $volid = $drive->{file};
5241 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5243 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5244 return if $scfg->{shared};
5245 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5250 foreach my $opt (sort keys %$local_volumes) {
5252 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5253 my $drive = parse_drive($opt, $conf->{$opt});
5255 #if remote storage is specified, use default format
5256 if ($targetstorage && $targetstorage ne "1") {
5257 $storeid = $targetstorage;
5258 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5259 $format = $defFormat;
5261 #else we use same format than original
5262 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5263 $format = qemu_img_format($scfg, $volid);
5266 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5267 my $newdrive = $drive;
5268 $newdrive->{format} = $format;
5269 $newdrive->{file} = $newvolid;
5270 my $drivestr = print_drive($newdrive);
5271 $local_volumes->{$opt} = $drivestr;
5272 #pass drive to conf for command line
5273 $conf->{$opt} = $drivestr;
5277 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
5279 if ($is_suspended) {
5280 # enforce machine type on suspended vm to ensure HW compatibility
5281 $forcemachine = $conf->{runningmachine};
5282 print "Resuming suspended VM\n";
5285 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5288 my $get_migration_ip = sub {
5289 my ($cidr, $nodename) = @_;
5291 return $migration_ip if defined($migration_ip);
5293 if (!defined($cidr)) {
5294 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5295 $cidr = $dc_conf->{migration}->{network};
5298 if (defined($cidr)) {
5299 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5301 die "could not get IP: no address configured on local " .
5302 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5304 die "could not get IP: multiple addresses configured on local " .
5305 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5307 $migration_ip = @$ips[0];
5310 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5311 if !defined($migration_ip);
5313 return $migration_ip;
5318 if ($statefile eq 'tcp
') {
5319 my $localip = "localhost";
5320 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5321 my $nodename = PVE::INotify::nodename();
5323 if (!defined($migration_type)) {
5324 if (defined($datacenterconf->{migration}->{type})) {
5325 $migration_type = $datacenterconf->{migration}->{type};
5327 $migration_type = 'secure
';
5331 if ($migration_type eq 'insecure
') {
5332 $localip = $get_migration_ip->($migration_network, $nodename);
5333 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5336 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5337 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5338 $migrate_uri = "tcp:${localip}:${migrate_port}";
5339 push @$cmd, '-incoming
', $migrate_uri;
5342 } elsif ($statefile eq 'unix
') {
5343 # should be default for secure migrations as a ssh TCP forward
5344 # tunnel is not deterministic reliable ready and fails regurarly
5345 # to set up in time, so use UNIX socket forwards
5346 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5347 unlink $socket_addr;
5349 $migrate_uri = "unix:$socket_addr";
5351 push @$cmd, '-incoming
', $migrate_uri;
5354 } elsif (-e $statefile) {
5355 push @$cmd, '-loadstate
', $statefile;
5357 my $statepath = PVE::Storage::path($storecfg, $statefile);
5358 push @$vollist, $statefile;
5359 push @$cmd, '-loadstate
', $statepath;
5366 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
5367 my $d = parse_hostpci($conf->{"hostpci$i"});
5369 my $pcidevices = $d->{pciid};
5370 foreach my $pcidevice (@$pcidevices) {
5371 my $pciid = $pcidevice->{id};
5373 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5374 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5375 die "no pci device info for device '$pciid'\n" if !$info;
5378 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5379 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5381 die "can't unbind
/bind pci group to vfio
'$pciid'\n"
5382 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5383 die "can
't reset pci device '$pciid'\n"
5384 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5389 PVE::Storage::activate_volumes($storecfg, $vollist);
5392 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5393 outfunc => sub {}, errfunc => sub {});
5395 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5396 # timeout should be more than enough here...
5397 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5399 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5400 : $defaults->{cpuunits};
5402 my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5403 my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5406 Slice => 'qemu
.slice
',
5408 CPUShares => $cpuunits
5411 if (my $cpulimit = $conf->{cpulimit}) {
5412 $properties{CPUQuota} = int($cpulimit * 100);
5414 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5416 my $run_qemu = sub {
5417 PVE::Tools::run_fork sub {
5418 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5419 run_command($cmd, %run_params);
5423 if ($conf->{hugepages}) {
5426 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5427 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5429 PVE::QemuServer::Memory::hugepages_mount();
5430 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5432 eval { $run_qemu->() };
5434 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5438 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5440 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5443 eval { $run_qemu->() };
5447 # deactivate volumes if start fails
5448 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5449 die "start failed: $err";
5452 print "migration listens on $migrate_uri\n" if $migrate_uri;
5454 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5455 eval { mon_cmd($vmid, "cont"); };
5459 #start nbd server for storage migration
5460 if ($targetstorage) {
5461 my $nodename = PVE::INotify::nodename();
5462 my $localip = $get_migration_ip->($migration_network, $nodename);
5463 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5464 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5466 mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet
', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5468 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5470 foreach my $opt (sort keys %$local_volumes) {
5471 my $volid = $local_volumes->{$opt};
5472 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5473 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5474 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5478 if ($migratedfrom) {
5480 set_migration_caps($vmid);
5485 print "spice listens on port $spice_port\n";
5486 if ($spice_ticket) {
5487 mon_cmd($vmid, "set_password", protocol => 'spice
', password => $spice_ticket);
5488 mon_cmd($vmid, "expire_password", protocol => 'spice
', time => "+30");
5493 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5494 if !$statefile && $conf->{balloon};
5496 foreach my $opt (keys %$conf) {
5497 next if $opt !~ m/^net\d+$/;
5498 my $nicconf = parse_net($conf->{$opt});
5499 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5503 mon_cmd($vmid, 'qom-set
',
5504 path => "machine/peripheral/balloon0",
5505 property => "guest-stats-polling-interval",
5506 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5508 if ($is_suspended) {
5509 print "Resumed VM, removing state\n";
5510 if (my $vmstate = $conf->{vmstate}) {
5511 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5512 PVE::Storage::vdisk_free($storecfg, $vmstate);
5514 delete $conf->@{qw(lock vmstate runningmachine)};
5515 PVE
::QemuConfig-
>write_config($vmid, $conf);
5518 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5522 sub vm_commandline
{
5523 my ($storecfg, $vmid, $snapname) = @_;
5525 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5529 my $snapshot = $conf->{snapshots
}->{$snapname};
5530 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5532 # check for a 'runningmachine' in snapshot
5533 $forcemachine = $snapshot->{runningmachine
} if $snapshot->{runningmachine
};
5535 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5540 my $defaults = load_defaults
();
5542 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine);
5544 return PVE
::Tools
::cmd2string
($cmd);
5548 my ($vmid, $skiplock) = @_;
5550 PVE
::QemuConfig-
>lock_config($vmid, sub {
5552 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5554 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5556 mon_cmd
($vmid, "system_reset");
5560 sub get_vm_volumes
{
5564 foreach_volid
($conf, sub {
5565 my ($volid, $attr) = @_;
5567 return if $volid =~ m
|^/|;
5569 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5572 push @$vollist, $volid;
5578 sub vm_stop_cleanup
{
5579 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5584 my $vollist = get_vm_volumes
($conf);
5585 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5588 foreach my $ext (qw(mon qmp pid vnc qga)) {
5589 unlink "/var/run/qemu-server/${vmid}.$ext";
5592 if ($conf->{ivshmem
}) {
5593 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5594 # just delete it for now, VMs which have this already open do not
5595 # are affected, but new VMs will get a separated one. If this
5596 # becomes an issue we either add some sort of ref-counting or just
5597 # add a "don't delete on stop" flag to the ivshmem format.
5598 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5601 foreach my $key (keys %$conf) {
5602 next if $key !~ m/^hostpci(\d+)$/;
5603 my $hostpciindex = $1;
5604 my $d = parse_hostpci
($conf->{$key});
5605 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5607 foreach my $pci (@{$d->{pciid
}}) {
5608 my $pciid = $pci->{id
};
5609 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5613 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5615 warn $@ if $@; # avoid errors - just warn
5618 # call only in locked context
5620 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5622 my $pid = check_running
($vmid, $nocheck);
5627 $conf = PVE
::QemuConfig-
>load_config($vmid);
5628 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5629 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5630 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5631 $timeout = $opts->{down
} if $opts->{down
};
5633 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5638 if (defined($conf) && parse_guest_agent
($conf)->{enabled
}) {
5639 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5641 mon_cmd
($vmid, "system_powerdown");
5644 mon_cmd
($vmid, "quit");
5650 $timeout = 60 if !defined($timeout);
5653 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5658 if ($count >= $timeout) {
5660 warn "VM still running - terminating now with SIGTERM\n";
5663 die "VM quit/powerdown failed - got timeout\n";
5666 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5671 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5674 die "VM quit/powerdown failed\n";
5682 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5687 if ($count >= $timeout) {
5688 warn "VM still running - terminating now with SIGKILL\n";
5693 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5696 # Note: use $nocheck to skip tests if VM configuration file exists.
5697 # We need that when migration VMs to other nodes (files already moved)
5698 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5700 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5702 $force = 1 if !defined($force) && !$shutdown;
5705 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5706 kill 15, $pid if $pid;
5707 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5708 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5712 PVE
::QemuConfig-
>lock_config($vmid, sub {
5713 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5718 my ($vmid, $timeout) = @_;
5720 PVE
::QemuConfig-
>lock_config($vmid, sub {
5723 # only reboot if running, as qmeventd starts it again on a stop event
5724 return if !check_running
($vmid);
5726 create_reboot_request
($vmid);
5728 my $storecfg = PVE
::Storage
::config
();
5729 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
5733 # avoid that the next normal shutdown will be confused for a reboot
5734 clear_reboot_request
($vmid);
5741 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5748 PVE
::QemuConfig-
>lock_config($vmid, sub {
5750 $conf = PVE
::QemuConfig-
>load_config($vmid);
5752 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5753 PVE
::QemuConfig-
>check_lock($conf)
5754 if !($skiplock || $is_backing_up);
5756 die "cannot suspend to disk during backup\n"
5757 if $is_backing_up && $includestate;
5759 if ($includestate) {
5760 $conf->{lock} = 'suspending';
5761 my $date = strftime
("%Y-%m-%d", localtime(time()));
5762 $storecfg = PVE
::Storage
::config
();
5763 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5764 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5765 PVE
::QemuConfig-
>write_config($vmid, $conf);
5767 mon_cmd
($vmid, "stop");
5771 if ($includestate) {
5773 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5776 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5778 my $state = mon_cmd
($vmid, "query-savevm");
5779 if (!$state->{status
}) {
5780 die "savevm not active\n";
5781 } elsif ($state->{status
} eq 'active') {
5784 } elsif ($state->{status
} eq 'completed') {
5785 print "State saved, quitting\n";
5787 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5788 die "query-savevm failed with error '$state->{error}'\n"
5790 die "query-savevm returned status '$state->{status}'\n";
5796 PVE
::QemuConfig-
>lock_config($vmid, sub {
5797 $conf = PVE
::QemuConfig-
>load_config($vmid);
5799 # cleanup, but leave suspending lock, to indicate something went wrong
5801 mon_cmd
($vmid, "savevm-end");
5802 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5803 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5804 delete $conf->@{qw(vmstate runningmachine)};
5805 PVE
::QemuConfig-
>write_config($vmid, $conf);
5811 die "lock changed unexpectedly\n"
5812 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5814 mon_cmd
($vmid, "quit");
5815 $conf->{lock} = 'suspended';
5816 PVE
::QemuConfig-
>write_config($vmid, $conf);
5822 my ($vmid, $skiplock, $nocheck) = @_;
5824 PVE
::QemuConfig-
>lock_config($vmid, sub {
5825 my $res = mon_cmd
($vmid, 'query-status');
5826 my $resume_cmd = 'cont';
5828 if ($res->{status
} && $res->{status
} eq 'suspended') {
5829 $resume_cmd = 'system_wakeup';
5834 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5836 PVE
::QemuConfig-
>check_lock($conf)
5837 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5840 mon_cmd
($vmid, $resume_cmd);
5845 my ($vmid, $skiplock, $key) = @_;
5847 PVE
::QemuConfig-
>lock_config($vmid, sub {
5849 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5851 # there is no qmp command, so we use the human monitor command
5852 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
5853 die $res if $res ne '';
5857 # vzdump restore implementaion
5859 sub tar_archive_read_firstfile
{
5860 my $archive = shift;
5862 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5864 # try to detect archive type first
5865 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5866 die "unable to open file '$archive'\n";
5867 my $firstfile = <$fh>;
5871 die "ERROR: archive contaions no data\n" if !$firstfile;
5877 sub tar_restore_cleanup
{
5878 my ($storecfg, $statfile) = @_;
5880 print STDERR
"starting cleanup\n";
5882 if (my $fd = IO
::File-
>new($statfile, "r")) {
5883 while (defined(my $line = <$fd>)) {
5884 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5887 if ($volid =~ m
|^/|) {
5888 unlink $volid || die 'unlink failed\n';
5890 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5892 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5894 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5896 print STDERR
"unable to parse line in statfile - $line";
5903 sub restore_archive
{
5904 my ($archive, $vmid, $user, $opts) = @_;
5906 my $format = $opts->{format
};
5909 if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5910 $format = 'tar' if !$format;
5912 } elsif ($archive =~ m/\.tar$/) {
5913 $format = 'tar' if !$format;
5914 } elsif ($archive =~ m/.tar.lzo$/) {
5915 $format = 'tar' if !$format;
5917 } elsif ($archive =~ m/\.vma$/) {
5918 $format = 'vma' if !$format;
5919 } elsif ($archive =~ m/\.vma\.gz$/) {
5920 $format = 'vma' if !$format;
5922 } elsif ($archive =~ m/\.vma\.lzo$/) {
5923 $format = 'vma' if !$format;
5926 $format = 'vma' if !$format; # default
5929 # try to detect archive format
5930 if ($format eq 'tar') {
5931 return restore_tar_archive
($archive, $vmid, $user, $opts);
5933 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5937 sub restore_update_config_line
{
5938 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5940 return if $line =~ m/^\#qmdump\#/;
5941 return if $line =~ m/^\#vzdump\#/;
5942 return if $line =~ m/^lock:/;
5943 return if $line =~ m/^unused\d+:/;
5944 return if $line =~ m/^parent:/;
5946 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5947 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5948 # try to convert old 1.X settings
5949 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5950 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5951 my ($model, $macaddr) = split(/\=/, $devconfig);
5952 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5955 bridge
=> "vmbr$ind",
5956 macaddr
=> $macaddr,
5958 my $netstr = print_net
($net);
5960 print $outfd "net$cookie->{netcount}: $netstr\n";
5961 $cookie->{netcount
}++;
5963 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5964 my ($id, $netstr) = ($1, $2);
5965 my $net = parse_net
($netstr);
5966 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5967 $netstr = print_net
($net);
5968 print $outfd "$id: $netstr\n";
5969 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5972 my $di = parse_drive
($virtdev, $value);
5973 if (defined($di->{backup
}) && !$di->{backup
}) {
5974 print $outfd "#$line";
5975 } elsif ($map->{$virtdev}) {
5976 delete $di->{format
}; # format can change on restore
5977 $di->{file
} = $map->{$virtdev};
5978 $value = print_drive
($di);
5979 print $outfd "$virtdev: $value\n";
5983 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5985 if ($vmgenid ne '0') {
5986 # always generate a new vmgenid if there was a valid one setup
5987 $vmgenid = generate_uuid
();
5989 print $outfd "vmgenid: $vmgenid\n";
5990 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5991 my ($uuid, $uuid_str);
5992 UUID
::generate
($uuid);
5993 UUID
::unparse
($uuid, $uuid_str);
5994 my $smbios1 = parse_smbios1
($2);
5995 $smbios1->{uuid
} = $uuid_str;
5996 print $outfd $1.print_smbios1
($smbios1)."\n";
6003 my ($cfg, $vmid) = @_;
6005 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
6007 my $volid_hash = {};
6008 foreach my $storeid (keys %$info) {
6009 foreach my $item (@{$info->{$storeid}}) {
6010 next if !($item->{volid
} && $item->{size
});
6011 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6012 $volid_hash->{$item->{volid
}} = $item;
6019 sub is_volume_in_use
{
6020 my ($storecfg, $conf, $skip_drive, $volid) = @_;
6022 my $path = PVE
::Storage
::path
($storecfg, $volid);
6024 my $scan_config = sub {
6025 my ($cref, $snapname) = @_;
6027 foreach my $key (keys %$cref) {
6028 my $value = $cref->{$key};
6029 if (is_valid_drivename
($key)) {
6030 next if $skip_drive && $key eq $skip_drive;
6031 my $drive = parse_drive
($key, $value);
6032 next if !$drive || !$drive->{file
} || drive_is_cdrom
($drive);
6033 return 1 if $volid eq $drive->{file
};
6034 if ($drive->{file
} =~ m!^/!) {
6035 return 1 if $drive->{file
} eq $path;
6037 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
}, 1);
6039 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid, 1);
6041 return 1 if $path eq PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
6049 return 1 if &$scan_config($conf);
6053 foreach my $snapname (keys %{$conf->{snapshots
}}) {
6054 return 1 if &$scan_config($conf->{snapshots
}->{$snapname}, $snapname);
6060 sub update_disksize
{
6061 my ($vmid, $conf, $volid_hash) = @_;
6064 my $prefix = "VM $vmid:";
6066 # used and unused disks
6067 my $referenced = {};
6069 # Note: it is allowed to define multiple storages with same path (alias), so
6070 # we need to check both 'volid' and real 'path' (two different volid can point
6071 # to the same path).
6073 my $referencedpath = {};
6076 foreach my $opt (keys %$conf) {
6077 if (is_valid_drivename
($opt)) {
6078 my $drive = parse_drive
($opt, $conf->{$opt});
6079 my $volid = $drive->{file
};
6082 $referenced->{$volid} = 1;
6083 if ($volid_hash->{$volid} &&
6084 (my $path = $volid_hash->{$volid}->{path
})) {
6085 $referencedpath->{$path} = 1;
6088 next if drive_is_cdrom
($drive);
6089 next if !$volid_hash->{$volid};
6091 $drive->{size
} = $volid_hash->{$volid}->{size
};
6092 my $new = print_drive
($drive);
6093 if ($new ne $conf->{$opt}) {
6095 $conf->{$opt} = $new;
6096 print "$prefix update disk '$opt' information.\n";
6101 # remove 'unusedX' entry if volume is used
6102 foreach my $opt (keys %$conf) {
6103 next if $opt !~ m/^unused\d+$/;
6104 my $volid = $conf->{$opt};
6105 my $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6106 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6107 print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6109 delete $conf->{$opt};
6112 $referenced->{$volid} = 1;
6113 $referencedpath->{$path} = 1 if $path;
6116 foreach my $volid (sort keys %$volid_hash) {
6117 next if $volid =~ m/vm-$vmid-state-/;
6118 next if $referenced->{$volid};
6119 my $path = $volid_hash->{$volid}->{path
};
6120 next if !$path; # just to be sure
6121 next if $referencedpath->{$path};
6123 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6124 print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6125 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6132 my ($vmid, $nolock, $dryrun) = @_;
6134 my $cfg = PVE
::Storage
::config
();
6136 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6137 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6138 foreach my $stor (keys %{$cfg->{ids
}}) {
6139 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
6142 print "rescan volumes...\n";
6143 my $volid_hash = scan_volids
($cfg, $vmid);
6145 my $updatefn = sub {
6148 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6150 PVE
::QemuConfig-
>check_lock($conf);
6153 foreach my $volid (keys %$volid_hash) {
6154 my $info = $volid_hash->{$volid};
6155 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6158 my $changes = update_disksize
($vmid, $conf, $vm_volids);
6160 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6163 if (defined($vmid)) {
6167 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6170 my $vmlist = config_list
();
6171 foreach my $vmid (keys %$vmlist) {
6175 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6181 sub restore_vma_archive
{
6182 my ($archive, $vmid, $user, $opts, $comp) = @_;
6184 my $readfrom = $archive;
6186 my $cfg = PVE
::Storage
::config
();
6188 my $bwlimit = $opts->{bwlimit
};
6190 my $dbg_cmdstring = '';
6191 my $add_pipe = sub {
6193 push @$commands, $cmd;
6194 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6195 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6200 if ($archive eq '-') {
6203 # If we use a backup from a PVE defined storage we also consider that
6204 # storage's rate limit:
6205 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6206 if (defined($volid)) {
6207 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6208 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6210 print STDERR
"applying read rate limit: $readlimit\n";
6211 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6212 $add_pipe->($cstream);
6219 if ($comp eq 'gzip') {
6220 $cmd = ['zcat', $readfrom];
6221 } elsif ($comp eq 'lzop') {
6222 $cmd = ['lzop', '-d', '-c', $readfrom];
6224 die "unknown compression method '$comp'\n";
6229 my $tmpdir = "/var/tmp/vzdumptmp$$";
6232 # disable interrupts (always do cleanups)
6236 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6238 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6239 POSIX
::mkfifo
($mapfifo, 0600);
6242 my $openfifo = sub {
6243 open($fifofh, '>', $mapfifo) || die $!;
6246 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6253 my $rpcenv = PVE
::RPCEnvironment
::get
();
6255 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6256 my $tmpfn = "$conffile.$$.tmp";
6258 # Note: $oldconf is undef if VM does not exists
6259 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6260 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6264 my $print_devmap = sub {
6265 my $virtdev_hash = {};
6267 my $cfgfn = "$tmpdir/qemu-server.conf";
6269 # we can read the config - that is already extracted
6270 my $fh = IO
::File-
>new($cfgfn, "r") ||
6271 "unable to read qemu-server.conf - $!\n";
6273 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6275 my $pve_firewall_dir = '/etc/pve/firewall';
6276 mkdir $pve_firewall_dir; # make sure the dir exists
6277 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6280 while (defined(my $line = <$fh>)) {
6281 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6282 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6283 die "archive does not contain data for drive '$virtdev'\n"
6284 if !$devinfo->{$devname};
6285 if (defined($opts->{storage
})) {
6286 $storeid = $opts->{storage
} || 'local';
6287 } elsif (!$storeid) {
6290 $format = 'raw' if !$format;
6291 $devinfo->{$devname}->{devname
} = $devname;
6292 $devinfo->{$devname}->{virtdev
} = $virtdev;
6293 $devinfo->{$devname}->{format
} = $format;
6294 $devinfo->{$devname}->{storeid
} = $storeid;
6296 # check permission on storage
6297 my $pool = $opts->{pool
}; # todo: do we need that?
6298 if ($user ne 'root@pam') {
6299 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6302 $storage_limits{$storeid} = $bwlimit;
6304 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6305 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6307 my $drive = parse_drive
($virtdev, $2);
6308 if (drive_is_cloudinit
($drive)) {
6309 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6310 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
6311 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6315 storeid
=> $opts->{storage
} // $storeid,
6316 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6317 file
=> $drive->{file
}, # to make drive_is_cloudinit check possible
6318 name
=> "vm-$vmid-cloudinit",
6321 $virtdev_hash->{$virtdev} = $d;
6326 foreach my $key (keys %storage_limits) {
6327 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6329 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6330 $storage_limits{$key} = $limit * 1024;
6333 foreach my $devname (keys %$devinfo) {
6334 die "found no device mapping information for device '$devname'\n"
6335 if !$devinfo->{$devname}->{virtdev
};
6338 # create empty/temp config
6340 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6341 foreach_drive
($oldconf, sub {
6342 my ($ds, $drive) = @_;
6344 return if drive_is_cdrom
($drive, 1);
6346 my $volid = $drive->{file
};
6347 return if !$volid || $volid =~ m
|^/|;
6349 my ($path, $owner) = PVE
::Storage
::path
($cfg, $volid);
6350 return if !$path || !$owner || ($owner != $vmid);
6352 # Note: only delete disk we want to restore
6353 # other volumes will become unused
6354 if ($virtdev_hash->{$ds}) {
6355 eval { PVE
::Storage
::vdisk_free
($cfg, $volid); };
6362 # delete vmstate files, after the restore we have no snapshots anymore
6363 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
6364 my $snap = $oldconf->{snapshots
}->{$snapname};
6365 if ($snap->{vmstate
}) {
6366 eval { PVE
::Storage
::vdisk_free
($cfg, $snap->{vmstate
}); };
6375 foreach my $virtdev (sort keys %$virtdev_hash) {
6376 my $d = $virtdev_hash->{$virtdev};
6377 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6378 my $storeid = $d->{storeid
};
6379 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
6382 if (my $limit = $storage_limits{$storeid}) {
6383 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6386 # test if requested format is supported
6387 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($cfg, $storeid);
6388 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6389 $d->{format
} = $defFormat if !$supported;
6392 if ($d->{is_cloudinit
}) {
6394 $name .= ".$d->{format}" if $d->{format
} ne 'raw';
6397 my $volid = PVE
::Storage
::vdisk_alloc
($cfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6398 print STDERR
"new volume ID is '$volid'\n";
6399 $d->{volid
} = $volid;
6401 PVE
::Storage
::activate_volumes
($cfg, [$volid]);
6403 my $write_zeros = 1;
6404 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6408 if (!$d->{is_cloudinit
}) {
6409 my $path = PVE
::Storage
::path
($cfg, $volid);
6411 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6413 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6415 $map->{$virtdev} = $volid;
6418 $fh->seek(0, 0) || die "seek failed - $!\n";
6420 my $outfd = new IO
::File
($tmpfn, "w") ||
6421 die "unable to write config for VM $vmid\n";
6423 my $cookie = { netcount
=> 0 };
6424 while (defined(my $line = <$fh>)) {
6425 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6438 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6439 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6441 $oldtimeout = alarm($timeout);
6448 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6449 my ($dev_id, $size, $devname) = ($1, $2, $3);
6450 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6451 } elsif ($line =~ m/^CTIME: /) {
6452 # we correctly received the vma config, so we can disable
6453 # the timeout now for disk allocation (set to 10 minutes, so
6454 # that we always timeout if something goes wrong)
6457 print $fifofh "done\n";
6458 my $tmp = $oldtimeout || 0;
6459 $oldtimeout = undef;
6465 print "restore vma archive: $dbg_cmdstring\n";
6466 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6470 alarm($oldtimeout) if $oldtimeout;
6473 foreach my $devname (keys %$devinfo) {
6474 my $volid = $devinfo->{$devname}->{volid
};
6475 push @$vollist, $volid if $volid;
6478 PVE
::Storage
::deactivate_volumes
($cfg, $vollist);
6486 foreach my $devname (keys %$devinfo) {
6487 my $volid = $devinfo->{$devname}->{volid
};
6490 if ($volid =~ m
|^/|) {
6491 unlink $volid || die 'unlink failed\n';
6493 PVE
::Storage
::vdisk_free
($cfg, $volid);
6495 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6497 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6504 rename($tmpfn, $conffile) ||
6505 die "unable to commit configuration file '$conffile'\n";
6507 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6509 eval { rescan
($vmid, 1); };
6513 sub restore_tar_archive
{
6514 my ($archive, $vmid, $user, $opts) = @_;
6516 if ($archive ne '-') {
6517 my $firstfile = tar_archive_read_firstfile
($archive);
6518 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6519 if $firstfile ne 'qemu-server.conf';
6522 my $storecfg = PVE
::Storage
::config
();
6524 # avoid zombie disks when restoring over an existing VM -> cleanup first
6525 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6526 # skiplock=1 because qmrestore has set the 'create' lock itself already
6527 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6528 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
6530 my $tocmd = "/usr/lib/qemu-server/qmextract";
6532 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6533 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6534 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6535 $tocmd .= ' --info' if $opts->{info
};
6537 # tar option "xf" does not autodetect compression when read from STDIN,
6538 # so we pipe to zcat
6539 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6540 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6542 my $tmpdir = "/var/tmp/vzdumptmp$$";
6545 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6546 local $ENV{VZDUMP_VMID
} = $vmid;
6547 local $ENV{VZDUMP_USER
} = $user;
6549 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6550 my $tmpfn = "$conffile.$$.tmp";
6552 # disable interrupts (always do cleanups)
6556 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6564 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6566 if ($archive eq '-') {
6567 print "extracting archive from STDIN\n";
6568 run_command
($cmd, input
=> "<&STDIN");
6570 print "extracting archive '$archive'\n";
6574 return if $opts->{info
};
6578 my $statfile = "$tmpdir/qmrestore.stat";
6579 if (my $fd = IO
::File-
>new($statfile, "r")) {
6580 while (defined (my $line = <$fd>)) {
6581 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6582 $map->{$1} = $2 if $1;
6584 print STDERR
"unable to parse line in statfile - $line\n";
6590 my $confsrc = "$tmpdir/qemu-server.conf";
6592 my $srcfd = new IO
::File
($confsrc, "r") ||
6593 die "unable to open file '$confsrc'\n";
6595 my $outfd = new IO
::File
($tmpfn, "w") ||
6596 die "unable to write config for VM $vmid\n";
6598 my $cookie = { netcount
=> 0 };
6599 while (defined (my $line = <$srcfd>)) {
6600 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6608 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6614 rename $tmpfn, $conffile ||
6615 die "unable to commit configuration file '$conffile'\n";
6617 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6619 eval { rescan
($vmid, 1); };
6623 sub foreach_storage_used_by_vm
{
6624 my ($conf, $func) = @_;
6628 foreach_drive
($conf, sub {
6629 my ($ds, $drive) = @_;
6630 return if drive_is_cdrom
($drive);
6632 my $volid = $drive->{file
};
6634 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6635 $sidhash->{$sid} = $sid if $sid;
6638 foreach my $sid (sort keys %$sidhash) {
6643 my $qemu_snap_storage = {
6646 sub do_snapshots_with_qemu
{
6647 my ($storecfg, $volid) = @_;
6649 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6650 my $scfg = $storecfg->{ids
}->{$storage_name};
6652 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6656 if ($volid =~ m/\.(qcow2|qed)$/){
6663 sub qga_check_running
{
6664 my ($vmid, $nowarn) = @_;
6666 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6668 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6674 sub template_create
{
6675 my ($vmid, $conf, $disk) = @_;
6677 my $storecfg = PVE
::Storage
::config
();
6679 foreach_drive
($conf, sub {
6680 my ($ds, $drive) = @_;
6682 return if drive_is_cdrom
($drive);
6683 return if $disk && $ds ne $disk;
6685 my $volid = $drive->{file
};
6686 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6688 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6689 $drive->{file
} = $voliddst;
6690 $conf->{$ds} = print_drive
($drive);
6691 PVE
::QemuConfig-
>write_config($vmid, $conf);
6695 sub convert_iscsi_path
{
6698 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6703 my $initiator_name = get_initiator_name
();
6705 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6706 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6709 die "cannot convert iscsi path '$path', unkown format\n";
6712 sub qemu_img_convert
{
6713 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6715 my $storecfg = PVE
::Storage
::config
();
6716 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6717 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6719 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6723 my $src_is_iscsi = 0;
6727 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6728 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6729 $src_format = qemu_img_format
($src_scfg, $src_volname);
6730 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6731 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6732 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
6733 } elsif (-f
$src_volid) {
6734 $src_path = $src_volid;
6735 if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6740 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6742 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6743 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6744 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6745 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6748 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6749 push @$cmd, '-l', "snapshot.name=$snapname"
6750 if $snapname && $src_format && $src_format eq "qcow2";
6751 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6752 push @$cmd, '-T', $cachemode if defined($cachemode);
6754 if ($src_is_iscsi) {
6755 push @$cmd, '--image-opts';
6756 $src_path = convert_iscsi_path
($src_path);
6757 } elsif ($src_format) {
6758 push @$cmd, '-f', $src_format;
6761 if ($dst_is_iscsi) {
6762 push @$cmd, '--target-image-opts';
6763 $dst_path = convert_iscsi_path
($dst_path);
6765 push @$cmd, '-O', $dst_format;
6768 push @$cmd, $src_path;
6770 if (!$dst_is_iscsi && $is_zero_initialized) {
6771 push @$cmd, "zeroinit:$dst_path";
6773 push @$cmd, $dst_path;
6778 if($line =~ m/\((\S+)\/100\
%\)/){
6780 my $transferred = int($size * $percent / 100);
6781 my $remaining = $size - $transferred;
6783 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6788 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6790 die "copy failed: $err" if $err;
6793 sub qemu_img_format
{
6794 my ($scfg, $volname) = @_;
6796 if ($scfg->{path
} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6803 sub qemu_drive_mirror
{
6804 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6806 $jobs = {} if !$jobs;
6810 $jobs->{"drive-$drive"} = {};
6812 if ($dst_volid =~ /^nbd:/) {
6813 $qemu_target = $dst_volid;
6816 my $storecfg = PVE
::Storage
::config
();
6817 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6819 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6821 $format = qemu_img_format
($dst_scfg, $dst_volname);
6823 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6825 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6828 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6829 $opts->{format
} = $format if $format;
6831 if (defined($bwlimit)) {
6832 $opts->{speed
} = $bwlimit * 1024;
6833 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6835 print "drive mirror is starting for drive-$drive\n";
6838 # if a job already runs for this device we get an error, catch it for cleanup
6839 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
6841 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6843 die "mirroring error: $err\n";
6846 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6849 sub qemu_drive_mirror_monitor
{
6850 my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6853 my $err_complete = 0;
6856 die "storage migration timed out\n" if $err_complete > 300;
6858 my $stats = mon_cmd
($vmid, "query-block-jobs");
6860 my $running_mirror_jobs = {};
6861 foreach my $stat (@$stats) {
6862 next if $stat->{type
} ne 'mirror';
6863 $running_mirror_jobs->{$stat->{device
}} = $stat;
6866 my $readycounter = 0;
6868 foreach my $job (keys %$jobs) {
6870 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6871 print "$job : finished\n";
6872 delete $jobs->{$job};
6876 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6878 my $busy = $running_mirror_jobs->{$job}->{busy
};
6879 my $ready = $running_mirror_jobs->{$job}->{ready
};
6880 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6881 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6882 my $remaining = $total - $transferred;
6883 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6885 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6888 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6891 last if scalar(keys %$jobs) == 0;
6893 if ($readycounter == scalar(keys %$jobs)) {
6894 print "all mirroring jobs are ready \n";
6895 last if $skipcomplete; #do the complete later
6897 if ($vmiddst && $vmiddst != $vmid) {
6898 my $agent_running = $qga && qga_check_running
($vmid);
6899 if ($agent_running) {
6900 print "freeze filesystem\n";
6901 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6903 print "suspend vm\n";
6904 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6907 # if we clone a disk for a new target vm, we don't switch the disk
6908 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6910 if ($agent_running) {
6911 print "unfreeze filesystem\n";
6912 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6914 print "resume vm\n";
6915 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6921 foreach my $job (keys %$jobs) {
6922 # try to switch the disk if source and destination are on the same guest
6923 print "$job: Completing block job...\n";
6925 eval { mon_cmd
($vmid, "block-job-complete", device
=> $job) };
6926 if ($@ =~ m/cannot be completed/) {
6927 print "$job: Block job cannot be completed, try again.\n";
6930 print "$job: Completed successfully.\n";
6931 $jobs->{$job}->{complete
} = 1;
6942 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6943 die "mirroring error: $err";
6948 sub qemu_blockjobs_cancel
{
6949 my ($vmid, $jobs) = @_;
6951 foreach my $job (keys %$jobs) {
6952 print "$job: Cancelling block job\n";
6953 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6954 $jobs->{$job}->{cancel
} = 1;
6958 my $stats = mon_cmd
($vmid, "query-block-jobs");
6960 my $running_jobs = {};
6961 foreach my $stat (@$stats) {
6962 $running_jobs->{$stat->{device
}} = $stat;
6965 foreach my $job (keys %$jobs) {
6967 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6968 print "$job: Done.\n";
6969 delete $jobs->{$job};
6973 last if scalar(keys %$jobs) == 0;
6980 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6981 $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6986 print "create linked clone of drive $drivename ($drive->{file})\n";
6987 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6988 push @$newvollist, $newvolid;
6991 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6992 $storeid = $storage if $storage;
6994 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6995 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
6997 print "create full clone of drive $drivename ($drive->{file})\n";
6999 if (drive_is_cloudinit
($drive)) {
7000 $name = "vm-$newvmid-cloudinit";
7001 $name .= ".$dst_format" if $dst_format ne 'raw';
7003 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
7005 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7006 push @$newvollist, $newvolid;
7008 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
7010 if (drive_is_cloudinit
($drive)) {
7014 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
7015 if (!$running || $snapname) {
7016 # TODO: handle bwlimits
7017 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
7020 my $kvmver = get_running_qemu_version
($vmid);
7021 if (!min_version
($kvmver, 2, 7)) {
7022 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7023 if $drive->{iothread
};
7026 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7031 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
7034 $disk->{format
} = undef;
7035 $disk->{file
} = $newvolid;
7036 $disk->{size
} = $size;
7041 sub get_running_qemu_version
{
7043 my $res = mon_cmd
($vmid, "query-version");
7044 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7047 sub qemu_use_old_bios_files
{
7048 my ($machine_type) = @_;
7050 return if !$machine_type;
7052 my $use_old_bios_files = undef;
7054 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7056 $use_old_bios_files = 1;
7058 my $version = PVE
::QemuServer
::Machine
::extract_version
($machine_type, kvm_user_version
());
7059 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7060 # load new efi bios files on migration. So this hack is required to allow
7061 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7062 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7063 $use_old_bios_files = !min_version
($version, 2, 4);
7066 return ($use_old_bios_files, $machine_type);
7069 sub create_efidisk
($$$$$) {
7070 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7072 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7073 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7075 my $vars_size_b = -s
$ovmf_vars;
7076 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7077 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7078 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7080 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7081 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7083 return ($volid, $size/1024);
7086 sub vm_iothreads_list
{
7089 my $res = mon_cmd
($vmid, 'query-iothreads');
7092 foreach my $iothread (@$res) {
7093 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7100 my ($conf, $drive) = @_;
7104 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7106 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7112 my $controller = int($drive->{index} / $maxdev);
7113 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single') ?
"virtioscsi" : "scsihw";
7115 return ($maxdev, $controller, $controller_prefix);
7118 sub add_hyperv_enlightenments
{
7119 my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7121 return if $winversion < 6;
7122 return if $bios && $bios eq 'ovmf' && $winversion < 8;
7124 if ($gpu_passthrough || defined($hv_vendor_id)) {
7125 $hv_vendor_id //= 'proxmox';
7126 push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7129 if (min_version
($machine_version, 2, 3)) {
7130 push @$cpuFlags , 'hv_spinlocks=0x1fff';
7131 push @$cpuFlags , 'hv_vapic';
7132 push @$cpuFlags , 'hv_time';
7134 push @$cpuFlags , 'hv_spinlocks=0xffff';
7137 if (min_version
($machine_version, 2, 6)) {
7138 push @$cpuFlags , 'hv_reset';
7139 push @$cpuFlags , 'hv_vpindex';
7140 push @$cpuFlags , 'hv_runtime';
7143 if ($winversion >= 7) {
7144 push @$cpuFlags , 'hv_relaxed';
7146 if (min_version
($machine_version, 2, 12)) {
7147 push @$cpuFlags , 'hv_synic';
7148 push @$cpuFlags , 'hv_stimer';
7151 if (min_version
($machine_version, 3, 1)) {
7152 push @$cpuFlags , 'hv_ipi';
7157 sub windows_version
{
7160 return 0 if !$ostype;
7164 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7166 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7168 } elsif ($ostype =~ m/^win(\d+)$/) {
7175 sub resolve_dst_disk_format
{
7176 my ($storecfg, $storeid, $src_volname, $format) = @_;
7177 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7180 # if no target format is specified, use the source disk format as hint
7182 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7183 $format = qemu_img_format
($scfg, $src_volname);
7189 # test if requested format is supported - else use default
7190 my $supported = grep { $_ eq $format } @$validFormats;
7191 $format = $defFormat if !$supported;
7195 sub resolve_first_disk
{
7197 my @disks = PVE
::QemuServer
::valid_drive_names
();
7199 foreach my $ds (reverse @disks) {
7200 next if !$conf->{$ds};
7201 my $disk = PVE
::QemuServer
::parse_drive
($ds, $conf->{$ds});
7202 next if PVE
::QemuServer
::drive_is_cdrom
($disk);
7209 my ($uuid, $uuid_str);
7210 UUID
::generate
($uuid);
7211 UUID
::unparse
($uuid, $uuid_str);
7215 sub generate_smbios1_uuid
{
7216 return "uuid=".generate_uuid
();
7222 mon_cmd
($vmid, 'nbd-server-stop');
7225 sub create_reboot_request
{
7227 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7228 or die "failed to create reboot trigger file: $!\n";
7232 sub clear_reboot_request
{
7234 my $path = "/run/qemu-server/$vmid.reboot";
7237 $res = unlink($path);
7238 die "could not remove reboot request for $vmid: $!"
7239 if !$res && $! != POSIX
::ENOENT
;
7244 # bash completion helper
7246 sub complete_backup_archives
{
7247 my ($cmdname, $pname, $cvalue) = @_;
7249 my $cfg = PVE
::Storage
::config
();
7253 if ($cvalue =~ m/^([^:]+):/) {
7257 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7260 foreach my $id (keys %$data) {
7261 foreach my $item (@{$data->{$id}}) {
7262 next if $item->{format
} !~ m/^vma\.(gz|lzo)$/;
7263 push @$res, $item->{volid
} if defined($item->{volid
});
7270 my $complete_vmid_full = sub {
7273 my $idlist = vmstatus
();
7277 foreach my $id (keys %$idlist) {
7278 my $d = $idlist->{$id};
7279 if (defined($running)) {
7280 next if $d->{template
};
7281 next if $running && $d->{status
} ne 'running';
7282 next if !$running && $d->{status
} eq 'running';
7291 return &$complete_vmid_full();
7294 sub complete_vmid_stopped
{
7295 return &$complete_vmid_full(0);
7298 sub complete_vmid_running
{
7299 return &$complete_vmid_full(1);
7302 sub complete_storage
{
7304 my $cfg = PVE
::Storage
::config
();
7305 my $ids = $cfg->{ids
};
7308 foreach my $sid (keys %$ids) {
7309 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7310 next if !$ids->{$sid}->{content
}->{images
};
7317 sub complete_migration_storage
{
7318 my ($cmd, $param, $current_value, $all_args) = @_;
7320 my $targetnode = @$all_args[1];
7322 my $cfg = PVE
::Storage
::config
();
7323 my $ids = $cfg->{ids
};
7326 foreach my $sid (keys %$ids) {
7327 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
7328 next if !$ids->{$sid}->{content
}->{images
};