1 package PVE
::QemuServer
;
23 use Storable
qw(dclone);
25 use PVE
::Exception
qw(raise raise_param_exc);
27 use PVE
::Tools
qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
28 use PVE
::JSONSchema
qw(get_standard_option);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
34 use PVE
::RPCEnvironment
;
35 use PVE
::GuestHelpers
;
36 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr);
37 use PVE
::QemuServer
::Memory
;
38 use PVE
::QemuServer
::USB
qw(parse_usb_device);
39 use PVE
::QemuServer
::Cloudinit
;
42 use Time
::HiRes
qw(gettimeofday);
43 use File
::Copy
qw(copy);
46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
49 "$EDK2_FW_BASE/OVMF_CODE.fd",
50 "$EDK2_FW_BASE/OVMF_VARS.fd"
53 "$EDK2_FW_BASE/AAVMF_CODE.fd",
54 "$EDK2_FW_BASE/AAVMF_VARS.fd"
58 my $qemu_snap_storage = { rbd
=> 1 };
60 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
64 # Note about locking: we use flock on the config file protect
65 # against concurent actions.
66 # Aditionaly, we have a 'lock' setting in the config file. This
67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
68 # allowed when such lock is set. But you can ignore this kind of
69 # lock with the --skiplock flag.
71 cfs_register_file
('/qemu-server/',
75 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
76 description
=> "Some command save/restore state from this location.",
82 PVE
::JSONSchema
::register_standard_option
('pve-qm-image-format', {
84 enum
=> [qw(raw cow qcow qed qcow2 vmdk cloop)],
85 description
=> "The drive's backing file's data format.",
89 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
90 description
=> "Specifies the Qemu machine type.",
92 pattern
=> '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
97 #no warnings 'redefine';
100 my ($controller, $vmid, $option, $value) = @_;
102 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
103 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
107 my $nodename = PVE
::INotify
::nodename
();
109 mkdir "/etc/pve/nodes/$nodename";
110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
113 my $var_run_tmpdir = "/var/run/qemu-server";
114 mkdir $var_run_tmpdir;
116 my $lock_dir = "/var/lock/qemu-server";
119 my $cpu_vendor_list = {
121 486 => 'GenuineIntel',
122 pentium
=> 'GenuineIntel',
123 pentium2
=> 'GenuineIntel',
124 pentium3
=> 'GenuineIntel',
125 coreduo
=> 'GenuineIntel',
126 core2duo
=> 'GenuineIntel',
127 Conroe
=> 'GenuineIntel',
128 Penryn
=> 'GenuineIntel',
129 Nehalem
=> 'GenuineIntel',
130 'Nehalem-IBRS' => 'GenuineIntel',
131 Westmere
=> 'GenuineIntel',
132 'Westmere-IBRS' => 'GenuineIntel',
133 SandyBridge
=> 'GenuineIntel',
134 'SandyBridge-IBRS' => 'GenuineIntel',
135 IvyBridge
=> 'GenuineIntel',
136 'IvyBridge-IBRS' => 'GenuineIntel',
137 Haswell
=> 'GenuineIntel',
138 'Haswell-IBRS' => 'GenuineIntel',
139 'Haswell-noTSX' => 'GenuineIntel',
140 'Haswell-noTSX-IBRS' => 'GenuineIntel',
141 Broadwell
=> 'GenuineIntel',
142 'Broadwell-IBRS' => 'GenuineIntel',
143 'Broadwell-noTSX' => 'GenuineIntel',
144 'Broadwell-noTSX-IBRS' => 'GenuineIntel',
145 'Skylake-Client' => 'GenuineIntel',
146 'Skylake-Client-IBRS' => 'GenuineIntel',
147 'Skylake-Server' => 'GenuineIntel',
148 'Skylake-Server-IBRS' => 'GenuineIntel',
151 athlon
=> 'AuthenticAMD',
152 phenom
=> 'AuthenticAMD',
153 Opteron_G1
=> 'AuthenticAMD',
154 Opteron_G2
=> 'AuthenticAMD',
155 Opteron_G3
=> 'AuthenticAMD',
156 Opteron_G4
=> 'AuthenticAMD',
157 Opteron_G5
=> 'AuthenticAMD',
158 EPYC
=> 'AuthenticAMD',
159 'EPYC-IBPB' => 'AuthenticAMD',
161 # generic types, use vendor from host node
170 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb|md-clear)/;
174 description
=> "Emulated CPU type.",
176 enum
=> [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
181 description
=> "Do not identify as a KVM virtual machine.",
188 pattern
=> qr/[a-zA-Z0-9]{1,12}/,
189 format_description
=> 'vendor-id',
190 description
=> 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
194 description
=> "List of additional CPU flags separated by ';'."
195 . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
196 . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb', 'md-clear'.",
197 format_description
=> '+FLAG[;-FLAG...]',
199 pattern
=> qr/$cpu_flag(;$cpu_flag)*/,
208 enum
=> [qw(i6300esb ib700)],
209 description
=> "Watchdog type to emulate.",
210 default => 'i6300esb',
215 enum
=> [qw(reset shutdown poweroff pause debug none)],
216 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
220 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
224 description
=> "Enable/disable Qemu GuestAgent.",
229 fstrim_cloned_disks
=> {
230 description
=> "Run fstrim after cloning/moving a disk.",
239 description
=> "Select the VGA type.",
244 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
247 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
259 description
=> "The size of the file in MB.",
263 pattern
=> '[a-zA-Z0-9\-]+',
265 format_description
=> 'string',
266 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
274 description
=> "Specifies whether a VM will be started during system bootup.",
280 description
=> "Automatic restart after crash (currently ignored).",
285 type
=> 'string', format
=> 'pve-hotplug-features',
286 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
287 default => 'network,disk,usb',
292 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
298 description
=> "Lock/unlock the VM.",
299 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
304 description
=> "Limit of CPU usage.",
305 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
313 description
=> "CPU weight for a VM.",
314 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
322 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
329 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
335 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
343 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
344 "It should not be necessary to set it.",
345 enum
=> PVE
::Tools
::kvmkeymaplist
(),
350 type
=> 'string', format
=> 'dns-name',
351 description
=> "Set a name for the VM. Only used on the configuration web interface.",
356 description
=> "SCSI controller model",
357 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
363 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
368 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
369 description
=> "Specify guest operating system.",
370 verbose_description
=> <<EODESC,
371 Specify guest operating system. This is used to enable special
372 optimization/features for specific operating systems:
375 other;; unspecified OS
376 wxp;; Microsoft Windows XP
377 w2k;; Microsoft Windows 2000
378 w2k3;; Microsoft Windows 2003
379 w2k8;; Microsoft Windows 2008
380 wvista;; Microsoft Windows Vista
381 win7;; Microsoft Windows 7
382 win8;; Microsoft Windows 8/2012/2012r2
383 win10;; Microsoft Windows 10/2016
384 l24;; Linux 2.4 Kernel
385 l26;; Linux 2.6/3.X Kernel
386 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
392 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
393 pattern
=> '[acdn]{1,4}',
398 type
=> 'string', format
=> 'pve-qm-bootdisk',
399 description
=> "Enable booting from specified disk.",
400 pattern
=> '(ide|sata|scsi|virtio)\d+',
405 description
=> "The number of CPUs. Please use option -sockets instead.",
412 description
=> "The number of CPU sockets.",
419 description
=> "The number of cores per socket.",
426 description
=> "Enable/disable NUMA.",
432 description
=> "Enable/disable hugepages memory.",
433 enum
=> [qw(any 2 1024)],
438 description
=> "Number of hotplugged vcpus.",
445 description
=> "Enable/disable ACPI.",
450 description
=> "Enable/disable Qemu GuestAgent and its properties.",
452 format
=> $agent_fmt,
457 description
=> "Enable/disable KVM hardware virtualization.",
463 description
=> "Enable/disable time drift fix.",
469 description
=> "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
474 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
478 type
=> 'string', format
=> $vga_fmt,
479 description
=> "Configure the VGA hardware.",
480 verbose_description
=> "Configure the VGA Hardware. If you want to use ".
481 "high resolution modes (>= 1280x1024x16) you may need to increase " .
482 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
483 "is 'std' for all OS types besides some Windows versions (XP and " .
484 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
485 "display server. For win* OS you can select how many independent " .
486 "displays you want, Linux guests can add displays them self.\n".
487 "You can also run without any graphic card, using a serial device as terminal.",
491 type
=> 'string', format
=> 'pve-qm-watchdog',
492 description
=> "Create a virtual hardware watchdog device.",
493 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled" .
494 " (by a guest action), the watchdog must be periodically polled " .
495 "by an agent inside the guest or else the watchdog will reset " .
496 "the guest (or execute the respective action specified)",
501 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
502 description
=> "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
503 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
506 startup
=> get_standard_option
('pve-startup-order'),
510 description
=> "Enable/disable Template.",
516 description
=> "Arbitrary arguments passed to kvm.",
517 verbose_description
=> <<EODESCR,
518 Arbitrary arguments passed to kvm, for example:
520 args: -no-reboot -no-hpet
522 NOTE: this option is for experts only.
529 description
=> "Enable/disable the USB tablet device.",
530 verbose_description
=> "Enable/disable the USB tablet device. This device is " .
531 "usually needed to allow absolute mouse positioning with VNC. " .
532 "Else the mouse runs out of sync with normal VNC clients. " .
533 "If you're running lots of console-only guests on one host, " .
534 "you may consider disabling this to save some context switches. " .
535 "This is turned off by default if you use spice (-vga=qxl).",
540 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
544 migrate_downtime
=> {
547 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
553 type
=> 'string', format
=> 'pve-qm-ide',
554 typetext
=> '<volume>',
555 description
=> "This is an alias for option -ide2",
559 description
=> "Emulated CPU type.",
563 parent
=> get_standard_option
('pve-snapshot-name', {
565 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
569 description
=> "Timestamp for snapshots.",
575 type
=> 'string', format
=> 'pve-volume-id',
576 description
=> "Reference to a volume which stores the VM state. This is used internally for snapshots.",
578 vmstatestorage
=> get_standard_option
('pve-storage-id', {
579 description
=> "Default storage for VM state volumes/files.",
582 runningmachine
=> get_standard_option
('pve-qemu-machine', {
583 description
=> "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
585 machine
=> get_standard_option
('pve-qemu-machine'),
587 description
=> "Virtual processor architecture. Defaults to the host.",
590 enum
=> [qw(x86_64 aarch64)],
593 description
=> "Specify SMBIOS type 1 fields.",
594 type
=> 'string', format
=> 'pve-qm-smbios1',
601 description
=> "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
607 enum
=> [ qw(seabios ovmf) ],
608 description
=> "Select BIOS implementation.",
609 default => 'seabios',
613 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
614 format_description
=> 'UUID',
615 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
616 verbose_description
=> "The VM generation ID (vmgenid) device exposes a".
617 " 128-bit integer value identifier to the guest OS. This allows to".
618 " notify the guest operating system when the virtual machine is".
619 " executed with a different configuration (e.g. snapshot execution".
620 " or creation from a template). The guest operating system notices".
621 " the change, and is then able to react as appropriate by marking".
622 " its copies of distributed databases as dirty, re-initializing its".
623 " random number generator, etc.\n".
624 "Note that auto-creation only works when done throug API/CLI create".
625 " or update methods, but not when manually editing the config file.",
626 default => "1 (autogenerated)",
631 format
=> 'pve-volume-id',
633 description
=> "Script that will be executed during various steps in the vms lifetime.",
637 format
=> $ivshmem_fmt,
638 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
643 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
644 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
653 description
=> 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
654 format
=> 'pve-volume-id',
655 format_description
=> 'volume',
660 description
=> 'Specify a custom file containing all network data passed to the VM via cloud-init.',
661 format
=> 'pve-volume-id',
662 format_description
=> 'volume',
667 description
=> 'Specify a custom file containing all user data passed to the VM via cloud-init.',
668 format
=> 'pve-volume-id',
669 format_description
=> 'volume',
672 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
674 my $confdesc_cloudinit = {
678 description
=> 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
679 enum
=> ['configdrive2', 'nocloud'],
684 description
=> "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
689 description
=> 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
694 description
=> 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
695 format
=> 'pve-qm-cicustom',
700 description
=> "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
704 type
=> 'string', format
=> 'address-list',
705 description
=> "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
710 format
=> 'urlencoded',
711 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
715 # what about other qemu settings ?
717 #machine => 'string',
730 ##soundhw => 'string',
732 while (my ($k, $v) = each %$confdesc) {
733 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
736 my $MAX_IDE_DISKS = 4;
737 my $MAX_SCSI_DISKS = 14;
738 my $MAX_VIRTIO_DISKS = 16;
739 my $MAX_SATA_DISKS = 6;
740 my $MAX_USB_DEVICES = 5;
742 my $MAX_UNUSED_DISKS = 256;
743 my $MAX_HOSTPCI_DEVICES = 4;
744 my $MAX_SERIAL_PORTS = 4;
745 my $MAX_PARALLEL_PORTS = 3;
751 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
752 description
=> "CPUs accessing this NUMA node.",
753 format_description
=> "id[-id];...",
757 description
=> "Amount of memory this NUMA node provides.",
762 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
763 description
=> "Host NUMA nodes to use.",
764 format_description
=> "id[-id];...",
769 enum
=> [qw(preferred bind interleave)],
770 description
=> "NUMA allocation policy.",
774 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
777 type
=> 'string', format
=> $numa_fmt,
778 description
=> "NUMA topology.",
780 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
782 for (my $i = 0; $i < $MAX_NUMA; $i++) {
783 $confdesc->{"numa$i"} = $numadesc;
786 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
787 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
788 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
789 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
791 my $net_fmt_bridge_descr = <<__EOD__;
792 Bridge to attach the network device to. The Proxmox VE standard bridge
795 If you do not specify a bridge, we create a kvm user (NATed) network
796 device, which provides DHCP and DNS services. The following addresses
803 The DHCP server assign addresses to the guest starting from 10.0.2.15.
807 macaddr
=> get_standard_option
('mac-addr', {
808 description
=> "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
812 description
=> "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
813 enum
=> $nic_model_list,
816 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
819 description
=> $net_fmt_bridge_descr,
820 format_description
=> 'bridge',
825 minimum
=> 0, maximum
=> 16,
826 description
=> 'Number of packet queues to be used on the device.',
832 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
837 minimum
=> 1, maximum
=> 4094,
838 description
=> 'VLAN tag to apply to packets on this interface.',
843 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
844 description
=> 'VLAN trunks to pass through this interface.',
845 format_description
=> 'vlanid[;vlanid...]',
850 description
=> 'Whether this interface should be protected by the firewall.',
855 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
862 type
=> 'string', format
=> $net_fmt,
863 description
=> "Specify network devices.",
866 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
871 format
=> 'pve-ipv4-config',
872 format_description
=> 'IPv4Format/CIDR',
873 description
=> 'IPv4 address in CIDR format.',
880 format_description
=> 'GatewayIPv4',
881 description
=> 'Default gateway for IPv4 traffic.',
887 format
=> 'pve-ipv6-config',
888 format_description
=> 'IPv6Format/CIDR',
889 description
=> 'IPv6 address in CIDR format.',
896 format_description
=> 'GatewayIPv6',
897 description
=> 'Default gateway for IPv6 traffic.',
902 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
905 type
=> 'string', format
=> 'pve-qm-ipconfig',
906 description
=> <<'EODESCR',
907 cloud-init: Specify IP addresses and gateways for the corresponding interface.
909 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
911 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
912 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
914 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
917 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
919 for (my $i = 0; $i < $MAX_NETS; $i++) {
920 $confdesc->{"net$i"} = $netdesc;
921 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
924 foreach my $key (keys %$confdesc_cloudinit) {
925 $confdesc->{$key} = $confdesc_cloudinit->{$key};
928 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
929 sub verify_volume_id_or_qm_path
{
930 my ($volid, $noerr) = @_;
932 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
936 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
937 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
939 return undef if $noerr;
947 my %drivedesc_base = (
948 volume
=> { alias
=> 'file' },
951 format
=> 'pve-volume-id-or-qm-path',
953 format_description
=> 'volume',
954 description
=> "The drive's backing volume.",
958 enum
=> [qw(cdrom disk)],
959 description
=> "The drive's media type.",
965 description
=> "Force the drive's physical geometry to have a specific cylinder count.",
970 description
=> "Force the drive's physical geometry to have a specific head count.",
975 description
=> "Force the drive's physical geometry to have a specific sector count.",
980 enum
=> [qw(none lba auto)],
981 description
=> "Force disk geometry bios translation mode.",
986 description
=> "Controls qemu's snapshot mode feature."
987 . " If activated, changes made to the disk are temporary and will"
988 . " be discarded when the VM is shutdown.",
993 enum
=> [qw(none writethrough writeback unsafe directsync)],
994 description
=> "The drive's cache mode",
997 format
=> get_standard_option
('pve-qm-image-format'),
1000 format
=> 'disk-size',
1001 format_description
=> 'DiskSize',
1002 description
=> "Disk size. This is purely informational and has no effect.",
1007 description
=> "Whether the drive should be included when making backups.",
1012 description
=> 'Whether the drive should considered for replication jobs.',
1018 enum
=> [qw(ignore report stop)],
1019 description
=> 'Read error action.',
1024 enum
=> [qw(enospc ignore report stop)],
1025 description
=> 'Write error action.',
1030 enum
=> [qw(native threads)],
1031 description
=> 'AIO type to use.',
1036 enum
=> [qw(ignore on)],
1037 description
=> 'Controls whether to pass discard/trim requests to the underlying storage.',
1042 description
=> 'Controls whether to detect and try to optimize writes of zeroes.',
1047 format
=> 'urlencoded',
1048 format_description
=> 'serial',
1049 maxLength
=> 20*3, # *3 since it's %xx url enoded
1050 description
=> "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1055 description
=> 'Mark this locally-managed volume as available on all nodes',
1056 verbose_description
=> "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1062 my %iothread_fmt = ( iothread
=> {
1064 description
=> "Whether to use iothreads for this drive",
1071 format
=> 'urlencoded',
1072 format_description
=> 'model',
1073 maxLength
=> 40*3, # *3 since it's %xx url enoded
1074 description
=> "The drive's reported model name, url-encoded, up to 40 bytes long.",
1082 description
=> "Number of queues.",
1088 my %scsiblock_fmt = (
1091 description
=> "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1100 description
=> "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1108 pattern
=> qr/^(0x)[0-9a-fA-F]{16}/,
1109 format_description
=> 'wwn',
1110 description
=> "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1115 my $add_throttle_desc = sub {
1116 my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1119 format_description
=> $unit,
1120 description
=> "Maximum $what in $longunit.",
1123 $d->{minimum
} = $minimum if defined($minimum);
1124 $drivedesc_base{$key} = $d;
1126 # throughput: (leaky bucket)
1127 $add_throttle_desc->('bps', 'integer', 'r/w speed', 'bps', 'bytes per second');
1128 $add_throttle_desc->('bps_rd', 'integer', 'read speed', 'bps', 'bytes per second');
1129 $add_throttle_desc->('bps_wr', 'integer', 'write speed', 'bps', 'bytes per second');
1130 $add_throttle_desc->('mbps', 'number', 'r/w speed', 'mbps', 'megabytes per second');
1131 $add_throttle_desc->('mbps_rd', 'number', 'read speed', 'mbps', 'megabytes per second');
1132 $add_throttle_desc->('mbps_wr', 'number', 'write speed', 'mbps', 'megabytes per second');
1133 $add_throttle_desc->('iops', 'integer', 'r/w I/O', 'iops', 'operations per second');
1134 $add_throttle_desc->('iops_rd', 'integer', 'read I/O', 'iops', 'operations per second');
1135 $add_throttle_desc->('iops_wr', 'integer', 'write I/O', 'iops', 'operations per second');
1137 # pools: (pool of IO before throttling starts taking effect)
1138 $add_throttle_desc->('mbps_max', 'number', 'unthrottled r/w pool', 'mbps', 'megabytes per second');
1139 $add_throttle_desc->('mbps_rd_max', 'number', 'unthrottled read pool', 'mbps', 'megabytes per second');
1140 $add_throttle_desc->('mbps_wr_max', 'number', 'unthrottled write pool', 'mbps', 'megabytes per second');
1141 $add_throttle_desc->('iops_max', 'integer', 'unthrottled r/w I/O pool', 'iops', 'operations per second');
1142 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool', 'iops', 'operations per second');
1143 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1146 $add_throttle_desc->('bps_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1147 $add_throttle_desc->('bps_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1148 $add_throttle_desc->('bps_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1149 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1150 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1151 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1154 $drivedesc_base{'bps_rd_length'} = { alias
=> 'bps_rd_max_length' };
1155 $drivedesc_base{'bps_wr_length'} = { alias
=> 'bps_wr_max_length' };
1156 $drivedesc_base{'iops_rd_length'} = { alias
=> 'iops_rd_max_length' };
1157 $drivedesc_base{'iops_wr_length'} = { alias
=> 'iops_wr_max_length' };
1165 PVE
::JSONSchema
::register_format
("pve-qm-ide", $ide_fmt);
1169 type
=> 'string', format
=> $ide_fmt,
1170 description
=> "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1172 PVE
::JSONSchema
::register_standard_option
("pve-qm-ide", $idedesc);
1184 type
=> 'string', format
=> $scsi_fmt,
1185 description
=> "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1187 PVE
::JSONSchema
::register_standard_option
("pve-qm-scsi", $scsidesc);
1196 type
=> 'string', format
=> $sata_fmt,
1197 description
=> "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1199 PVE
::JSONSchema
::register_standard_option
("pve-qm-sata", $satadesc);
1207 type
=> 'string', format
=> $virtio_fmt,
1208 description
=> "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1210 PVE
::JSONSchema
::register_standard_option
("pve-qm-virtio", $virtiodesc);
1212 my $alldrive_fmt = {
1223 volume
=> { alias
=> 'file' },
1226 format
=> 'pve-volume-id-or-qm-path',
1228 format_description
=> 'volume',
1229 description
=> "The drive's backing volume.",
1231 format
=> get_standard_option
('pve-qm-image-format'),
1234 format
=> 'disk-size',
1235 format_description
=> 'DiskSize',
1236 description
=> "Disk size. This is purely informational and has no effect.",
1241 my $efidisk_desc = {
1243 type
=> 'string', format
=> $efidisk_fmt,
1244 description
=> "Configure a Disk for storing EFI vars",
1247 PVE
::JSONSchema
::register_standard_option
("pve-qm-efidisk", $efidisk_desc);
1252 type
=> 'string', format
=> 'pve-qm-usb-device',
1253 format_description
=> 'HOSTUSBDEVICE|spice',
1254 description
=> <<EODESCR,
1255 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1257 'bus-port(.port)*' (decimal numbers) or
1258 'vendor_id:product_id' (hexadeciaml numbers) or
1261 You can use the 'lsusb -t' command to list existing usb devices.
1263 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1265 The value 'spice' can be used to add a usb redirection devices for spice.
1271 description
=> "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1278 type
=> 'string', format
=> $usb_fmt,
1279 description
=> "Configure an USB device (n is 0 to 4).",
1281 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1283 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1288 pattern
=> qr/$PCIRE(;$PCIRE)*/,
1289 format_description
=> 'HOSTPCIID[;HOSTPCIID2...]',
1290 description
=> <<EODESCR,
1291 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1292 of PCI virtual functions of the host. HOSTPCIID syntax is:
1294 'bus:dev.func' (hexadecimal numbers)
1296 You can us the 'lspci' command to list existing PCI devices.
1301 description
=> "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1307 pattern
=> '[^,;]+',
1308 format_description
=> 'string',
1309 description
=> "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1314 description
=> "Choose the PCI-express bus (needs the 'q35' machine model).",
1320 description
=> "Enable vfio-vga device support.",
1326 format_description
=> 'string',
1327 pattern
=> '[^/\.:]+',
1329 description
=> <<EODESCR
1330 The type of mediated device to use.
1331 An instance of this type will be created on startup of the VM and
1332 will be cleaned up when the VM stops.
1336 PVE
::JSONSchema
::register_format
('pve-qm-hostpci', $hostpci_fmt);
1340 type
=> 'string', format
=> 'pve-qm-hostpci',
1341 description
=> "Map host PCI devices into guest.",
1342 verbose_description
=> <<EODESCR,
1343 Map host PCI devices into guest.
1345 NOTE: This option allows direct access to host hardware. So it is no longer
1346 possible to migrate such machines - use with special care.
1348 CAUTION: Experimental! User reported problems with this option.
1351 PVE
::JSONSchema
::register_standard_option
("pve-qm-hostpci", $hostpcidesc);
1356 pattern
=> '(/dev/.+|socket)',
1357 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1358 verbose_description
=> <<EODESCR,
1359 Create a serial device inside the VM (n is 0 to 3), and pass through a
1360 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1361 host side (use 'qm terminal' to open a terminal connection).
1363 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1365 CAUTION: Experimental! User reported problems with this option.
1372 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1373 description
=> "Map host parallel devices (n is 0 to 2).",
1374 verbose_description
=> <<EODESCR,
1375 Map host parallel devices (n is 0 to 2).
1377 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1379 CAUTION: Experimental! User reported problems with this option.
1383 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1384 $confdesc->{"parallel$i"} = $paralleldesc;
1387 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1388 $confdesc->{"serial$i"} = $serialdesc;
1391 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
1392 $confdesc->{"hostpci$i"} = $hostpcidesc;
1395 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) {
1396 $drivename_hash->{"ide$i"} = 1;
1397 $confdesc->{"ide$i"} = $idedesc;
1400 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) {
1401 $drivename_hash->{"sata$i"} = 1;
1402 $confdesc->{"sata$i"} = $satadesc;
1405 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) {
1406 $drivename_hash->{"scsi$i"} = 1;
1407 $confdesc->{"scsi$i"} = $scsidesc ;
1410 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) {
1411 $drivename_hash->{"virtio$i"} = 1;
1412 $confdesc->{"virtio$i"} = $virtiodesc;
1415 $drivename_hash->{efidisk0
} = 1;
1416 $confdesc->{efidisk0
} = $efidisk_desc;
1418 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1419 $confdesc->{"usb$i"} = $usbdesc;
1424 type
=> 'string', format
=> 'pve-volume-id',
1425 description
=> "Reference to unused volumes. This is used internally, and should not be modified manually.",
1428 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1429 $confdesc->{"unused$i"} = $unuseddesc;
1432 my $kvm_api_version = 0;
1435 return $kvm_api_version if $kvm_api_version;
1437 open my $fh, '<', '/dev/kvm'
1440 # 0xae00 => KVM_GET_API_VERSION
1441 $kvm_api_version = ioctl($fh, 0xae00, 0);
1443 return $kvm_api_version;
1446 my $kvm_user_version;
1448 sub kvm_user_version
{
1450 return $kvm_user_version if $kvm_user_version;
1452 $kvm_user_version = 'unknown';
1456 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1457 $kvm_user_version = $2;
1461 eval { run_command
("kvm -version", outfunc
=> $code); };
1464 return $kvm_user_version;
1468 sub kernel_has_vhost_net
{
1469 return -c
'/dev/vhost-net';
1472 sub valid_drive_names
{
1473 # order is important - used to autoselect boot disk
1474 return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1475 (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1476 (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1477 (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1481 sub is_valid_drivename
{
1484 return defined($drivename_hash->{$dev});
1489 return defined($confdesc->{$key});
1493 return $nic_model_list;
1496 sub os_list_description
{
1500 wxp
=> 'Windows XP',
1501 w2k
=> 'Windows 2000',
1502 w2k3
=>, 'Windows 2003',
1503 w2k8
=> 'Windows 2008',
1504 wvista
=> 'Windows Vista',
1505 win7
=> 'Windows 7',
1506 win8
=> 'Windows 8/2012',
1507 win10
=> 'Windows 10/2016',
1515 sub get_cdrom_path
{
1517 return $cdrom_path if $cdrom_path;
1519 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1520 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1521 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1525 my ($storecfg, $vmid, $cdrom) = @_;
1527 if ($cdrom eq 'cdrom') {
1528 return get_cdrom_path
();
1529 } elsif ($cdrom eq 'none') {
1531 } elsif ($cdrom =~ m
|^/|) {
1534 return PVE
::Storage
::path
($storecfg, $cdrom);
1538 # try to convert old style file names to volume IDs
1539 sub filename_to_volume_id
{
1540 my ($vmid, $file, $media) = @_;
1542 if (!($file eq 'none' || $file eq 'cdrom' ||
1543 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1545 return undef if $file =~ m
|/|;
1547 if ($media && $media eq 'cdrom') {
1548 $file = "local:iso/$file";
1550 $file = "local:$vmid/$file";
1557 sub verify_media_type
{
1558 my ($opt, $vtype, $media) = @_;
1563 if ($media eq 'disk') {
1565 } elsif ($media eq 'cdrom') {
1568 die "internal error";
1571 return if ($vtype eq $etype);
1573 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1576 sub cleanup_drive_path
{
1577 my ($opt, $storecfg, $drive) = @_;
1579 # try to convert filesystem paths to volume IDs
1581 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1582 ($drive->{file
} !~ m
|^/dev/.+|) &&
1583 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1584 ($drive->{file
} !~ m/^\d+$/)) {
1585 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1586 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1587 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1588 verify_media_type
($opt, $vtype, $drive->{media
});
1589 $drive->{file
} = $volid;
1592 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1595 sub parse_hotplug_features
{
1600 return $res if $data eq '0';
1602 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1604 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1605 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1608 die "invalid hotplug feature '$feature'\n";
1614 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1615 sub pve_verify_hotplug_features
{
1616 my ($value, $noerr) = @_;
1618 return $value if parse_hotplug_features
($value);
1620 return undef if $noerr;
1622 die "unable to parse hotplug option\n";
1625 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1626 # [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1627 # [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1628 # [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1629 # [,iothread=on][,serial=serial][,model=model]
1632 my ($key, $data) = @_;
1634 my ($interface, $index);
1636 if ($key =~ m/^([^\d]+)(\d+)$/) {
1643 my $desc = $key =~ /^unused\d+$/ ?
$alldrive_fmt
1644 : $confdesc->{$key}->{format
};
1646 warn "invalid drive key: $key\n";
1649 my $res = eval { PVE
::JSONSchema
::parse_property_string
($desc, $data) };
1650 return undef if !$res;
1651 $res->{interface
} = $interface;
1652 $res->{index} = $index;
1655 foreach my $opt (qw(bps bps_rd bps_wr)) {
1656 if (my $bps = defined(delete $res->{$opt})) {
1657 if (defined($res->{"m$opt"})) {
1658 warn "both $opt and m$opt specified\n";
1662 $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1666 # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1667 for my $requirement (
1668 [mbps_max
=> 'mbps'],
1669 [mbps_rd_max
=> 'mbps_rd'],
1670 [mbps_wr_max
=> 'mbps_wr'],
1671 [miops_max
=> 'miops'],
1672 [miops_rd_max
=> 'miops_rd'],
1673 [miops_wr_max
=> 'miops_wr'],
1674 [bps_max_length
=> 'mbps_max'],
1675 [bps_rd_max_length
=> 'mbps_rd_max'],
1676 [bps_wr_max_length
=> 'mbps_wr_max'],
1677 [iops_max_length
=> 'iops_max'],
1678 [iops_rd_max_length
=> 'iops_rd_max'],
1679 [iops_wr_max_length
=> 'iops_wr_max']) {
1680 my ($option, $requires) = @$requirement;
1681 if ($res->{$option} && !$res->{$requires}) {
1682 warn "$option requires $requires\n";
1687 return undef if $error;
1689 return undef if $res->{mbps_rd
} && $res->{mbps
};
1690 return undef if $res->{mbps_wr
} && $res->{mbps
};
1691 return undef if $res->{iops_rd
} && $res->{iops
};
1692 return undef if $res->{iops_wr
} && $res->{iops
};
1694 if ($res->{media
} && ($res->{media
} eq 'cdrom')) {
1695 return undef if $res->{snapshot
} || $res->{trans
} || $res->{format
};
1696 return undef if $res->{heads
} || $res->{secs
} || $res->{cyls
};
1697 return undef if $res->{interface
} eq 'virtio';
1700 if (my $size = $res->{size
}) {
1701 return undef if !defined($res->{size
} = PVE
::JSONSchema
::parse_size
($size));
1708 my ($vmid, $drive) = @_;
1709 my $data = { %$drive };
1710 delete $data->{$_} for qw(index interface);
1711 return PVE
::JSONSchema
::print_property_string
($data, $alldrive_fmt);
1715 my($fh, $noerr) = @_;
1718 my $SG_GET_VERSION_NUM = 0x2282;
1720 my $versionbuf = "\x00" x
8;
1721 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1723 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1726 my $version = unpack("I", $versionbuf);
1727 if ($version < 30000) {
1728 die "scsi generic interface too old\n" if !$noerr;
1732 my $buf = "\x00" x
36;
1733 my $sensebuf = "\x00" x
8;
1734 my $cmd = pack("C x3 C x1", 0x12, 36);
1736 # see /usr/include/scsi/sg.h
1737 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1739 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1740 length($sensebuf), 0, length($buf), $buf,
1741 $cmd, $sensebuf, 6000);
1743 $ret = ioctl($fh, $SG_IO, $packet);
1745 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1749 my @res = unpack($sg_io_hdr_t, $packet);
1750 if ($res[17] || $res[18]) {
1751 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1756 (my $byte0, my $byte1, $res->{vendor
},
1757 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1759 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1760 $res->{type
} = $byte0 & 31;
1768 my $fh = IO
::File-
>new("+<$path") || return undef;
1769 my $res = scsi_inquiry
($fh, 1);
1775 sub machine_type_is_q35
{
1778 return $conf->{machine
} && ($conf->{machine
} =~ m/q35/) ?
1 : 0;
1781 sub print_tabletdevice_full
{
1782 my ($conf, $arch) = @_;
1784 my $q35 = machine_type_is_q35
($conf);
1786 # we use uhci for old VMs because tablet driver was buggy in older qemu
1788 if (machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1794 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1797 sub print_keyboarddevice_full
{
1798 my ($conf, $arch, $machine) = @_;
1800 return undef if $arch ne 'aarch64';
1802 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1805 sub print_drivedevice_full
{
1806 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1811 if ($drive->{interface
} eq 'virtio') {
1812 my $pciaddr = print_pci_addr
("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1813 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1814 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread
};
1815 } elsif ($drive->{interface
} eq 'scsi') {
1817 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1818 my $unit = $drive->{index} % $maxdev;
1819 my $devicetype = 'hd';
1821 if (drive_is_cdrom
($drive)) {
1824 if ($drive->{file
} =~ m
|^/|) {
1825 $path = $drive->{file
};
1826 if (my $info = path_is_scsi
($path)) {
1827 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1828 $devicetype = 'block';
1829 } elsif ($info->{type
} == 1) { # tape
1830 $devicetype = 'generic';
1834 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1837 if($path =~ m/^iscsi\:\/\
//){
1838 $devicetype = 'generic';
1842 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1843 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1845 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1848 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1849 $device .= ",rotation_rate=1";
1851 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1853 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1854 my $maxdev = ($drive->{interface
} eq 'sata') ?
$MAX_SATA_DISKS : 2;
1855 my $controller = int($drive->{index} / $maxdev);
1856 my $unit = $drive->{index} % $maxdev;
1857 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1859 $device = "ide-$devicetype";
1860 if ($drive->{interface
} eq 'ide') {
1861 $device .= ",bus=ide.$controller,unit=$unit";
1863 $device .= ",bus=ahci$controller.$unit";
1865 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1867 if ($devicetype eq 'hd') {
1868 if (my $model = $drive->{model
}) {
1869 $model = URI
::Escape
::uri_unescape
($model);
1870 $device .= ",model=$model";
1872 if ($drive->{ssd
}) {
1873 $device .= ",rotation_rate=1";
1876 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1877 } elsif ($drive->{interface
} eq 'usb') {
1879 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1881 die "unsupported interface type";
1884 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1886 if (my $serial = $drive->{serial
}) {
1887 $serial = URI
::Escape
::uri_unescape
($serial);
1888 $device .= ",serial=$serial";
1895 sub get_initiator_name
{
1898 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return undef;
1899 while (defined(my $line = <$fh>)) {
1900 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1909 sub print_drive_full
{
1910 my ($storecfg, $vmid, $drive) = @_;
1913 my $volid = $drive->{file
};
1916 if (drive_is_cdrom
($drive)) {
1917 $path = get_iso_path
($storecfg, $vmid, $volid);
1919 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1921 $path = PVE
::Storage
::path
($storecfg, $volid);
1922 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1923 $format = qemu_img_format
($scfg, $volname);
1931 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1932 foreach my $o (@qemu_drive_options) {
1933 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1936 # snapshot only accepts on|off
1937 if (defined($drive->{snapshot
})) {
1938 my $v = $drive->{snapshot
} ?
'on' : 'off';
1939 $opts .= ",snapshot=$v";
1942 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1943 my ($dir, $qmpname) = @$type;
1944 if (my $v = $drive->{"mbps$dir"}) {
1945 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1947 if (my $v = $drive->{"mbps${dir}_max"}) {
1948 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1950 if (my $v = $drive->{"bps${dir}_max_length"}) {
1951 $opts .= ",throttling.bps$qmpname-max-length=$v";
1953 if (my $v = $drive->{"iops${dir}"}) {
1954 $opts .= ",throttling.iops$qmpname=$v";
1956 if (my $v = $drive->{"iops${dir}_max"}) {
1957 $opts .= ",throttling.iops$qmpname-max=$v";
1959 if (my $v = $drive->{"iops${dir}_max_length"}) {
1960 $opts .= ",throttling.iops$qmpname-max-length=$v";
1964 $opts .= ",format=$format" if $format && !$drive->{format
};
1966 my $cache_direct = 0;
1968 if (my $cache = $drive->{cache
}) {
1969 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1970 } elsif (!drive_is_cdrom
($drive)) {
1971 $opts .= ",cache=none";
1975 # aio native works only with O_DIRECT
1976 if (!$drive->{aio
}) {
1978 $opts .= ",aio=native";
1980 $opts .= ",aio=threads";
1984 if (!drive_is_cdrom
($drive)) {
1986 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1987 $detectzeroes = 'off';
1988 } elsif ($drive->{discard
}) {
1989 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1991 # This used to be our default with discard not being specified:
1992 $detectzeroes = 'on';
1994 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1997 my $pathinfo = $path ?
"file=$path," : '';
1999 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2002 sub print_netdevice_full
{
2003 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2005 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
2007 my $device = $net->{model
};
2008 if ($net->{model
} eq 'virtio') {
2009 $device = 'virtio-net-pci';
2012 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
2013 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2014 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
2015 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2016 my $vectors = $net->{queues
} * 2 + 2;
2017 $tmpstr .= ",vectors=$vectors,mq=on";
2019 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
2021 if ($use_old_bios_files) {
2023 if ($device eq 'virtio-net-pci') {
2024 $romfile = 'pxe-virtio.rom';
2025 } elsif ($device eq 'e1000') {
2026 $romfile = 'pxe-e1000.rom';
2027 } elsif ($device eq 'ne2k') {
2028 $romfile = 'pxe-ne2k_pci.rom';
2029 } elsif ($device eq 'pcnet') {
2030 $romfile = 'pxe-pcnet.rom';
2031 } elsif ($device eq 'rtl8139') {
2032 $romfile = 'pxe-rtl8139.rom';
2034 $tmpstr .= ",romfile=$romfile" if $romfile;
2040 sub print_netdev_full
{
2041 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2044 if ($netid =~ m/^net(\d+)$/) {
2048 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
2050 my $ifname = "tap${vmid}i$i";
2052 # kvm uses TUNSETIFF ioctl, and that limits ifname length
2053 die "interface name '$ifname' is too long (max 15 character)\n"
2054 if length($ifname) >= 16;
2056 my $vhostparam = '';
2057 if (is_native
($arch)) {
2058 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
2061 my $vmname = $conf->{name
} || "vm$vmid";
2064 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
2066 if ($net->{bridge
}) {
2067 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2069 $netdev = "type=user,id=$netid,hostname=$vmname";
2072 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
2078 sub print_cpu_device
{
2079 my ($conf, $id) = @_;
2081 my $kvm = $conf->{kvm
} // 1;
2082 my $cpu = $kvm ?
"kvm64" : "qemu64";
2083 if (my $cputype = $conf->{cpu
}) {
2084 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
2085 or die "Cannot parse cpu description: $cputype\n";
2086 $cpu = $cpuconf->{cputype
};
2089 my $cores = $conf->{cores
} || 1;
2091 my $current_core = ($id - 1) % $cores;
2092 my $current_socket = int(($id - 1 - $current_core)/$cores);
2094 return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2098 'cirrus' => 'cirrus-vga',
2100 'vmware' => 'vmware-svga',
2101 'virtio' => 'virtio-vga',
2104 sub print_vga_device
{
2105 my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2107 my $type = $vga_map->{$vga->{type
}};
2108 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2109 $type = 'virtio-gpu';
2111 my $vgamem_mb = $vga->{memory
};
2113 $type = $id ?
'qxl' : 'qxl-vga';
2115 die "no devicetype for $vga->{type}\n" if !$type;
2119 if ($vga->{type
} eq 'virtio') {
2120 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
2121 $memory = ",max_hostmem=$bytes";
2123 # from https://www.spice-space.org/multiple-monitors.html
2124 $memory = ",vgamem_mb=$vga->{memory}";
2125 my $ram = $vgamem_mb * 4;
2126 my $vram = $vgamem_mb * 2;
2127 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2129 $memory = ",vgamem_mb=$vga->{memory}";
2131 } elsif ($qxlnum && $id) {
2132 $memory = ",ram_size=67108864,vram_size=33554432";
2135 my $q35 = machine_type_is_q35
($conf);
2136 my $vgaid = "vga" . ($id // '');
2139 if ($q35 && $vgaid eq 'vga') {
2140 # the first display uses pcie.0 bus on q35 machines
2141 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
2143 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
2146 return "$type,id=${vgaid}${memory}${pciaddr}";
2149 sub drive_is_cloudinit
{
2151 return $drive->{file
} =~ m
@[:/]vm-\d
+-cloudinit
(?
:\
.$QEMU_FORMAT_RE)?
$@;
2154 sub drive_is_cdrom
{
2155 my ($drive, $exclude_cloudinit) = @_;
2157 return 0 if $exclude_cloudinit && drive_is_cloudinit
($drive);
2159 return $drive && $drive->{media
} && ($drive->{media
} eq 'cdrom');
2163 sub parse_number_sets
{
2166 foreach my $part (split(/;/, $set)) {
2167 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2168 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2169 push @$res, [ $1, $2 ];
2171 die "invalid range: $part\n";
2180 my $res = PVE
::JSONSchema
::parse_property_string
($numa_fmt, $data);
2181 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
2182 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
2189 return undef if !$value;
2191 my $res = PVE
::JSONSchema
::parse_property_string
($hostpci_fmt, $value);
2193 my @idlist = split(/;/, $res->{host
});
2194 delete $res->{host
};
2195 foreach my $id (@idlist) {
2196 if ($id =~ m/\./) { # full id 00:00.1
2197 push @{$res->{pciid
}}, {
2200 } else { # partial id 00:00
2201 $res->{pciid
} = PVE
::SysFSTools
::lspci
($id);
2207 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2211 my $res = eval { PVE
::JSONSchema
::parse_property_string
($net_fmt, $data) };
2216 if (!defined($res->{macaddr
})) {
2217 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
2218 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
2223 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2224 sub parse_ipconfig
{
2227 my $res = eval { PVE
::JSONSchema
::parse_property_string
($ipconfig_fmt, $data) };
2233 if ($res->{gw
} && !$res->{ip
}) {
2234 warn 'gateway specified without specifying an IP address';
2237 if ($res->{gw6
} && !$res->{ip6
}) {
2238 warn 'IPv6 gateway specified without specifying an IPv6 address';
2241 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
2242 warn 'gateway specified together with DHCP';
2245 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
2247 warn "IPv6 gateway specified together with $res->{ip6} address";
2251 if (!$res->{ip
} && !$res->{ip6
}) {
2252 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
2261 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
2264 sub add_random_macs
{
2265 my ($settings) = @_;
2267 foreach my $opt (keys %$settings) {
2268 next if $opt !~ m/^net(\d+)$/;
2269 my $net = parse_net
($settings->{$opt});
2271 $settings->{$opt} = print_net
($net);
2275 sub vm_is_volid_owner
{
2276 my ($storecfg, $vmid, $volid) = @_;
2278 if ($volid !~ m
|^/|) {
2280 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
2281 if ($owner && ($owner == $vmid)) {
2289 sub split_flagged_list
{
2290 my $text = shift || '';
2291 $text =~ s/[,;]/ /g;
2293 return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2296 sub join_flagged_list
{
2297 my ($how, $lst) = @_;
2298 join $how, map { $lst->{$_} . $_ } keys %$lst;
2301 sub vmconfig_delete_pending_option
{
2302 my ($conf, $key, $force) = @_;
2304 delete $conf->{pending
}->{$key};
2305 my $pending_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
2306 $pending_delete_hash->{$key} = $force ?
'!' : '';
2307 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
2310 sub vmconfig_undelete_pending_option
{
2311 my ($conf, $key) = @_;
2313 my $pending_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
2314 delete $pending_delete_hash->{$key};
2316 if (%$pending_delete_hash) {
2317 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
2319 delete $conf->{pending
}->{delete};
2323 sub vmconfig_register_unused_drive
{
2324 my ($storecfg, $vmid, $conf, $drive) = @_;
2326 if (drive_is_cloudinit
($drive)) {
2327 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2329 } elsif (!drive_is_cdrom
($drive)) {
2330 my $volid = $drive->{file
};
2331 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2332 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2337 sub vmconfig_cleanup_pending
{
2340 # remove pending changes when nothing changed
2342 foreach my $opt (keys %{$conf->{pending
}}) {
2343 if (defined($conf->{$opt}) && ($conf->{pending
}->{$opt} eq $conf->{$opt})) {
2345 delete $conf->{pending
}->{$opt};
2349 my $current_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
2350 my $pending_delete_hash = {};
2351 while (my ($opt, $force) = each %$current_delete_hash) {
2352 if (defined($conf->{$opt})) {
2353 $pending_delete_hash->{$opt} = $force;
2359 if (%$pending_delete_hash) {
2360 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
2362 delete $conf->{pending
}->{delete};
2368 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2372 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2373 format_description
=> 'UUID',
2374 description
=> "Set SMBIOS1 UUID.",
2379 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2380 format_description
=> 'Base64 encoded string',
2381 description
=> "Set SMBIOS1 version.",
2386 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2387 format_description
=> 'Base64 encoded string',
2388 description
=> "Set SMBIOS1 serial number.",
2393 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2394 format_description
=> 'Base64 encoded string',
2395 description
=> "Set SMBIOS1 manufacturer.",
2400 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2401 format_description
=> 'Base64 encoded string',
2402 description
=> "Set SMBIOS1 product ID.",
2407 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2408 format_description
=> 'Base64 encoded string',
2409 description
=> "Set SMBIOS1 SKU string.",
2414 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2415 format_description
=> 'Base64 encoded string',
2416 description
=> "Set SMBIOS1 family string.",
2421 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2429 my $res = eval { PVE
::JSONSchema
::parse_property_string
($smbios1_fmt, $data) };
2436 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2439 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2441 PVE
::JSONSchema
::register_format
('pve-qm-bootdisk', \
&verify_bootdisk
);
2442 sub verify_bootdisk
{
2443 my ($value, $noerr) = @_;
2445 return $value if is_valid_drivename
($value);
2447 return undef if $noerr;
2449 die "invalid boot disk '$value'\n";
2452 sub parse_watchdog
{
2455 return undef if !$value;
2457 my $res = eval { PVE
::JSONSchema
::parse_property_string
($watchdog_fmt, $value) };
2462 sub parse_guest_agent
{
2465 return {} if !defined($value->{agent
});
2467 my $res = eval { PVE
::JSONSchema
::parse_property_string
($agent_fmt, $value->{agent
}) };
2470 # if the agent is disabled ignore the other potentially set properties
2471 return {} if !$res->{enabled
};
2478 return {} if !$value;
2479 my $res = eval { PVE
::JSONSchema
::parse_property_string
($vga_fmt, $value) };
2484 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2485 sub verify_usb_device
{
2486 my ($value, $noerr) = @_;
2488 return $value if parse_usb_device
($value);
2490 return undef if $noerr;
2492 die "unable to parse usb device\n";
2495 # add JSON properties for create and set function
2496 sub json_config_properties
{
2499 foreach my $opt (keys %$confdesc) {
2500 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2501 $prop->{$opt} = $confdesc->{$opt};
2507 # return copy of $confdesc_cloudinit to generate documentation
2508 sub cloudinit_config_properties
{
2510 return dclone
($confdesc_cloudinit);
2514 my ($key, $value) = @_;
2516 die "unknown setting '$key'\n" if !$confdesc->{$key};
2518 my $type = $confdesc->{$key}->{type
};
2520 if (!defined($value)) {
2521 die "got undefined value\n";
2524 if ($value =~ m/[\n\r]/) {
2525 die "property contains a line feed\n";
2528 if ($type eq 'boolean') {
2529 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2530 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2531 die "type check ('boolean') failed - got '$value'\n";
2532 } elsif ($type eq 'integer') {
2533 return int($1) if $value =~ m/^(\d+)$/;
2534 die "type check ('integer') failed - got '$value'\n";
2535 } elsif ($type eq 'number') {
2536 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2537 die "type check ('number') failed - got '$value'\n";
2538 } elsif ($type eq 'string') {
2539 if (my $fmt = $confdesc->{$key}->{format
}) {
2540 PVE
::JSONSchema
::check_format
($fmt, $value);
2543 $value =~ s/^\"(.*)\"$/$1/;
2546 die "internal error"
2553 my $conf = PVE
::QemuConfig-
>config_file($vmid);
2554 utime undef, undef, $conf;
2558 my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2560 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
2562 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2564 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2566 if ($conf->{template
}) {
2567 # check if any base image is still used by a linked clone
2568 foreach_drive
($conf, sub {
2569 my ($ds, $drive) = @_;
2571 return if drive_is_cdrom
($drive);
2573 my $volid = $drive->{file
};
2575 return if !$volid || $volid =~ m
|^/|;
2577 die "base volume '$volid' is still in use by linked cloned\n"
2578 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2583 # only remove disks owned by this VM
2584 foreach_drive
($conf, sub {
2585 my ($ds, $drive) = @_;
2587 return if drive_is_cdrom
($drive, 1);
2589 my $volid = $drive->{file
};
2591 return if !$volid || $volid =~ m
|^/|;
2593 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2594 return if !$path || !$owner || ($owner != $vmid);
2597 PVE
::Storage
::vdisk_free
($storecfg, $volid);
2599 warn "Could not remove disk '$volid', check manually: $@" if $@;
2603 if ($keep_empty_config) {
2604 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
2609 # also remove unused disk
2611 my $dl = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2614 PVE
::Storage
::foreach_volid
($dl, sub {
2615 my ($volid, $sid, $volname, $d) = @_;
2616 PVE
::Storage
::vdisk_free
($storecfg, $volid);
2625 sub parse_vm_config
{
2626 my ($filename, $raw) = @_;
2628 return undef if !defined($raw);
2631 digest
=> Digest
::SHA
::sha1_hex
($raw),
2636 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2637 || die "got strange filename '$filename'";
2645 my @lines = split(/\n/, $raw);
2646 foreach my $line (@lines) {
2647 next if $line =~ m/^\s*$/;
2649 if ($line =~ m/^\[PENDING\]\s*$/i) {
2650 $section = 'pending';
2651 if (defined($descr)) {
2653 $conf->{description
} = $descr;
2656 $conf = $res->{$section} = {};
2659 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2661 if (defined($descr)) {
2663 $conf->{description
} = $descr;
2666 $conf = $res->{snapshots
}->{$section} = {};
2670 if ($line =~ m/^\#(.*)\s*$/) {
2671 $descr = '' if !defined($descr);
2672 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2676 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2677 $descr = '' if !defined($descr);
2678 $descr .= PVE
::Tools
::decode_text
($2);
2679 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2680 $conf->{snapstate
} = $1;
2681 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2684 $conf->{$key} = $value;
2685 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2687 if ($section eq 'pending') {
2688 $conf->{delete} = $value; # we parse this later
2690 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2692 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2695 eval { $value = check_type
($key, $value); };
2697 warn "vm $vmid - unable to parse value of '$key' - $@";
2699 $key = 'ide2' if $key eq 'cdrom';
2700 my $fmt = $confdesc->{$key}->{format
};
2701 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2702 my $v = parse_drive
($key, $value);
2703 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2704 $v->{file
} = $volid;
2705 $value = print_drive
($vmid, $v);
2707 warn "vm $vmid - unable to parse value of '$key'\n";
2712 $conf->{$key} = $value;
2717 if (defined($descr)) {
2719 $conf->{description
} = $descr;
2721 delete $res->{snapstate
}; # just to be sure
2726 sub write_vm_config
{
2727 my ($filename, $conf) = @_;
2729 delete $conf->{snapstate
}; # just to be sure
2731 if ($conf->{cdrom
}) {
2732 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2733 $conf->{ide2
} = $conf->{cdrom
};
2734 delete $conf->{cdrom
};
2737 # we do not use 'smp' any longer
2738 if ($conf->{sockets
}) {
2739 delete $conf->{smp
};
2740 } elsif ($conf->{smp
}) {
2741 $conf->{sockets
} = $conf->{smp
};
2742 delete $conf->{cores
};
2743 delete $conf->{smp
};
2746 my $used_volids = {};
2748 my $cleanup_config = sub {
2749 my ($cref, $pending, $snapname) = @_;
2751 foreach my $key (keys %$cref) {
2752 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2753 $key eq 'snapstate' || $key eq 'pending';
2754 my $value = $cref->{$key};
2755 if ($key eq 'delete') {
2756 die "propertry 'delete' is only allowed in [PENDING]\n"
2758 # fixme: check syntax?
2761 eval { $value = check_type
($key, $value); };
2762 die "unable to parse value of '$key' - $@" if $@;
2764 $cref->{$key} = $value;
2766 if (!$snapname && is_valid_drivename
($key)) {
2767 my $drive = parse_drive
($key, $value);
2768 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2773 &$cleanup_config($conf);
2775 &$cleanup_config($conf->{pending
}, 1);
2777 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2778 die "internal error" if $snapname eq 'pending';
2779 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2782 # remove 'unusedX' settings if we re-add a volume
2783 foreach my $key (keys %$conf) {
2784 my $value = $conf->{$key};
2785 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2786 delete $conf->{$key};
2790 my $generate_raw_config = sub {
2791 my ($conf, $pending) = @_;
2795 # add description as comment to top of file
2796 if (defined(my $descr = $conf->{description
})) {
2798 foreach my $cl (split(/\n/, $descr)) {
2799 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2802 $raw .= "#\n" if $pending;
2806 foreach my $key (sort keys %$conf) {
2807 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2808 $raw .= "$key: $conf->{$key}\n";
2813 my $raw = &$generate_raw_config($conf);
2815 if (scalar(keys %{$conf->{pending
}})){
2816 $raw .= "\n[PENDING]\n";
2817 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2820 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2821 $raw .= "\n[$snapname]\n";
2822 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2832 # we use static defaults from our JSON schema configuration
2833 foreach my $key (keys %$confdesc) {
2834 if (defined(my $default = $confdesc->{$key}->{default})) {
2835 $res->{$key} = $default;
2843 my $vmlist = PVE
::Cluster
::get_vmlist
();
2845 return $res if !$vmlist || !$vmlist->{ids
};
2846 my $ids = $vmlist->{ids
};
2848 foreach my $vmid (keys %$ids) {
2849 my $d = $ids->{$vmid};
2850 next if !$d->{node
} || $d->{node
} ne $nodename;
2851 next if !$d->{type
} || $d->{type
} ne 'qemu';
2852 $res->{$vmid}->{exists} = 1;
2857 # test if VM uses local resources (to prevent migration)
2858 sub check_local_resources
{
2859 my ($conf, $noerr) = @_;
2863 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2864 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2866 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2868 foreach my $k (keys %$conf) {
2869 next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2870 # sockets are safe: they will recreated be on the target side post-migrate
2871 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2872 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2875 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2880 # check if used storages are available on all nodes (use by migrate)
2881 sub check_storage_availability
{
2882 my ($storecfg, $conf, $node) = @_;
2884 foreach_drive
($conf, sub {
2885 my ($ds, $drive) = @_;
2887 my $volid = $drive->{file
};
2890 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2893 # check if storage is available on both nodes
2894 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2895 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2899 # list nodes where all VM images are available (used by has_feature API)
2901 my ($conf, $storecfg) = @_;
2903 my $nodelist = PVE
::Cluster
::get_nodelist
();
2904 my $nodehash = { map { $_ => 1 } @$nodelist };
2905 my $nodename = PVE
::INotify
::nodename
();
2907 foreach_drive
($conf, sub {
2908 my ($ds, $drive) = @_;
2910 my $volid = $drive->{file
};
2913 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2915 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2916 if ($scfg->{disable
}) {
2918 } elsif (my $avail = $scfg->{nodes
}) {
2919 foreach my $node (keys %$nodehash) {
2920 delete $nodehash->{$node} if !$avail->{$node};
2922 } elsif (!$scfg->{shared
}) {
2923 foreach my $node (keys %$nodehash) {
2924 delete $nodehash->{$node} if $node ne $nodename
2933 sub check_local_storage_availability
{
2934 my ($conf, $storecfg) = @_;
2936 my $nodelist = PVE
::Cluster
::get_nodelist
();
2937 my $nodehash = { map { $_ => {} } @$nodelist };
2939 foreach_drive
($conf, sub {
2940 my ($ds, $drive) = @_;
2942 my $volid = $drive->{file
};
2945 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2947 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2949 if ($scfg->{disable
}) {
2950 foreach my $node (keys %$nodehash) {
2951 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2953 } elsif (my $avail = $scfg->{nodes
}) {
2954 foreach my $node (keys %$nodehash) {
2955 if (!$avail->{$node}) {
2956 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2963 foreach my $node (values %$nodehash) {
2964 if (my $unavail = $node->{unavailable_storages
}) {
2965 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2973 my ($pidfile, $pid) = @_;
2975 my $fh = IO
::File-
>new("/proc/$pid/cmdline", "r");
2979 return undef if !$line;
2980 my @param = split(/\0/, $line);
2982 my $cmd = $param[0];
2983 return if !$cmd || ($cmd !~ m
|kvm
$| && $cmd !~ m
@(?
:^|/)qemu-system-[^/]+$@);
2985 for (my $i = 0; $i < scalar (@param); $i++) {
2988 if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2989 my $p = $param[$i+1];
2990 return 1 if $p && ($p eq $pidfile);
2999 my ($vmid, $nocheck, $node) = @_;
3001 my $filename = PVE
::QemuConfig-
>config_file($vmid, $node);
3003 die "unable to find configuration file for VM $vmid - no such machine\n"
3004 if !$nocheck && ! -f
$filename;
3006 my $pidfile = pidfile_name
($vmid);
3008 if (my $fd = IO
::File-
>new("<$pidfile")) {
3013 my $mtime = $st->mtime;
3014 if ($mtime > time()) {
3015 warn "file '$filename' modified in future\n";
3018 if ($line =~ m/^(\d+)$/) {
3020 if (check_cmdline
($pidfile, $pid)) {
3021 if (my $pinfo = PVE
::ProcFSTools
::check_process_running
($pid)) {
3033 my $vzlist = config_list
();
3035 my $fd = IO
::Dir-
>new($var_run_tmpdir) || return $vzlist;
3037 while (defined(my $de = $fd->read)) {
3038 next if $de !~ m/^(\d+)\.pid$/;
3040 next if !defined($vzlist->{$vmid});
3041 if (my $pid = check_running
($vmid)) {
3042 $vzlist->{$vmid}->{pid
} = $pid;
3050 my ($storecfg, $conf) = @_;
3052 my $bootdisk = $conf->{bootdisk
};
3053 return undef if !$bootdisk;
3054 return undef if !is_valid_drivename
($bootdisk);
3056 return undef if !$conf->{$bootdisk};
3058 my $drive = parse_drive
($bootdisk, $conf->{$bootdisk});
3059 return undef if !defined($drive);
3061 return undef if drive_is_cdrom
($drive);
3063 my $volid = $drive->{file
};
3064 return undef if !$volid;
3066 return $drive->{size
};
3069 our $vmstatus_return_properties = {
3070 vmid
=> get_standard_option
('pve-vmid'),
3072 description
=> "Qemu process status.",
3074 enum
=> ['stopped', 'running'],
3077 description
=> "Maximum memory in bytes.",
3080 renderer
=> 'bytes',
3083 description
=> "Root disk size in bytes.",
3086 renderer
=> 'bytes',
3089 description
=> "VM name.",
3094 description
=> "Qemu QMP agent status.",
3099 description
=> "PID of running qemu process.",
3104 description
=> "Uptime.",
3107 renderer
=> 'duration',
3110 description
=> "Maximum usable CPUs.",
3115 description
=> "The current config lock, if any.",
3121 my $last_proc_pid_stat;
3123 # get VM status information
3124 # This must be fast and should not block ($full == false)
3125 # We only query KVM using QMP if $full == true (this can be slow)
3127 my ($opt_vmid, $full) = @_;
3131 my $storecfg = PVE
::Storage
::config
();
3133 my $list = vzlist
();
3134 my $defaults = load_defaults
();
3136 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
3138 my $cpucount = $cpuinfo->{cpus
} || 1;
3140 foreach my $vmid (keys %$list) {
3141 next if $opt_vmid && ($vmid ne $opt_vmid);
3143 my $cfspath = PVE
::QemuConfig-
>cfs_config_path($vmid);
3144 my $conf = PVE
::Cluster
::cfs_read_file
($cfspath) || {};
3146 my $d = { vmid
=> $vmid };
3147 $d->{pid
} = $list->{$vmid}->{pid
};
3149 # fixme: better status?
3150 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
3152 my $size = disksize
($storecfg, $conf);
3153 if (defined($size)) {
3154 $d->{disk
} = 0; # no info available
3155 $d->{maxdisk
} = $size;
3161 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
3162 * ($conf->{cores
} || $defaults->{cores
});
3163 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
3164 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
3166 $d->{name
} = $conf->{name
} || "VM $vmid";
3167 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
3168 : $defaults->{memory
}*(1024*1024);
3170 if ($conf->{balloon
}) {
3171 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
3172 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
3173 : $defaults->{shares
};
3184 $d->{diskwrite
} = 0;
3186 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
3188 $d->{serial
} = 1 if conf_has_serial
($conf);
3189 $d->{lock} = $conf->{lock} if $conf->{lock};
3194 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
3195 foreach my $dev (keys %$netdev) {
3196 next if $dev !~ m/^tap([1-9]\d*)i/;
3198 my $d = $res->{$vmid};
3201 $d->{netout
} += $netdev->{$dev}->{receive
};
3202 $d->{netin
} += $netdev->{$dev}->{transmit
};
3205 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
3206 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
3211 my $ctime = gettimeofday
;
3213 foreach my $vmid (keys %$list) {
3215 my $d = $res->{$vmid};
3216 my $pid = $d->{pid
};
3219 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
3220 next if !$pstat; # not running
3222 my $used = $pstat->{utime} + $pstat->{stime
};
3224 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
3226 if ($pstat->{vsize
}) {
3227 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
3230 my $old = $last_proc_pid_stat->{$pid};
3232 $last_proc_pid_stat->{$pid} = {
3240 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
3242 if ($dtime > 1000) {
3243 my $dutime = $used - $old->{used
};
3245 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
3246 $last_proc_pid_stat->{$pid} = {
3252 $d->{cpu
} = $old->{cpu
};
3256 return $res if !$full;
3258 my $qmpclient = PVE
::QMPClient-
>new();
3260 my $ballooncb = sub {
3261 my ($vmid, $resp) = @_;
3263 my $info = $resp->{'return'};
3264 return if !$info->{max_mem
};
3266 my $d = $res->{$vmid};
3268 # use memory assigned to VM
3269 $d->{maxmem
} = $info->{max_mem
};
3270 $d->{balloon
} = $info->{actual
};
3272 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
3273 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
3274 $d->{freemem
} = $info->{free_mem
};
3277 $d->{ballooninfo
} = $info;
3280 my $blockstatscb = sub {
3281 my ($vmid, $resp) = @_;
3282 my $data = $resp->{'return'} || [];
3283 my $totalrdbytes = 0;
3284 my $totalwrbytes = 0;
3286 for my $blockstat (@$data) {
3287 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
3288 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
3290 $blockstat->{device
} =~ s/drive-//;
3291 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
3293 $res->{$vmid}->{diskread
} = $totalrdbytes;
3294 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
3297 my $statuscb = sub {
3298 my ($vmid, $resp) = @_;
3300 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3301 # this fails if ballon driver is not loaded, so this must be
3302 # the last commnand (following command are aborted if this fails).
3303 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3305 my $status = 'unknown';
3306 if (!defined($status = $resp->{'return'}->{status
})) {
3307 warn "unable to get VM status\n";
3311 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3314 foreach my $vmid (keys %$list) {
3315 next if $opt_vmid && ($vmid ne $opt_vmid);
3316 next if !$res->{$vmid}->{pid
}; # not running
3317 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3320 $qmpclient->queue_execute(undef, 2);
3322 foreach my $vmid (keys %$list) {
3323 next if $opt_vmid && ($vmid ne $opt_vmid);
3324 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3331 my ($conf, $func, @param) = @_;
3333 foreach my $ds (valid_drive_names
()) {
3334 next if !defined($conf->{$ds});
3336 my $drive = parse_drive
($ds, $conf->{$ds});
3339 &$func($ds, $drive, @param);
3344 my ($conf, $func, @param) = @_;
3348 my $test_volid = sub {
3349 my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3353 $volhash->{$volid}->{cdrom
} //= 1;
3354 $volhash->{$volid}->{cdrom
} = 0 if !$is_cdrom;
3356 $volhash->{$volid}->{replicate
} //= 0;
3357 $volhash->{$volid}->{replicate
} = 1 if $replicate;
3359 $volhash->{$volid}->{shared
} //= 0;
3360 $volhash->{$volid}->{shared
} = 1 if $shared;
3362 $volhash->{$volid}->{referenced_in_config
} //= 0;
3363 $volhash->{$volid}->{referenced_in_config
} = 1 if !defined($snapname);
3365 $volhash->{$volid}->{referenced_in_snapshot
}->{$snapname} = 1
3366 if defined($snapname);
3367 $volhash->{$volid}->{size
} = $size if $size;
3370 foreach_drive
($conf, sub {
3371 my ($ds, $drive) = @_;
3372 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, undef, $drive->{size
});
3375 foreach my $snapname (keys %{$conf->{snapshots
}}) {
3376 my $snap = $conf->{snapshots
}->{$snapname};
3377 $test_volid->($snap->{vmstate
}, 0, 1, $snapname);
3378 foreach_drive
($snap, sub {
3379 my ($ds, $drive) = @_;
3380 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, $snapname);
3384 foreach my $volid (keys %$volhash) {
3385 &$func($volid, $volhash->{$volid}, @param);
3389 sub conf_has_serial
{
3392 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3393 if ($conf->{"serial$i"}) {
3401 sub vga_conf_has_spice
{
3404 my $vgaconf = parse_vga
($vga);
3405 my $vgatype = $vgaconf->{type
};
3406 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3411 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3412 sub get_host_arch
() {
3413 $host_arch = (POSIX
::uname
())[4] if !$host_arch;
3419 return get_host_arch
() eq $arch;
3422 my $default_machines = {
3427 sub get_basic_machine_info
{
3428 my ($conf, $forcemachine) = @_;
3430 my $arch = $conf->{arch
} // get_host_arch
();
3431 my $machine = $forcemachine || $conf->{machine
} || $default_machines->{$arch};
3432 return ($arch, $machine);
3435 sub get_ovmf_files
($) {
3438 my $ovmf = $OVMF->{$arch}
3439 or die "no OVMF images known for architecture '$arch'\n";
3445 aarch64
=> '/usr/bin/qemu-system-aarch64',
3446 x86_64
=> '/usr/bin/qemu-system-x86_64',
3448 sub get_command_for_arch
($) {
3450 return '/usr/bin/kvm' if is_native
($arch);
3452 my $cmd = $Arch2Qemu->{$arch}
3453 or die "don't know how to emulate architecture '$arch'\n";
3457 sub get_cpu_options
{
3458 my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3461 my $ostype = $conf->{ostype
};
3463 my $cpu = $kvm ?
"kvm64" : "qemu64";
3464 if ($arch eq 'aarch64') {
3465 $cpu = 'cortex-a57';
3468 if (my $cputype = $conf->{cpu
}) {
3469 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
3470 or die "Cannot parse cpu description: $cputype\n";
3471 $cpu = $cpuconf->{cputype
};
3472 $kvm_off = 1 if $cpuconf->{hidden
};
3473 $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3475 if (defined(my $flags = $cpuconf->{flags
})) {
3476 push @$cpuFlags, split(";", $flags);
3480 push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3482 push @$cpuFlags , '-x2apic'
3483 if $conf->{ostype
} && $conf->{ostype
} eq 'solaris';
3485 push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3487 push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3489 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3491 push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3492 push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3495 add_hyperv_enlightenments
($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios
}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3497 push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3499 push @$cpuFlags, 'kvm=off' if $kvm_off;
3501 if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3502 push @$cpuFlags, "vendor=${cpu_vendor}"
3503 if $cpu_vendor ne 'default';
3504 } elsif ($arch ne 'aarch64') {
3505 die "internal error"; # should not happen
3508 $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3510 return ('-cpu', $cpu);
3513 sub config_to_command
{
3514 my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3517 my $globalFlags = [];
3518 my $machineFlags = [];
3523 my $kvmver = kvm_user_version
();
3524 my $vernum = 0; # unknown
3525 my $ostype = $conf->{ostype
};
3526 my $winversion = windows_version
($ostype);
3527 my $kvm = $conf->{kvm
};
3529 my ($arch, $machine_type) = get_basic_machine_info
($conf, $forcemachine);
3530 $kvm //= 1 if is_native
($arch);
3533 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3534 if !defined kvm_version
();
3537 if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3538 $vernum = $1*1000000+$2*1000;
3539 } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3540 $vernum = $1*1000000+$2*1000+$3;
3543 die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3545 my $have_ovz = -f
'/proc/vz/vestat';
3547 my $q35 = machine_type_is_q35
($conf);
3548 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3549 my $use_old_bios_files = undef;
3550 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3552 my $cpuunits = defined($conf->{cpuunits
}) ?
3553 $conf->{cpuunits
} : $defaults->{cpuunits
};
3555 push @$cmd, get_command_for_arch
($arch);
3557 push @$cmd, '-id', $vmid;
3559 my $vmname = $conf->{name
} || "vm$vmid";
3561 push @$cmd, '-name', $vmname;
3565 my $qmpsocket = qmp_socket
($vmid);
3566 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3567 push @$cmd, '-mon', "chardev=qmp,mode=control";
3569 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 12)) {
3570 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3571 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3574 push @$cmd, '-pidfile' , pidfile_name
($vmid);
3576 push @$cmd, '-daemonize';
3578 if ($conf->{smbios1
}) {
3579 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3580 if ($smbios_conf->{base64
}) {
3581 # Do not pass base64 flag to qemu
3582 delete $smbios_conf->{base64
};
3583 my $smbios_string = "";
3584 foreach my $key (keys %$smbios_conf) {
3586 if ($key eq "uuid") {
3587 $value = $smbios_conf->{uuid
}
3589 $value = decode_base64
($smbios_conf->{$key});
3591 # qemu accepts any binary data, only commas need escaping by double comma
3593 $smbios_string .= "," . $key . "=" . $value if $value;
3595 push @$cmd, '-smbios', "type=1" . $smbios_string;
3597 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3601 if ($conf->{vmgenid
}) {
3602 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3605 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3606 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3607 die "uefi base image not found\n" if ! -f
$ovmf_code;
3611 if (my $efidisk = $conf->{efidisk0
}) {
3612 my $d = PVE
::JSONSchema
::parse_property_string
($efidisk_fmt, $efidisk);
3613 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3614 $format = $d->{format
};
3616 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3617 if (!defined($format)) {
3618 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3619 $format = qemu_img_format
($scfg, $volname);
3623 die "efidisk format must be specified\n"
3624 if !defined($format);
3627 warn "no efidisk configured! Using temporary efivars disk.\n";
3628 $path = "/tmp/$vmid-ovmf.fd";
3629 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3633 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3634 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3639 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3640 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 4, 0)) {
3641 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3643 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3647 # add usb controllers
3648 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
($conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3649 push @$devices, @usbcontrollers if @usbcontrollers;
3650 my $vga = parse_vga
($conf->{vga
});
3652 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3653 $vga->{type
} = 'qxl' if $qxlnum;
3655 if (!$vga->{type
}) {
3656 if ($arch eq 'aarch64') {
3657 $vga->{type
} = 'virtio';
3658 } elsif (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 9)) {
3659 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3661 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3665 # enable absolute mouse coordinates (needed by vnc)
3667 if (defined($conf->{tablet
})) {
3668 $tablet = $conf->{tablet
};
3670 $tablet = $defaults->{tablet
};
3671 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3672 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3676 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3677 my $kbd = print_keyboarddevice_full
($conf, $arch);
3678 push @$devices, '-device', $kbd if defined($kbd);
3682 my $gpu_passthrough;
3685 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
3686 my $d = parse_hostpci
($conf->{"hostpci$i"});
3689 my $pcie = $d->{pcie
};
3691 die "q35 machine model is not enabled" if !$q35;
3692 # win7 wants to have the pcie devices directly on the pcie bus
3693 # instead of in the root port
3694 if ($winversion == 7) {
3695 $pciaddr = print_pcie_addr
("hostpci${i}bus0");
3697 $pciaddr = print_pcie_addr
("hostpci$i");
3700 $pciaddr = print_pci_addr
("hostpci$i", $bridges, $arch, $machine_type);
3703 my $rombar = defined($d->{rombar
}) && !$d->{rombar
} ?
',rombar=0' : '';
3704 my $romfile = $d->{romfile
};
3707 if ($d->{'x-vga'}) {
3708 $xvga = ',x-vga=on';
3710 $vga->{type
} = 'none' if !defined($conf->{vga
});
3711 $gpu_passthrough = 1;
3713 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3717 my $pcidevices = $d->{pciid
};
3718 my $multifunction = 1 if @$pcidevices > 1;
3720 if ($d->{mdev
} && scalar(@$pcidevices) == 1) {
3721 my $id = $pcidevices->[0]->{id
};
3722 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $i);
3723 $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3724 } elsif ($d->{mdev
}) {
3725 warn "ignoring mediated device with multifunction device\n";
3729 foreach my $pcidevice (@$pcidevices) {
3731 my $id = "hostpci$i";
3732 $id .= ".$j" if $multifunction;
3733 my $addr = $pciaddr;
3734 $addr .= ".$j" if $multifunction;
3735 my $devicestr = "vfio-pci";
3737 $devicestr .= ",sysfsdev=$sysfspath";
3739 $devicestr .= ",host=$pcidevice->{id}";
3741 $devicestr .= ",id=$id$addr";
3744 $devicestr .= "$rombar$xvga";
3745 $devicestr .= ",multifunction=on" if $multifunction;
3746 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3749 push @$devices, '-device', $devicestr;
3755 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
($conf, $usbdesc->{format
}, $MAX_USB_DEVICES);
3756 push @$devices, @usbdevices if @usbdevices;
3758 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3759 if (my $path = $conf->{"serial$i"}) {
3760 if ($path eq 'socket') {
3761 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3762 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3763 # On aarch64, serial0 is the UART device. Qemu only allows
3764 # connecting UART devices via the '-serial' command line, as
3765 # the device has a fixed slot on the hardware...
3766 if ($arch eq 'aarch64' && $i == 0) {
3767 push @$devices, '-serial', "chardev:serial$i";
3769 push @$devices, '-device', "isa-serial,chardev=serial$i";
3772 die "no such serial device\n" if ! -c
$path;
3773 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3774 push @$devices, '-device', "isa-serial,chardev=serial$i";
3780 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3781 if (my $path = $conf->{"parallel$i"}) {
3782 die "no such parallel device\n" if ! -c
$path;
3783 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3784 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3785 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3789 if (my $audiodevice = $conf->{audio0
}) {
3790 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3792 if ($audiodevice eq 'AC97') {
3793 push @$devices, '-device', "AC97,id=sound0${audiopciaddr}";
3794 } elsif ($audiodevice =~ /intel\-hda$/) {
3795 push @$devices, '-device', "${audiodevice},id=sound5${audiopciaddr}";
3796 push @$devices, '-device', "hda-micro,id=sound5-codec0,bus=sound5.0,cad=0";
3797 push @$devices, '-device', "hda-duplex,id=sound5-codec1,bus=sound5.0,cad=1";
3799 die "unkown audio device '$audiodevice', implement me!";
3804 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3805 $sockets = $conf->{sockets
} if $conf->{sockets
};
3807 my $cores = $conf->{cores
} || 1;
3809 my $maxcpus = $sockets * $cores;
3811 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3813 my $allowed_vcpus = $cpuinfo->{cpus
};
3815 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3816 if ($allowed_vcpus < $maxcpus);
3818 if($hotplug_features->{cpu
} && qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 7)) {
3820 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3821 for (my $i = 2; $i <= $vcpus; $i++) {
3822 my $cpustr = print_cpu_device
($conf,$i);
3823 push @$cmd, '-device', $cpustr;
3828 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3830 push @$cmd, '-nodefaults';
3832 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
3834 my $bootindex_hash = {};
3836 foreach my $o (split(//, $bootorder)) {
3837 $bootindex_hash->{$o} = $i*100;
3841 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3843 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3845 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3847 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3848 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3849 my $socket = vnc_socket
($vmid);
3850 push @$cmd, '-vnc', "unix:$socket,password";
3852 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3853 push @$cmd, '-nographic';
3857 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3859 my $useLocaltime = $conf->{localtime};
3861 if ($winversion >= 5) { # windows
3862 $useLocaltime = 1 if !defined($conf->{localtime});
3864 # use time drift fix when acpi is enabled
3865 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3866 $tdf = 1 if !defined($conf->{tdf
});
3870 if ($winversion >= 6) {
3871 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3872 push @$cmd, '-no-hpet';
3875 push @$rtcFlags, 'driftfix=slew' if $tdf;
3878 push @$machineFlags, 'accel=tcg';
3881 if ($machine_type) {
3882 push @$machineFlags, "type=${machine_type}";
3885 if (($conf->{startdate
}) && ($conf->{startdate
} ne 'now')) {
3886 push @$rtcFlags, "base=$conf->{startdate}";
3887 } elsif ($useLocaltime) {
3888 push @$rtcFlags, 'base=localtime';
3891 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3893 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3895 push @$cmd, '-S' if $conf->{freeze
};
3897 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3899 if (parse_guest_agent
($conf)->{enabled
}) {
3900 my $qgasocket = qmp_socket
($vmid, 1);
3901 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3902 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3903 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3904 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3912 for(my $i = 1; $i < $qxlnum; $i++){
3913 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3916 # assume other OS works like Linux
3917 my ($ram, $vram) = ("134217728", "67108864");
3918 if ($vga->{memory
}) {
3919 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3920 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3922 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3923 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3927 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3929 my $nodename = PVE
::INotify
::nodename
();
3930 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3931 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3932 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3933 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3934 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3936 push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3938 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3939 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3940 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3943 # enable balloon by default, unless explicitly disabled
3944 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3945 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3946 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3949 if ($conf->{watchdog
}) {
3950 my $wdopts = parse_watchdog
($conf->{watchdog
});
3951 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3952 my $watchdog = $wdopts->{model
} || 'i6300esb';
3953 push @$devices, '-device', "$watchdog$pciaddr";
3954 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3958 my $scsicontroller = {};
3959 my $ahcicontroller = {};
3960 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3962 # Add iscsi initiator name if available
3963 if (my $initiator = get_initiator_name
()) {
3964 push @$devices, '-iscsi', "initiator-name=$initiator";
3967 foreach_drive
($conf, sub {
3968 my ($ds, $drive) = @_;
3970 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3971 push @$vollist, $drive->{file
};
3974 # ignore efidisk here, already added in bios/fw handling code above
3975 return if $drive->{interface
} eq 'efidisk';
3977 $use_virtio = 1 if $ds =~ m/^virtio/;
3979 if (drive_is_cdrom
($drive)) {
3980 if ($bootindex_hash->{d
}) {
3981 $drive->{bootindex
} = $bootindex_hash->{d
};
3982 $bootindex_hash->{d
} += 1;
3985 if ($bootindex_hash->{c
}) {
3986 $drive->{bootindex
} = $bootindex_hash->{c
} if $conf->{bootdisk
} && ($conf->{bootdisk
} eq $ds);
3987 $bootindex_hash->{c
} += 1;
3991 if($drive->{interface
} eq 'virtio'){
3992 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3995 if ($drive->{interface
} eq 'scsi') {
3997 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3999 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
4000 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
4003 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
4004 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4005 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4006 } elsif ($drive->{iothread
}) {
4007 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4011 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
4012 $queues = ",num_queues=$drive->{queues}";
4015 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4016 $scsicontroller->{$controller}=1;
4019 if ($drive->{interface
} eq 'sata') {
4020 my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4021 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
4022 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4023 $ahcicontroller->{$controller}=1;
4026 my $drive_cmd = print_drive_full
($storecfg, $vmid, $drive);
4027 push @$devices, '-drive',$drive_cmd;
4028 push @$devices, '-device', print_drivedevice_full
($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4031 for (my $i = 0; $i < $MAX_NETS; $i++) {
4032 next if !$conf->{"net$i"};
4033 my $d = parse_net
($conf->{"net$i"});
4036 $use_virtio = 1 if $d->{model
} eq 'virtio';
4038 if ($bootindex_hash->{n
}) {
4039 $d->{bootindex
} = $bootindex_hash->{n
};
4040 $bootindex_hash->{n
} += 1;
4043 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, "net$i");
4044 push @$devices, '-netdev', $netdevfull;
4046 my $netdevicefull = print_netdevice_full
($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4047 push @$devices, '-device', $netdevicefull;
4050 if ($conf->{ivshmem
}) {
4051 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4055 $bus = print_pcie_addr
("ivshmem");
4057 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4060 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4061 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4063 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4064 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4069 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3)) {
4074 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4076 while (my ($k, $v) = each %$bridges) {
4077 $pciaddr = print_pci_addr
("pci.$k", undef, $arch, $machine_type);
4078 unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4082 push @$cmd, @$devices;
4083 push @$cmd, '-rtc', join(',', @$rtcFlags)
4084 if scalar(@$rtcFlags);
4085 push @$cmd, '-machine', join(',', @$machineFlags)
4086 if scalar(@$machineFlags);
4087 push @$cmd, '-global', join(',', @$globalFlags)
4088 if scalar(@$globalFlags);
4090 if (my $vmstate = $conf->{vmstate
}) {
4091 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4092 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
4093 push @$cmd, '-loadstate', $statepath;
4097 if ($conf->{args
}) {
4098 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4102 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
4107 return "${var_run_tmpdir}/$vmid.vnc";
4113 my $res = vm_mon_cmd
($vmid, 'query-spice');
4115 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4119 my ($vmid, $qga, $name) = @_;
4120 my $sockettype = $qga ?
'qga' : 'qmp';
4121 my $ext = $name ?
'-'.$name : '';
4122 return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4127 return "${var_run_tmpdir}/$vmid.pid";
4130 sub vm_devices_list
{
4133 my $res = vm_mon_cmd
($vmid, 'query-pci');
4134 my $devices_to_check = [];
4136 foreach my $pcibus (@$res) {
4137 push @$devices_to_check, @{$pcibus->{devices
}},
4140 while (@$devices_to_check) {
4142 for my $d (@$devices_to_check) {
4143 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4144 next if !$d->{'pci_bridge'};
4146 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4147 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4149 $devices_to_check = $to_check;
4152 my $resblock = vm_mon_cmd
($vmid, 'query-block');
4153 foreach my $block (@$resblock) {
4154 if($block->{device
} =~ m/^drive-(\S+)/){
4159 my $resmice = vm_mon_cmd
($vmid, 'query-mice');
4160 foreach my $mice (@$resmice) {
4161 if ($mice->{name
} eq 'QEMU HID Tablet') {
4162 $devices->{tablet
} = 1;
4167 # for usb devices there is no query-usb
4168 # but we can iterate over the entries in
4169 # qom-list path=/machine/peripheral
4170 my $resperipheral = vm_mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4171 foreach my $per (@$resperipheral) {
4172 if ($per->{name
} =~ m/^usb\d+$/) {
4173 $devices->{$per->{name
}} = 1;
4181 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4183 my $q35 = machine_type_is_q35
($conf);
4185 my $devices_list = vm_devices_list
($vmid);
4186 return 1 if defined($devices_list->{$deviceid});
4188 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4190 if ($deviceid eq 'tablet') {
4192 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4194 } elsif ($deviceid eq 'keyboard') {
4196 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4198 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4200 die "usb hotplug currently not reliable\n";
4201 # since we can't reliably hot unplug all added usb devices
4202 # and usb passthrough disables live migration
4203 # we disable usb hotplugging for now
4204 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device));
4206 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4208 qemu_iothread_add
($vmid, $deviceid, $device);
4210 qemu_driveadd
($storecfg, $vmid, $device);
4211 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4213 qemu_deviceadd
($vmid, $devicefull);
4214 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4216 eval { qemu_drivedel
($vmid, $deviceid); };
4221 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4224 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4225 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4226 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4228 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4230 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4231 qemu_iothread_add
($vmid, $deviceid, $device);
4232 $devicefull .= ",iothread=iothread-$deviceid";
4235 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4236 $devicefull .= ",num_queues=$device->{queues}";
4239 qemu_deviceadd
($vmid, $devicefull);
4240 qemu_deviceaddverify
($vmid, $deviceid);
4242 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4244 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4245 qemu_driveadd
($storecfg, $vmid, $device);
4247 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4248 eval { qemu_deviceadd
($vmid, $devicefull); };
4250 eval { qemu_drivedel
($vmid, $deviceid); };
4255 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4257 return undef if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4259 my $machine_type = PVE
::QemuServer
::qemu_machine_pxe
($vmid, $conf);
4260 my $use_old_bios_files = undef;
4261 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4263 my $netdevicefull = print_netdevice_full
($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4264 qemu_deviceadd
($vmid, $netdevicefull);
4266 qemu_deviceaddverify
($vmid, $deviceid);
4267 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4270 eval { qemu_netdevdel
($vmid, $deviceid); };
4275 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4278 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4279 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4281 qemu_deviceadd
($vmid, $devicefull);
4282 qemu_deviceaddverify
($vmid, $deviceid);
4285 die "can't hotplug device '$deviceid'\n";
4291 # fixme: this should raise exceptions on error!
4292 sub vm_deviceunplug
{
4293 my ($vmid, $conf, $deviceid) = @_;
4295 my $devices_list = vm_devices_list
($vmid);
4296 return 1 if !defined($devices_list->{$deviceid});
4298 die "can't unplug bootdisk" if $conf->{bootdisk
} && $conf->{bootdisk
} eq $deviceid;
4300 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4302 qemu_devicedel
($vmid, $deviceid);
4304 } elsif ($deviceid =~ m/^usb\d+$/) {
4306 die "usb hotplug currently not reliable\n";
4307 # when unplugging usb devices this way,
4308 # there may be remaining usb controllers/hubs
4309 # so we disable it for now
4310 qemu_devicedel
($vmid, $deviceid);
4311 qemu_devicedelverify
($vmid, $deviceid);
4313 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4315 qemu_devicedel
($vmid, $deviceid);
4316 qemu_devicedelverify
($vmid, $deviceid);
4317 qemu_drivedel
($vmid, $deviceid);
4318 qemu_iothread_del
($conf, $vmid, $deviceid);
4320 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4322 qemu_devicedel
($vmid, $deviceid);
4323 qemu_devicedelverify
($vmid, $deviceid);
4324 qemu_iothread_del
($conf, $vmid, $deviceid);
4326 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4328 qemu_devicedel
($vmid, $deviceid);
4329 qemu_drivedel
($vmid, $deviceid);
4330 qemu_deletescsihw
($conf, $vmid, $deviceid);
4332 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4334 qemu_devicedel
($vmid, $deviceid);
4335 qemu_devicedelverify
($vmid, $deviceid);
4336 qemu_netdevdel
($vmid, $deviceid);
4339 die "can't unplug device '$deviceid'\n";
4345 sub qemu_deviceadd
{
4346 my ($vmid, $devicefull) = @_;
4348 $devicefull = "driver=".$devicefull;
4349 my %options = split(/[=,]/, $devicefull);
4351 vm_mon_cmd
($vmid, "device_add" , %options);
4354 sub qemu_devicedel
{
4355 my ($vmid, $deviceid) = @_;
4357 my $ret = vm_mon_cmd
($vmid, "device_del", id
=> $deviceid);
4360 sub qemu_iothread_add
{
4361 my($vmid, $deviceid, $device) = @_;
4363 if ($device->{iothread
}) {
4364 my $iothreads = vm_iothreads_list
($vmid);
4365 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4369 sub qemu_iothread_del
{
4370 my($conf, $vmid, $deviceid) = @_;
4372 my $confid = $deviceid;
4373 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4374 $confid = 'scsi' . $1;
4376 my $device = parse_drive
($confid, $conf->{$confid});
4377 if ($device->{iothread
}) {
4378 my $iothreads = vm_iothreads_list
($vmid);
4379 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4383 sub qemu_objectadd
{
4384 my($vmid, $objectid, $qomtype) = @_;
4386 vm_mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4391 sub qemu_objectdel
{
4392 my($vmid, $objectid) = @_;
4394 vm_mon_cmd
($vmid, "object-del", id
=> $objectid);
4400 my ($storecfg, $vmid, $device) = @_;
4402 my $drive = print_drive_full
($storecfg, $vmid, $device);
4403 $drive =~ s/\\/\\\\/g;
4404 my $ret = vm_human_monitor_command
($vmid, "drive_add auto \"$drive\"");
4406 # If the command succeeds qemu prints: "OK
"
4407 return 1 if $ret =~ m/OK/s;
4409 die "adding drive failed
: $ret\n";
4413 my($vmid, $deviceid) = @_;
4415 my $ret = vm_human_monitor_command($vmid, "drive_del drive-
$deviceid");
4418 return 1 if $ret eq "";
4420 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4421 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4423 die "deleting drive
$deviceid failed
: $ret\n";
4426 sub qemu_deviceaddverify {
4427 my ($vmid, $deviceid) = @_;
4429 for (my $i = 0; $i <= 5; $i++) {
4430 my $devices_list = vm_devices_list($vmid);
4431 return 1 if defined($devices_list->{$deviceid});
4435 die "error on hotplug device
'$deviceid'\n";
4439 sub qemu_devicedelverify {
4440 my ($vmid, $deviceid) = @_;
4442 # need to verify that the device is correctly removed as device_del
4443 # is async and empty return is not reliable
4445 for (my $i = 0; $i <= 5; $i++) {
4446 my $devices_list = vm_devices_list($vmid);
4447 return 1 if !defined($devices_list->{$deviceid});
4451 die "error on hot-unplugging device
'$deviceid'\n";
4454 sub qemu_findorcreatescsihw {
4455 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4457 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4459 my $scsihwid="$controller_prefix$controller";
4460 my $devices_list = vm_devices_list($vmid);
4462 if(!defined($devices_list->{$scsihwid})) {
4463 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4469 sub qemu_deletescsihw {
4470 my ($conf, $vmid, $opt) = @_;
4472 my $device = parse_drive($opt, $conf->{$opt});
4474 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4475 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4479 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4481 my $devices_list = vm_devices_list($vmid);
4482 foreach my $opt (keys %{$devices_list}) {
4483 if (PVE::QemuServer::is_valid_drivename($opt)) {
4484 my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4485 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4491 my $scsihwid="scsihw
$controller";
4493 vm_deviceunplug($vmid, $conf, $scsihwid);
4498 sub qemu_add_pci_bridge {
4499 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4505 print_pci_addr($device, $bridges, $arch, $machine_type);
4507 while (my ($k, $v) = each %$bridges) {
4510 return 1 if !defined($bridgeid) || $bridgeid < 1;
4512 my $bridge = "pci
.$bridgeid";
4513 my $devices_list = vm_devices_list($vmid);
4515 if (!defined($devices_list->{$bridge})) {
4516 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4522 sub qemu_set_link_status {
4523 my ($vmid, $device, $up) = @_;
4525 vm_mon_cmd($vmid, "set_link
", name => $device,
4526 up => $up ? JSON::true : JSON::false);
4529 sub qemu_netdevadd {
4530 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4532 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4533 my %options = split(/[=,]/, $netdev);
4535 vm_mon_cmd($vmid, "netdev_add
", %options);
4539 sub qemu_netdevdel {
4540 my ($vmid, $deviceid) = @_;
4542 vm_mon_cmd($vmid, "netdev_del
", id => $deviceid);
4545 sub qemu_usb_hotplug {
4546 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4550 # remove the old one first
4551 vm_deviceunplug($vmid, $conf, $deviceid);
4553 # check if xhci controller is necessary and available
4554 if ($device->{usb3}) {
4556 my $devicelist = vm_devices_list($vmid);
4558 if (!$devicelist->{xhci}) {
4559 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4560 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4563 my $d = parse_usb_device($device->{host});
4564 $d->{usb3} = $device->{usb3};
4567 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4570 sub qemu_cpu_hotplug {
4571 my ($vmid, $conf, $vcpus) = @_;
4573 my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4576 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4577 $sockets = $conf->{sockets} if $conf->{sockets};
4578 my $cores = $conf->{cores} || 1;
4579 my $maxcpus = $sockets * $cores;
4581 $vcpus = $maxcpus if !$vcpus;
4583 die "you can
't add more vcpus than maxcpus\n"
4584 if $vcpus > $maxcpus;
4586 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4588 if ($vcpus < $currentvcpus) {
4590 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4592 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4593 qemu_devicedel($vmid, "cpu$i");
4595 my $currentrunningvcpus = undef;
4597 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4598 last if scalar(@{$currentrunningvcpus}) == $i-1;
4599 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4603 #update conf after each succesfull cpu unplug
4604 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4605 PVE::QemuConfig->write_config($vmid, $conf);
4608 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4614 my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4615 die "vcpus in running vm does not match its configuration\n"
4616 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4618 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4620 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4621 my $cpustr = print_cpu_device($conf, $i);
4622 qemu_deviceadd($vmid, $cpustr);
4625 my $currentrunningvcpus = undef;
4627 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4628 last if scalar(@{$currentrunningvcpus}) == $i;
4629 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4633 #update conf after each succesfull cpu hotplug
4634 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4635 PVE::QemuConfig->write_config($vmid, $conf);
4639 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4640 vm_mon_cmd($vmid, "cpu-add", id => int($i));
4645 sub qemu_block_set_io_throttle {
4646 my ($vmid, $deviceid,
4647 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4648 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4649 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4650 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4652 return if !check_running($vmid) ;
4654 vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4656 bps_rd => int($bps_rd),
4657 bps_wr => int($bps_wr),
4659 iops_rd => int($iops_rd),
4660 iops_wr => int($iops_wr),
4661 bps_max => int($bps_max),
4662 bps_rd_max => int($bps_rd_max),
4663 bps_wr_max => int($bps_wr_max),
4664 iops_max => int($iops_max),
4665 iops_rd_max => int($iops_rd_max),
4666 iops_wr_max => int($iops_wr_max),
4667 bps_max_length => int($bps_max_length),
4668 bps_rd_max_length => int($bps_rd_max_length),
4669 bps_wr_max_length => int($bps_wr_max_length),
4670 iops_max_length => int($iops_max_length),
4671 iops_rd_max_length => int($iops_rd_max_length),
4672 iops_wr_max_length => int($iops_wr_max_length),
4677 # old code, only used to shutdown old VM after update
4679 my ($fh, $timeout) = @_;
4681 my $sel = new IO::Select;
4688 while (scalar (@ready = $sel->can_read($timeout))) {
4690 if ($count = $fh->sysread($buf, 8192)) {
4691 if ($buf =~ /^(.*)\(qemu\) $/s) {
4698 if (!defined($count)) {
4705 die "monitor read timeout\n" if !scalar(@ready);
4710 sub qemu_block_resize {
4711 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4713 my $running = check_running($vmid);
4715 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4717 return if !$running;
4719 vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4723 sub qemu_volume_snapshot {
4724 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4726 my $running = check_running($vmid);
4728 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4729 vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4731 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4735 sub qemu_volume_snapshot_delete {
4736 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4738 my $running = check_running($vmid);
4743 my $conf = PVE::QemuConfig->load_config($vmid);
4744 foreach_drive($conf, sub {
4745 my ($ds, $drive) = @_;
4746 $running = 1 if $drive->{file} eq $volid;
4750 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4751 vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4753 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4757 sub set_migration_caps {
4763 "auto-converge" => 1,
4765 "x-rdma-pin-all" => 0,
4770 my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4772 for my $supported_capability (@$supported_capabilities) {
4774 capability => $supported_capability->{capability},
4775 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4779 vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4782 my $fast_plug_option = {
4790 'vmstatestorage
' => 1,
4794 # hotplug changes in [PENDING]
4795 # $selection hash can be used to only apply specified options, for
4796 # example: { cores => 1 } (only apply changed 'cores
')
4797 # $errors ref is used to return error messages
4798 sub vmconfig_hotplug_pending {
4799 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4801 my $defaults = load_defaults();
4802 my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4804 # commit values which do not have any impact on running VM first
4805 # Note: those option cannot raise errors, we we do not care about
4806 # $selection and always apply them.
4808 my $add_error = sub {
4809 my ($opt, $msg) = @_;
4810 $errors->{$opt} = "hotplug problem - $msg";
4814 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4815 if ($fast_plug_option->{$opt}) {
4816 $conf->{$opt} = $conf->{pending}->{$opt};
4817 delete $conf->{pending}->{$opt};
4823 PVE::QemuConfig->write_config($vmid, $conf);
4824 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4827 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4829 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4830 while (my ($opt, $force) = each %$pending_delete_hash) {
4831 next if $selection && !$selection->{$opt};
4833 if ($opt eq 'hotplug
') {
4834 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4835 } elsif ($opt eq 'tablet
') {
4836 die "skip\n" if !$hotplug_features->{usb};
4837 if ($defaults->{tablet}) {
4838 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4839 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4840 if $arch eq 'aarch64
';
4842 vm_deviceunplug($vmid, $conf, 'tablet
');
4843 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4845 } elsif ($opt =~ m/^usb\d+/) {
4847 # since we cannot reliably hot unplug usb devices
4848 # we are disabling it
4849 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4850 vm_deviceunplug($vmid, $conf, $opt);
4851 } elsif ($opt eq 'vcpus
') {
4852 die "skip\n" if !$hotplug_features->{cpu};
4853 qemu_cpu_hotplug($vmid, $conf, undef);
4854 } elsif ($opt eq 'balloon
') {
4855 # enable balloon device is not hotpluggable
4856 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4857 # here we reset the ballooning value to memory
4858 my $balloon = $conf->{memory} || $defaults->{memory};
4859 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4860 } elsif ($fast_plug_option->{$opt}) {
4862 } elsif ($opt =~ m/^net(\d+)$/) {
4863 die "skip\n" if !$hotplug_features->{network};
4864 vm_deviceunplug($vmid, $conf, $opt);
4865 } elsif (is_valid_drivename($opt)) {
4866 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4867 vm_deviceunplug($vmid, $conf, $opt);
4868 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4869 } elsif ($opt =~ m/^memory$/) {
4870 die "skip\n" if !$hotplug_features->{memory};
4871 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4872 } elsif ($opt eq 'cpuunits
') {
4873 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4874 } elsif ($opt eq 'cpulimit
') {
4875 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4881 &$add_error($opt, $err) if $err ne "skip\n";
4883 # save new config if hotplug was successful
4884 delete $conf->{$opt};
4885 vmconfig_undelete_pending_option($conf, $opt);
4886 PVE::QemuConfig->write_config($vmid, $conf);
4887 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4891 my $apply_pending_cloudinit;
4892 $apply_pending_cloudinit = sub {
4893 my ($key, $value) = @_;
4894 $apply_pending_cloudinit = sub {}; # once is enough
4896 my @cloudinit_opts = keys %$confdesc_cloudinit;
4897 foreach my $opt (keys %{$conf->{pending}}) {
4898 next if !grep { $_ eq $opt } @cloudinit_opts;
4899 $conf->{$opt} = delete $conf->{pending}->{$opt};
4902 my $new_conf = { %$conf };
4903 $new_conf->{$key} = $value;
4904 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4907 foreach my $opt (keys %{$conf->{pending}}) {
4908 next if $selection && !$selection->{$opt};
4909 my $value = $conf->{pending}->{$opt};
4911 if ($opt eq 'hotplug
') {
4912 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4913 } elsif ($opt eq 'tablet
') {
4914 die "skip\n" if !$hotplug_features->{usb};
4916 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4917 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4918 if $arch eq 'aarch64
';
4919 } elsif ($value == 0) {
4920 vm_deviceunplug($vmid, $conf, 'tablet
');
4921 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4923 } elsif ($opt =~ m/^usb\d+$/) {
4925 # since we cannot reliably hot unplug usb devices
4926 # we are disabling it
4927 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4928 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4929 die "skip\n" if !$d;
4930 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4931 } elsif ($opt eq 'vcpus
') {
4932 die "skip\n" if !$hotplug_features->{cpu};
4933 qemu_cpu_hotplug($vmid, $conf, $value);
4934 } elsif ($opt eq 'balloon
') {
4935 # enable/disable balloning device is not hotpluggable
4936 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4937 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4938 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4940 # allow manual ballooning if shares is set to zero
4941 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4942 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4943 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4945 } elsif ($opt =~ m/^net(\d+)$/) {
4946 # some changes can be done without hotplug
4947 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4948 $vmid, $opt, $value, $arch, $machine_type);
4949 } elsif (is_valid_drivename($opt)) {
4950 # some changes can be done without hotplug
4951 my $drive = parse_drive($opt, $value);
4952 if (drive_is_cloudinit($drive)) {
4953 &$apply_pending_cloudinit($opt, $value);
4955 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4956 $vmid, $opt, $value, 1, $arch, $machine_type);
4957 } elsif ($opt =~ m/^memory$/) { #dimms
4958 die "skip\n" if !$hotplug_features->{memory};
4959 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4960 } elsif ($opt eq 'cpuunits
') {
4961 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4962 } elsif ($opt eq 'cpulimit
') {
4963 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4964 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4966 die "skip\n"; # skip non-hot-pluggable options
4970 &$add_error($opt, $err) if $err ne "skip\n";
4972 # save new config if hotplug was successful
4973 $conf->{$opt} = $value;
4974 delete $conf->{pending}->{$opt};
4975 PVE::QemuConfig->write_config($vmid, $conf);
4976 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4981 sub try_deallocate_drive {
4982 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4984 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4985 my $volid = $drive->{file};
4986 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4987 my $sid = PVE::Storage::parse_volume_id($volid);
4988 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4990 # check if the disk is really unused
4991 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4992 if is_volume_in_use($storecfg, $conf, $key, $volid);
4993 PVE::Storage::vdisk_free($storecfg, $volid);
4996 # If vm is not owner of this disk remove from config
5004 sub vmconfig_delete_or_detach_drive {
5005 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5007 my $drive = parse_drive($opt, $conf->{$opt});
5009 my $rpcenv = PVE::RPCEnvironment::get();
5010 my $authuser = $rpcenv->get_user();
5013 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
5014 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5016 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5020 sub vmconfig_apply_pending {
5021 my ($vmid, $conf, $storecfg) = @_;
5025 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5026 while (my ($opt, $force) = each %$pending_delete_hash) {
5027 die "internal error" if $opt =~ m/^unused/;
5028 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5029 if (!defined($conf->{$opt})) {
5030 vmconfig_undelete_pending_option($conf, $opt);
5031 PVE::QemuConfig->write_config($vmid, $conf);
5032 } elsif (is_valid_drivename($opt)) {
5033 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5034 vmconfig_undelete_pending_option($conf, $opt);
5035 delete $conf->{$opt};
5036 PVE::QemuConfig->write_config($vmid, $conf);
5038 vmconfig_undelete_pending_option($conf, $opt);
5039 delete $conf->{$opt};
5040 PVE::QemuConfig->write_config($vmid, $conf);
5044 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5046 foreach my $opt (keys %{$conf->{pending}}) { # add/change
5047 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5049 if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5050 # skip if nothing changed
5051 } elsif (is_valid_drivename($opt)) {
5052 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5053 if defined($conf->{$opt});
5054 $conf->{$opt} = $conf->{pending}->{$opt};
5056 $conf->{$opt} = $conf->{pending}->{$opt};
5059 delete $conf->{pending}->{$opt};
5060 PVE::QemuConfig->write_config($vmid, $conf);
5064 my $safe_num_ne = sub {
5067 return 0 if !defined($a) && !defined($b);
5068 return 1 if !defined($a);
5069 return 1 if !defined($b);
5074 my $safe_string_ne = sub {
5077 return 0 if !defined($a) && !defined($b);
5078 return 1 if !defined($a);
5079 return 1 if !defined($b);
5084 sub vmconfig_update_net {
5085 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5087 my $newnet = parse_net($value);
5089 if ($conf->{$opt}) {
5090 my $oldnet = parse_net($conf->{$opt});
5092 if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5093 &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5094 &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5095 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5097 # for non online change, we try to hot-unplug
5098 die "skip\n" if !$hotplug;
5099 vm_deviceunplug($vmid, $conf, $opt);
5102 die "internal error" if $opt !~ m/net(\d+)/;
5103 my $iface = "tap${vmid}i$1";
5105 if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5106 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5107 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5108 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5109 PVE::Network::tap_unplug($iface);
5110 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5111 } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5112 # Rate can be applied on its own but any change above needs to
5113 # include the rate in tap_plug since OVS resets everything.
5114 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5117 if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5118 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5126 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5132 sub vmconfig_update_disk {
5133 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5135 # fixme: do we need force?
5137 my $drive = parse_drive($opt, $value);
5139 if ($conf->{$opt}) {
5141 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
5143 my $media = $drive->{media} || 'disk
';
5144 my $oldmedia = $old_drive->{media} || 'disk
';
5145 die "unable to change media type\n" if $media ne $oldmedia;
5147 if (!drive_is_cdrom($old_drive)) {
5149 if ($drive->{file} ne $old_drive->{file}) {
5151 die "skip\n" if !$hotplug;
5153 # unplug and register as unused
5154 vm_deviceunplug($vmid, $conf, $opt);
5155 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5158 # update existing disk
5160 # skip non hotpluggable value
5161 if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5162 &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5163 &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5164 &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5169 if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5170 &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5171 &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5172 &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5173 &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5174 &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5175 &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5176 &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5177 &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5178 &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5179 &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5180 &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5181 &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5182 &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5183 &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5184 &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5185 &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5186 &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5188 qemu_block_set_io_throttle($vmid,"drive-$opt",
5189 ($drive->{mbps} || 0)*1024*1024,
5190 ($drive->{mbps_rd} || 0)*1024*1024,
5191 ($drive->{mbps_wr} || 0)*1024*1024,
5192 $drive->{iops} || 0,
5193 $drive->{iops_rd} || 0,
5194 $drive->{iops_wr} || 0,
5195 ($drive->{mbps_max} || 0)*1024*1024,
5196 ($drive->{mbps_rd_max} || 0)*1024*1024,
5197 ($drive->{mbps_wr_max} || 0)*1024*1024,
5198 $drive->{iops_max} || 0,
5199 $drive->{iops_rd_max} || 0,
5200 $drive->{iops_wr_max} || 0,
5201 $drive->{bps_max_length} || 1,
5202 $drive->{bps_rd_max_length} || 1,
5203 $drive->{bps_wr_max_length} || 1,
5204 $drive->{iops_max_length} || 1,
5205 $drive->{iops_rd_max_length} || 1,
5206 $drive->{iops_wr_max_length} || 1);
5215 if ($drive->{file} eq 'none
') {
5216 vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5217 if (drive_is_cloudinit($old_drive)) {
5218 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5221 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5222 vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5223 vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5231 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5233 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5234 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5238 my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5239 $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5241 PVE::QemuConfig->lock_config($vmid, sub {
5242 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5244 die "you can't start a vm
if it
's a template\n" if PVE::QemuConfig->is_template($conf);
5246 my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended
');
5248 PVE::QemuConfig->check_lock($conf)
5249 if !($skiplock || $is_suspended);
5251 die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5253 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5254 vmconfig_apply_pending($vmid, $conf, $storecfg);
5255 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5258 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5260 my $defaults = load_defaults();
5262 # set environment variable useful inside network script
5263 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5265 my $local_volumes = {};
5267 if ($targetstorage) {
5268 foreach_drive($conf, sub {
5269 my ($ds, $drive) = @_;
5271 return if drive_is_cdrom($drive);
5273 my $volid = $drive->{file};
5277 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5279 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5280 return if $scfg->{shared};
5281 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5286 foreach my $opt (sort keys %$local_volumes) {
5288 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5289 my $drive = parse_drive($opt, $conf->{$opt});
5291 #if remote storage is specified, use default format
5292 if ($targetstorage && $targetstorage ne "1") {
5293 $storeid = $targetstorage;
5294 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5295 $format = $defFormat;
5297 #else we use same format than original
5298 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5299 $format = qemu_img_format($scfg, $volid);
5302 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5303 my $newdrive = $drive;
5304 $newdrive->{format} = $format;
5305 $newdrive->{file} = $newvolid;
5306 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5307 $local_volumes->{$opt} = $drivestr;
5308 #pass drive to conf for command line
5309 $conf->{$opt} = $drivestr;
5313 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
5315 if ($is_suspended) {
5316 # enforce machine type on suspended vm to ensure HW compatibility
5317 $forcemachine = $conf->{runningmachine};
5318 print "Resuming suspended VM\n";
5321 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5323 my $migrate_port = 0;
5326 if ($statefile eq 'tcp
') {
5327 my $localip = "localhost";
5328 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5329 my $nodename = PVE::INotify::nodename();
5331 if (!defined($migration_type)) {
5332 if (defined($datacenterconf->{migration}->{type})) {
5333 $migration_type = $datacenterconf->{migration}->{type};
5335 $migration_type = 'secure
';
5339 if ($migration_type eq 'insecure
') {
5340 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5341 if ($migrate_network_addr) {
5342 $localip = $migrate_network_addr;
5344 $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5347 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5350 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5351 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5352 $migrate_uri = "tcp:${localip}:${migrate_port}";
5353 push @$cmd, '-incoming
', $migrate_uri;
5356 } elsif ($statefile eq 'unix
') {
5357 # should be default for secure migrations as a ssh TCP forward
5358 # tunnel is not deterministic reliable ready and fails regurarly
5359 # to set up in time, so use UNIX socket forwards
5360 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5361 unlink $socket_addr;
5363 $migrate_uri = "unix:$socket_addr";
5365 push @$cmd, '-incoming
', $migrate_uri;
5369 push @$cmd, '-loadstate
', $statefile;
5376 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
5377 my $d = parse_hostpci($conf->{"hostpci$i"});
5379 my $pcidevices = $d->{pciid};
5380 foreach my $pcidevice (@$pcidevices) {
5381 my $pciid = $pcidevice->{id};
5383 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5384 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5385 die "no pci device info for device '$pciid'\n" if !$info;
5388 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5389 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5391 die "can't unbind
/bind pci group to vfio
'$pciid'\n"
5392 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5393 die "can
't reset pci device '$pciid'\n"
5394 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5399 PVE::Storage::activate_volumes($storecfg, $vollist);
5402 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5403 outfunc => sub {}, errfunc => sub {});
5405 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5406 # timeout should be more than enough here...
5407 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5409 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5410 : $defaults->{cpuunits};
5412 my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5413 my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5416 Slice => 'qemu
.slice
',
5418 CPUShares => $cpuunits
5421 if (my $cpulimit = $conf->{cpulimit}) {
5422 $properties{CPUQuota} = int($cpulimit * 100);
5424 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5426 my $run_qemu = sub {
5427 PVE::Tools::run_fork sub {
5428 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5429 run_command($cmd, %run_params);
5433 if ($conf->{hugepages}) {
5436 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5437 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5439 PVE::QemuServer::Memory::hugepages_mount();
5440 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5442 eval { $run_qemu->() };
5444 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5448 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5450 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5453 eval { $run_qemu->() };
5457 # deactivate volumes if start fails
5458 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5459 die "start failed: $err";
5462 print "migration listens on $migrate_uri\n" if $migrate_uri;
5464 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5465 eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5469 #start nbd server for storage migration
5470 if ($targetstorage) {
5471 my $nodename = PVE::INotify::nodename();
5472 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5473 my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5474 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5475 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5477 vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet
', data => { host => "${localip}", port => "${migrate_port}" } } );
5479 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5481 foreach my $opt (sort keys %$local_volumes) {
5482 my $volid = $local_volumes->{$opt};
5483 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5484 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5485 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5489 if ($migratedfrom) {
5491 set_migration_caps($vmid);
5496 print "spice listens on port $spice_port\n";
5497 if ($spice_ticket) {
5498 vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice
', password => $spice_ticket);
5499 vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice
', time => "+30");
5504 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5505 if !$statefile && $conf->{balloon};
5507 foreach my $opt (keys %$conf) {
5508 next if $opt !~ m/^net\d+$/;
5509 my $nicconf = parse_net($conf->{$opt});
5510 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5514 vm_mon_cmd_nocheck($vmid, 'qom-set
',
5515 path => "machine/peripheral/balloon0",
5516 property => "guest-stats-polling-interval",
5517 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5519 if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5520 print "Resumed VM, removing state\n";
5521 delete $conf->@{qw(lock vmstate runningmachine)};
5522 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5523 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5524 PVE
::QemuConfig-
>write_config($vmid, $conf);
5527 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5532 my ($vmid, $execute, %params) = @_;
5534 my $cmd = { execute
=> $execute, arguments
=> \
%params };
5535 vm_qmp_command
($vmid, $cmd);
5538 sub vm_mon_cmd_nocheck
{
5539 my ($vmid, $execute, %params) = @_;
5541 my $cmd = { execute
=> $execute, arguments
=> \
%params };
5542 vm_qmp_command
($vmid, $cmd, 1);
5545 sub vm_qmp_command
{
5546 my ($vmid, $cmd, $nocheck) = @_;
5551 if ($cmd->{arguments
}) {
5552 $timeout = delete $cmd->{arguments
}->{timeout
};
5556 die "VM $vmid not running\n" if !check_running
($vmid, $nocheck);
5557 my $sname = qmp_socket
($vmid);
5558 if (-e
$sname) { # test if VM is reasonambe new and supports qmp/qga
5559 my $qmpclient = PVE
::QMPClient-
>new();
5561 $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5563 die "unable to open monitor socket\n";
5567 syslog
("err", "VM $vmid qmp command failed - $err");
5574 sub vm_human_monitor_command
{
5575 my ($vmid, $cmdline) = @_;
5578 execute
=> 'human-monitor-command',
5579 arguments
=> { 'command-line' => $cmdline},
5582 return vm_qmp_command
($vmid, $cmd);
5585 sub vm_commandline
{
5586 my ($storecfg, $vmid, $snapname) = @_;
5588 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5591 my $snapshot = $conf->{snapshots
}->{$snapname};
5592 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5594 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5599 my $defaults = load_defaults
();
5601 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults);
5603 return PVE
::Tools
::cmd2string
($cmd);
5607 my ($vmid, $skiplock) = @_;
5609 PVE
::QemuConfig-
>lock_config($vmid, sub {
5611 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5613 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5615 vm_mon_cmd
($vmid, "system_reset");
5619 sub get_vm_volumes
{
5623 foreach_volid
($conf, sub {
5624 my ($volid, $attr) = @_;
5626 return if $volid =~ m
|^/|;
5628 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5631 push @$vollist, $volid;
5637 sub vm_stop_cleanup
{
5638 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5643 my $vollist = get_vm_volumes
($conf);
5644 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5647 foreach my $ext (qw(mon qmp pid vnc qga)) {
5648 unlink "/var/run/qemu-server/${vmid}.$ext";
5651 if ($conf->{ivshmem
}) {
5652 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5653 # just delete it for now, VMs which have this already open do not
5654 # are affected, but new VMs will get a separated one. If this
5655 # becomes an issue we either add some sort of ref-counting or just
5656 # add a "don't delete on stop" flag to the ivshmem format.
5657 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5660 foreach my $key (keys %$conf) {
5661 next if $key !~ m/^hostpci(\d+)$/;
5662 my $hostpciindex = $1;
5663 my $d = parse_hostpci
($conf->{$key});
5664 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5666 foreach my $pci (@{$d->{pciid
}}) {
5667 my $pciid = $pci->{id
};
5668 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5672 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5674 warn $@ if $@; # avoid errors - just warn
5677 # Note: use $nockeck to skip tests if VM configuration file exists.
5678 # We need that when migration VMs to other nodes (files already moved)
5679 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5681 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5683 $force = 1 if !defined($force) && !$shutdown;
5686 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5687 kill 15, $pid if $pid;
5688 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5689 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5693 PVE
::QemuConfig-
>lock_config($vmid, sub {
5695 my $pid = check_running
($vmid, $nocheck);
5700 $conf = PVE
::QemuConfig-
>load_config($vmid);
5701 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5702 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5703 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5704 $timeout = $opts->{down
} if $opts->{down
};
5706 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5711 if (defined($conf) && parse_guest_agent
($conf)->{enabled
}) {
5712 vm_qmp_command
($vmid, {
5713 execute
=> "guest-shutdown",
5714 arguments
=> { timeout
=> $timeout }
5717 vm_qmp_command
($vmid, { execute
=> "system_powerdown" }, $nocheck);
5720 vm_qmp_command
($vmid, { execute
=> "quit" }, $nocheck);
5726 $timeout = 60 if !defined($timeout);
5729 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5734 if ($count >= $timeout) {
5736 warn "VM still running - terminating now with SIGTERM\n";
5739 die "VM quit/powerdown failed - got timeout\n";
5742 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5747 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5750 die "VM quit/powerdown failed\n";
5758 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5763 if ($count >= $timeout) {
5764 warn "VM still running - terminating now with SIGKILL\n";
5769 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5774 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5781 PVE
::QemuConfig-
>lock_config($vmid, sub {
5783 $conf = PVE
::QemuConfig-
>load_config($vmid);
5785 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5786 PVE
::QemuConfig-
>check_lock($conf)
5787 if !($skiplock || $is_backing_up);
5789 die "cannot suspend to disk during backup\n"
5790 if $is_backing_up && $includestate;
5792 if ($includestate) {
5793 $conf->{lock} = 'suspending';
5794 my $date = strftime
("%Y-%m-%d", localtime(time()));
5795 $storecfg = PVE
::Storage
::config
();
5796 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5797 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5798 PVE
::QemuConfig-
>write_config($vmid, $conf);
5800 vm_mon_cmd
($vmid, "stop");
5804 if ($includestate) {
5806 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5809 vm_mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5811 my $state = vm_mon_cmd_nocheck
($vmid, "query-savevm");
5812 if (!$state->{status
}) {
5813 die "savevm not active\n";
5814 } elsif ($state->{status
} eq 'active') {
5817 } elsif ($state->{status
} eq 'completed') {
5818 print "State saved, quitting\n";
5820 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5821 die "query-savevm failed with error '$state->{error}'\n"
5823 die "query-savevm returned status '$state->{status}'\n";
5829 PVE
::QemuConfig-
>lock_config($vmid, sub {
5830 $conf = PVE
::QemuConfig-
>load_config($vmid);
5832 # cleanup, but leave suspending lock, to indicate something went wrong
5834 vm_mon_cmd
($vmid, "savevm-end");
5835 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5836 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5837 delete $conf->@{qw(vmstate runningmachine)};
5838 PVE
::QemuConfig-
>write_config($vmid, $conf);
5844 die "lock changed unexpectedly\n"
5845 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5847 vm_qmp_command
($vmid, { execute
=> "quit" });
5848 $conf->{lock} = 'suspended';
5849 PVE
::QemuConfig-
>write_config($vmid, $conf);
5855 my ($vmid, $skiplock, $nocheck) = @_;
5857 PVE
::QemuConfig-
>lock_config($vmid, sub {
5858 my $vm_mon_cmd = $nocheck ? \
&vm_mon_cmd_nocheck
: \
&vm_mon_cmd
;
5859 my $res = $vm_mon_cmd->($vmid, 'query-status');
5860 my $resume_cmd = 'cont';
5862 if ($res->{status
} && $res->{status
} eq 'suspended') {
5863 $resume_cmd = 'system_wakeup';
5868 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5870 PVE
::QemuConfig-
>check_lock($conf)
5871 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5874 $vm_mon_cmd->($vmid, $resume_cmd);
5879 my ($vmid, $skiplock, $key) = @_;
5881 PVE
::QemuConfig-
>lock_config($vmid, sub {
5883 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5885 # there is no qmp command, so we use the human monitor command
5886 my $res = vm_human_monitor_command
($vmid, "sendkey $key");
5887 die $res if $res ne '';
5892 my ($storecfg, $vmid, $skiplock) = @_;
5894 PVE
::QemuConfig-
>lock_config($vmid, sub {
5896 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5898 if (!check_running
($vmid)) {
5899 destroy_vm
($storecfg, $vmid, undef, $skiplock);
5901 die "VM $vmid is running - destroy failed\n";
5906 # vzdump restore implementaion
5908 sub tar_archive_read_firstfile
{
5909 my $archive = shift;
5911 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5913 # try to detect archive type first
5914 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5915 die "unable to open file '$archive'\n";
5916 my $firstfile = <$fh>;
5920 die "ERROR: archive contaions no data\n" if !$firstfile;
5926 sub tar_restore_cleanup
{
5927 my ($storecfg, $statfile) = @_;
5929 print STDERR
"starting cleanup\n";
5931 if (my $fd = IO
::File-
>new($statfile, "r")) {
5932 while (defined(my $line = <$fd>)) {
5933 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5936 if ($volid =~ m
|^/|) {
5937 unlink $volid || die 'unlink failed\n';
5939 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5941 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5943 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5945 print STDERR
"unable to parse line in statfile - $line";
5952 sub restore_archive
{
5953 my ($archive, $vmid, $user, $opts) = @_;
5955 my $format = $opts->{format
};
5958 if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5959 $format = 'tar' if !$format;
5961 } elsif ($archive =~ m/\.tar$/) {
5962 $format = 'tar' if !$format;
5963 } elsif ($archive =~ m/.tar.lzo$/) {
5964 $format = 'tar' if !$format;
5966 } elsif ($archive =~ m/\.vma$/) {
5967 $format = 'vma' if !$format;
5968 } elsif ($archive =~ m/\.vma\.gz$/) {
5969 $format = 'vma' if !$format;
5971 } elsif ($archive =~ m/\.vma\.lzo$/) {
5972 $format = 'vma' if !$format;
5975 $format = 'vma' if !$format; # default
5978 # try to detect archive format
5979 if ($format eq 'tar') {
5980 return restore_tar_archive
($archive, $vmid, $user, $opts);
5982 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5986 sub restore_update_config_line
{
5987 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5989 return if $line =~ m/^\#qmdump\#/;
5990 return if $line =~ m/^\#vzdump\#/;
5991 return if $line =~ m/^lock:/;
5992 return if $line =~ m/^unused\d+:/;
5993 return if $line =~ m/^parent:/;
5995 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5996 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5997 # try to convert old 1.X settings
5998 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5999 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6000 my ($model, $macaddr) = split(/\=/, $devconfig);
6001 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6004 bridge
=> "vmbr$ind",
6005 macaddr
=> $macaddr,
6007 my $netstr = print_net
($net);
6009 print $outfd "net$cookie->{netcount}: $netstr\n";
6010 $cookie->{netcount
}++;
6012 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6013 my ($id, $netstr) = ($1, $2);
6014 my $net = parse_net
($netstr);
6015 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6016 $netstr = print_net
($net);
6017 print $outfd "$id: $netstr\n";
6018 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6021 my $di = parse_drive
($virtdev, $value);
6022 if (defined($di->{backup
}) && !$di->{backup
}) {
6023 print $outfd "#$line";
6024 } elsif ($map->{$virtdev}) {
6025 delete $di->{format
}; # format can change on restore
6026 $di->{file
} = $map->{$virtdev};
6027 $value = print_drive
($vmid, $di);
6028 print $outfd "$virtdev: $value\n";
6032 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6034 if ($vmgenid ne '0') {
6035 # always generate a new vmgenid if there was a valid one setup
6036 $vmgenid = generate_uuid
();
6038 print $outfd "vmgenid: $vmgenid\n";
6039 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6040 my ($uuid, $uuid_str);
6041 UUID
::generate
($uuid);
6042 UUID
::unparse
($uuid, $uuid_str);
6043 my $smbios1 = parse_smbios1
($2);
6044 $smbios1->{uuid
} = $uuid_str;
6045 print $outfd $1.print_smbios1
($smbios1)."\n";
6052 my ($cfg, $vmid) = @_;
6054 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
6056 my $volid_hash = {};
6057 foreach my $storeid (keys %$info) {
6058 foreach my $item (@{$info->{$storeid}}) {
6059 next if !($item->{volid
} && $item->{size
});
6060 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6061 $volid_hash->{$item->{volid
}} = $item;
6068 sub is_volume_in_use
{
6069 my ($storecfg, $conf, $skip_drive, $volid) = @_;
6071 my $path = PVE
::Storage
::path
($storecfg, $volid);
6073 my $scan_config = sub {
6074 my ($cref, $snapname) = @_;
6076 foreach my $key (keys %$cref) {
6077 my $value = $cref->{$key};
6078 if (is_valid_drivename
($key)) {
6079 next if $skip_drive && $key eq $skip_drive;
6080 my $drive = parse_drive
($key, $value);
6081 next if !$drive || !$drive->{file
} || drive_is_cdrom
($drive);
6082 return 1 if $volid eq $drive->{file
};
6083 if ($drive->{file
} =~ m!^/!) {
6084 return 1 if $drive->{file
} eq $path;
6086 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
}, 1);
6088 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid, 1);
6090 return 1 if $path eq PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
6098 return 1 if &$scan_config($conf);
6102 foreach my $snapname (keys %{$conf->{snapshots
}}) {
6103 return 1 if &$scan_config($conf->{snapshots
}->{$snapname}, $snapname);
6109 sub update_disksize
{
6110 my ($vmid, $conf, $volid_hash) = @_;
6113 my $prefix = "VM $vmid:";
6115 # used and unused disks
6116 my $referenced = {};
6118 # Note: it is allowed to define multiple storages with same path (alias), so
6119 # we need to check both 'volid' and real 'path' (two different volid can point
6120 # to the same path).
6122 my $referencedpath = {};
6125 foreach my $opt (keys %$conf) {
6126 if (is_valid_drivename
($opt)) {
6127 my $drive = parse_drive
($opt, $conf->{$opt});
6128 my $volid = $drive->{file
};
6131 $referenced->{$volid} = 1;
6132 if ($volid_hash->{$volid} &&
6133 (my $path = $volid_hash->{$volid}->{path
})) {
6134 $referencedpath->{$path} = 1;
6137 next if drive_is_cdrom
($drive);
6138 next if !$volid_hash->{$volid};
6140 $drive->{size
} = $volid_hash->{$volid}->{size
};
6141 my $new = print_drive
($vmid, $drive);
6142 if ($new ne $conf->{$opt}) {
6144 $conf->{$opt} = $new;
6145 print "$prefix update disk '$opt' information.\n";
6150 # remove 'unusedX' entry if volume is used
6151 foreach my $opt (keys %$conf) {
6152 next if $opt !~ m/^unused\d+$/;
6153 my $volid = $conf->{$opt};
6154 my $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6155 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6156 print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6158 delete $conf->{$opt};
6161 $referenced->{$volid} = 1;
6162 $referencedpath->{$path} = 1 if $path;
6165 foreach my $volid (sort keys %$volid_hash) {
6166 next if $volid =~ m/vm-$vmid-state-/;
6167 next if $referenced->{$volid};
6168 my $path = $volid_hash->{$volid}->{path
};
6169 next if !$path; # just to be sure
6170 next if $referencedpath->{$path};
6172 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6173 print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6174 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6181 my ($vmid, $nolock, $dryrun) = @_;
6183 my $cfg = PVE
::Storage
::config
();
6185 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6186 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6187 foreach my $stor (keys %{$cfg->{ids
}}) {
6188 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
6191 print "rescan volumes...\n";
6192 my $volid_hash = scan_volids
($cfg, $vmid);
6194 my $updatefn = sub {
6197 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6199 PVE
::QemuConfig-
>check_lock($conf);
6202 foreach my $volid (keys %$volid_hash) {
6203 my $info = $volid_hash->{$volid};
6204 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6207 my $changes = update_disksize
($vmid, $conf, $vm_volids);
6209 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6212 if (defined($vmid)) {
6216 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6219 my $vmlist = config_list
();
6220 foreach my $vmid (keys %$vmlist) {
6224 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6230 sub restore_vma_archive
{
6231 my ($archive, $vmid, $user, $opts, $comp) = @_;
6233 my $readfrom = $archive;
6235 my $cfg = PVE
::Storage
::config
();
6237 my $bwlimit = $opts->{bwlimit
};
6239 my $dbg_cmdstring = '';
6240 my $add_pipe = sub {
6242 push @$commands, $cmd;
6243 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6244 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6249 if ($archive eq '-') {
6252 # If we use a backup from a PVE defined storage we also consider that
6253 # storage's rate limit:
6254 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6255 if (defined($volid)) {
6256 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6257 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6259 print STDERR
"applying read rate limit: $readlimit\n";
6260 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6261 $add_pipe->($cstream);
6268 if ($comp eq 'gzip') {
6269 $cmd = ['zcat', $readfrom];
6270 } elsif ($comp eq 'lzop') {
6271 $cmd = ['lzop', '-d', '-c', $readfrom];
6273 die "unknown compression method '$comp'\n";
6278 my $tmpdir = "/var/tmp/vzdumptmp$$";
6281 # disable interrupts (always do cleanups)
6285 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6287 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6288 POSIX
::mkfifo
($mapfifo, 0600);
6291 my $openfifo = sub {
6292 open($fifofh, '>', $mapfifo) || die $!;
6295 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6302 my $rpcenv = PVE
::RPCEnvironment
::get
();
6304 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6305 my $tmpfn = "$conffile.$$.tmp";
6307 # Note: $oldconf is undef if VM does not exists
6308 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6309 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6313 my $print_devmap = sub {
6314 my $virtdev_hash = {};
6316 my $cfgfn = "$tmpdir/qemu-server.conf";
6318 # we can read the config - that is already extracted
6319 my $fh = IO
::File-
>new($cfgfn, "r") ||
6320 "unable to read qemu-server.conf - $!\n";
6322 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6324 my $pve_firewall_dir = '/etc/pve/firewall';
6325 mkdir $pve_firewall_dir; # make sure the dir exists
6326 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6329 while (defined(my $line = <$fh>)) {
6330 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6331 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6332 die "archive does not contain data for drive '$virtdev'\n"
6333 if !$devinfo->{$devname};
6334 if (defined($opts->{storage
})) {
6335 $storeid = $opts->{storage
} || 'local';
6336 } elsif (!$storeid) {
6339 $format = 'raw' if !$format;
6340 $devinfo->{$devname}->{devname
} = $devname;
6341 $devinfo->{$devname}->{virtdev
} = $virtdev;
6342 $devinfo->{$devname}->{format
} = $format;
6343 $devinfo->{$devname}->{storeid
} = $storeid;
6345 # check permission on storage
6346 my $pool = $opts->{pool
}; # todo: do we need that?
6347 if ($user ne 'root@pam') {
6348 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6351 $storage_limits{$storeid} = $bwlimit;
6353 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6354 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6356 my $drive = parse_drive
($virtdev, $2);
6357 if (drive_is_cloudinit
($drive)) {
6358 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6359 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
6360 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6364 storeid
=> $opts->{storage
} // $storeid,
6365 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6366 file
=> $drive->{file
}, # to make drive_is_cloudinit check possible
6367 name
=> "vm-$vmid-cloudinit",
6370 $virtdev_hash->{$virtdev} = $d;
6375 foreach my $key (keys %storage_limits) {
6376 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6378 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6379 $storage_limits{$key} = $limit * 1024;
6382 foreach my $devname (keys %$devinfo) {
6383 die "found no device mapping information for device '$devname'\n"
6384 if !$devinfo->{$devname}->{virtdev
};
6387 # create empty/temp config
6389 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6390 foreach_drive
($oldconf, sub {
6391 my ($ds, $drive) = @_;
6393 return if !$drive->{is_cloudinit
} && drive_is_cdrom
($drive);
6395 my $volid = $drive->{file
};
6396 return if !$volid || $volid =~ m
|^/|;
6398 my ($path, $owner) = PVE
::Storage
::path
($cfg, $volid);
6399 return if !$path || !$owner || ($owner != $vmid);
6401 # Note: only delete disk we want to restore
6402 # other volumes will become unused
6403 if ($virtdev_hash->{$ds}) {
6404 eval { PVE
::Storage
::vdisk_free
($cfg, $volid); };
6411 # delete vmstate files, after the restore we have no snapshots anymore
6412 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
6413 my $snap = $oldconf->{snapshots
}->{$snapname};
6414 if ($snap->{vmstate
}) {
6415 eval { PVE
::Storage
::vdisk_free
($cfg, $snap->{vmstate
}); };
6424 foreach my $virtdev (sort keys %$virtdev_hash) {
6425 my $d = $virtdev_hash->{$virtdev};
6426 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6427 my $storeid = $d->{storeid
};
6428 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
6431 if (my $limit = $storage_limits{$storeid}) {
6432 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6435 # test if requested format is supported
6436 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($cfg, $storeid);
6437 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6438 $d->{format
} = $defFormat if !$supported;
6441 if ($d->{is_cloudinit
}) {
6443 $name .= ".$d->{format}" if $d->{format
} ne 'raw';
6446 my $volid = PVE
::Storage
::vdisk_alloc
($cfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6447 print STDERR
"new volume ID is '$volid'\n";
6448 $d->{volid
} = $volid;
6450 PVE
::Storage
::activate_volumes
($cfg, [$volid]);
6452 my $write_zeros = 1;
6453 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6457 if (!$d->{is_cloudinit
}) {
6458 my $path = PVE
::Storage
::path
($cfg, $volid);
6460 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6462 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6464 $map->{$virtdev} = $volid;
6467 $fh->seek(0, 0) || die "seek failed - $!\n";
6469 my $outfd = new IO
::File
($tmpfn, "w") ||
6470 die "unable to write config for VM $vmid\n";
6472 my $cookie = { netcount
=> 0 };
6473 while (defined(my $line = <$fh>)) {
6474 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6487 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6488 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6490 $oldtimeout = alarm($timeout);
6497 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6498 my ($dev_id, $size, $devname) = ($1, $2, $3);
6499 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6500 } elsif ($line =~ m/^CTIME: /) {
6501 # we correctly received the vma config, so we can disable
6502 # the timeout now for disk allocation (set to 10 minutes, so
6503 # that we always timeout if something goes wrong)
6506 print $fifofh "done\n";
6507 my $tmp = $oldtimeout || 0;
6508 $oldtimeout = undef;
6514 print "restore vma archive: $dbg_cmdstring\n";
6515 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6519 alarm($oldtimeout) if $oldtimeout;
6522 foreach my $devname (keys %$devinfo) {
6523 my $volid = $devinfo->{$devname}->{volid
};
6524 push @$vollist, $volid if $volid;
6527 PVE
::Storage
::deactivate_volumes
($cfg, $vollist);
6535 foreach my $devname (keys %$devinfo) {
6536 my $volid = $devinfo->{$devname}->{volid
};
6539 if ($volid =~ m
|^/|) {
6540 unlink $volid || die 'unlink failed\n';
6542 PVE
::Storage
::vdisk_free
($cfg, $volid);
6544 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6546 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6553 rename($tmpfn, $conffile) ||
6554 die "unable to commit configuration file '$conffile'\n";
6556 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6558 eval { rescan
($vmid, 1); };
6562 sub restore_tar_archive
{
6563 my ($archive, $vmid, $user, $opts) = @_;
6565 if ($archive ne '-') {
6566 my $firstfile = tar_archive_read_firstfile
($archive);
6567 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6568 if $firstfile ne 'qemu-server.conf';
6571 my $storecfg = PVE
::Storage
::config
();
6573 # destroy existing data - keep empty config
6574 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6575 destroy_vm
($storecfg, $vmid, 1) if -f
$vmcfgfn;
6577 my $tocmd = "/usr/lib/qemu-server/qmextract";
6579 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6580 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6581 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6582 $tocmd .= ' --info' if $opts->{info
};
6584 # tar option "xf" does not autodetect compression when read from STDIN,
6585 # so we pipe to zcat
6586 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6587 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6589 my $tmpdir = "/var/tmp/vzdumptmp$$";
6592 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6593 local $ENV{VZDUMP_VMID
} = $vmid;
6594 local $ENV{VZDUMP_USER
} = $user;
6596 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6597 my $tmpfn = "$conffile.$$.tmp";
6599 # disable interrupts (always do cleanups)
6603 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6611 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6613 if ($archive eq '-') {
6614 print "extracting archive from STDIN\n";
6615 run_command
($cmd, input
=> "<&STDIN");
6617 print "extracting archive '$archive'\n";
6621 return if $opts->{info
};
6625 my $statfile = "$tmpdir/qmrestore.stat";
6626 if (my $fd = IO
::File-
>new($statfile, "r")) {
6627 while (defined (my $line = <$fd>)) {
6628 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6629 $map->{$1} = $2 if $1;
6631 print STDERR
"unable to parse line in statfile - $line\n";
6637 my $confsrc = "$tmpdir/qemu-server.conf";
6639 my $srcfd = new IO
::File
($confsrc, "r") ||
6640 die "unable to open file '$confsrc'\n";
6642 my $outfd = new IO
::File
($tmpfn, "w") ||
6643 die "unable to write config for VM $vmid\n";
6645 my $cookie = { netcount
=> 0 };
6646 while (defined (my $line = <$srcfd>)) {
6647 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6659 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6666 rename $tmpfn, $conffile ||
6667 die "unable to commit configuration file '$conffile'\n";
6669 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6671 eval { rescan
($vmid, 1); };
6675 sub foreach_storage_used_by_vm
{
6676 my ($conf, $func) = @_;
6680 foreach_drive
($conf, sub {
6681 my ($ds, $drive) = @_;
6682 return if drive_is_cdrom
($drive);
6684 my $volid = $drive->{file
};
6686 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6687 $sidhash->{$sid} = $sid if $sid;
6690 foreach my $sid (sort keys %$sidhash) {
6695 sub do_snapshots_with_qemu
{
6696 my ($storecfg, $volid) = @_;
6698 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6699 my $scfg = $storecfg->{ids
}->{$storage_name};
6701 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6705 if ($volid =~ m/\.(qcow2|qed)$/){
6712 sub qga_check_running
{
6713 my ($vmid, $nowarn) = @_;
6715 eval { vm_mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6717 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6723 sub template_create
{
6724 my ($vmid, $conf, $disk) = @_;
6726 my $storecfg = PVE
::Storage
::config
();
6728 foreach_drive
($conf, sub {
6729 my ($ds, $drive) = @_;
6731 return if drive_is_cdrom
($drive);
6732 return if $disk && $ds ne $disk;
6734 my $volid = $drive->{file
};
6735 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6737 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6738 $drive->{file
} = $voliddst;
6739 $conf->{$ds} = print_drive
($vmid, $drive);
6740 PVE
::QemuConfig-
>write_config($vmid, $conf);
6744 sub convert_iscsi_path
{
6747 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6752 my $initiator_name = get_initiator_name
();
6754 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6755 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6758 die "cannot convert iscsi path '$path', unkown format\n";
6761 sub qemu_img_convert
{
6762 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6764 my $storecfg = PVE
::Storage
::config
();
6765 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6766 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6768 if ($src_storeid && $dst_storeid) {
6770 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6772 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6773 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6775 my $src_format = qemu_img_format
($src_scfg, $src_volname);
6776 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6778 my $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6779 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6781 my $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6782 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6785 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6786 push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6787 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6788 push @$cmd, '-T', 'none' if $src_scfg->{type
} eq 'zfspool';
6790 if ($src_is_iscsi) {
6791 push @$cmd, '--image-opts';
6792 $src_path = convert_iscsi_path
($src_path);
6794 push @$cmd, '-f', $src_format;
6797 if ($dst_is_iscsi) {
6798 push @$cmd, '--target-image-opts';
6799 $dst_path = convert_iscsi_path
($dst_path);
6801 push @$cmd, '-O', $dst_format;
6804 push @$cmd, $src_path;
6806 if (!$dst_is_iscsi && $is_zero_initialized) {
6807 push @$cmd, "zeroinit:$dst_path";
6809 push @$cmd, $dst_path;
6814 if($line =~ m/\((\S+)\/100\
%\)/){
6816 my $transferred = int($size * $percent / 100);
6817 my $remaining = $size - $transferred;
6819 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6824 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6826 die "copy failed: $err" if $err;
6830 sub qemu_img_format
{
6831 my ($scfg, $volname) = @_;
6833 if ($scfg->{path
} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6840 sub qemu_drive_mirror
{
6841 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6843 $jobs = {} if !$jobs;
6847 $jobs->{"drive-$drive"} = {};
6849 if ($dst_volid =~ /^nbd:/) {
6850 $qemu_target = $dst_volid;
6853 my $storecfg = PVE
::Storage
::config
();
6854 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6856 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6858 $format = qemu_img_format
($dst_scfg, $dst_volname);
6860 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6862 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6865 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6866 $opts->{format
} = $format if $format;
6868 if (defined($bwlimit)) {
6869 $opts->{speed
} = $bwlimit * 1024;
6870 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6872 print "drive mirror is starting for drive-$drive\n";
6875 # if a job already runs for this device we get an error, catch it for cleanup
6876 eval { vm_mon_cmd
($vmid, "drive-mirror", %$opts); };
6878 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6880 die "mirroring error: $err\n";
6883 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6886 sub qemu_drive_mirror_monitor
{
6887 my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6890 my $err_complete = 0;
6893 die "storage migration timed out\n" if $err_complete > 300;
6895 my $stats = vm_mon_cmd
($vmid, "query-block-jobs");
6897 my $running_mirror_jobs = {};
6898 foreach my $stat (@$stats) {
6899 next if $stat->{type
} ne 'mirror';
6900 $running_mirror_jobs->{$stat->{device
}} = $stat;
6903 my $readycounter = 0;
6905 foreach my $job (keys %$jobs) {
6907 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6908 print "$job : finished\n";
6909 delete $jobs->{$job};
6913 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6915 my $busy = $running_mirror_jobs->{$job}->{busy
};
6916 my $ready = $running_mirror_jobs->{$job}->{ready
};
6917 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6918 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6919 my $remaining = $total - $transferred;
6920 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6922 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6925 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6928 last if scalar(keys %$jobs) == 0;
6930 if ($readycounter == scalar(keys %$jobs)) {
6931 print "all mirroring jobs are ready \n";
6932 last if $skipcomplete; #do the complete later
6934 if ($vmiddst && $vmiddst != $vmid) {
6935 my $agent_running = $qga && qga_check_running
($vmid);
6936 if ($agent_running) {
6937 print "freeze filesystem\n";
6938 eval { PVE
::QemuServer
::vm_mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6940 print "suspend vm\n";
6941 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6944 # if we clone a disk for a new target vm, we don't switch the disk
6945 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6947 if ($agent_running) {
6948 print "unfreeze filesystem\n";
6949 eval { PVE
::QemuServer
::vm_mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6951 print "resume vm\n";
6952 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6958 foreach my $job (keys %$jobs) {
6959 # try to switch the disk if source and destination are on the same guest
6960 print "$job: Completing block job...\n";
6962 eval { vm_mon_cmd
($vmid, "block-job-complete", device
=> $job) };
6963 if ($@ =~ m/cannot be completed/) {
6964 print "$job: Block job cannot be completed, try again.\n";
6967 print "$job: Completed successfully.\n";
6968 $jobs->{$job}->{complete
} = 1;
6979 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6980 die "mirroring error: $err";
6985 sub qemu_blockjobs_cancel
{
6986 my ($vmid, $jobs) = @_;
6988 foreach my $job (keys %$jobs) {
6989 print "$job: Cancelling block job\n";
6990 eval { vm_mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6991 $jobs->{$job}->{cancel
} = 1;
6995 my $stats = vm_mon_cmd
($vmid, "query-block-jobs");
6997 my $running_jobs = {};
6998 foreach my $stat (@$stats) {
6999 $running_jobs->{$stat->{device
}} = $stat;
7002 foreach my $job (keys %$jobs) {
7004 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
7005 print "$job: Done.\n";
7006 delete $jobs->{$job};
7010 last if scalar(keys %$jobs) == 0;
7017 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7018 $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7023 print "create linked clone of drive $drivename ($drive->{file})\n";
7024 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
7025 push @$newvollist, $newvolid;
7028 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
7029 $storeid = $storage if $storage;
7031 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
7032 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
7034 print "create full clone of drive $drivename ($drive->{file})\n";
7036 if (drive_is_cloudinit
($drive)) {
7037 $name = "vm-$newvmid-cloudinit";
7039 # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7040 if ($dst_format ne 'raw') {
7041 $name .= ".$dst_format";
7044 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7045 push @$newvollist, $newvolid;
7047 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
7049 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
7050 if (!$running || $snapname) {
7051 # TODO: handle bwlimits
7052 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
7055 my $kvmver = get_running_qemu_version
($vmid);
7056 if (!qemu_machine_feature_enabled
(undef, $kvmver, 2, 7)) {
7057 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7058 if $drive->{iothread
};
7061 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7065 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
7068 $disk->{format
} = undef;
7069 $disk->{file
} = $newvolid;
7070 $disk->{size
} = $size;
7075 # this only works if VM is running
7076 sub get_current_qemu_machine
{
7079 my $cmd = { execute
=> 'query-machines', arguments
=> {} };
7080 my $res = vm_qmp_command
($vmid, $cmd);
7082 my ($current, $default);
7083 foreach my $e (@$res) {
7084 $default = $e->{name
} if $e->{'is-default'};
7085 $current = $e->{name
} if $e->{'is-current'};
7088 # fallback to the default machine if current is not supported by qemu
7089 return $current || $default || 'pc';
7092 sub get_running_qemu_version
{
7094 my $cmd = { execute
=> 'query-version', arguments
=> {} };
7095 my $res = vm_qmp_command
($vmid, $cmd);
7096 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7099 sub qemu_machine_feature_enabled
{
7100 my ($machine, $kvmver, $version_major, $version_minor) = @_;
7105 if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7107 $current_major = $3;
7108 $current_minor = $4;
7110 } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7112 $current_major = $1;
7113 $current_minor = $2;
7116 return 1 if $current_major > $version_major ||
7117 ($current_major == $version_major &&
7118 $current_minor >= $version_minor);
7121 sub qemu_machine_pxe
{
7122 my ($vmid, $conf, $machine) = @_;
7124 $machine = PVE
::QemuServer
::get_current_qemu_machine
($vmid) if !$machine;
7126 if ($conf->{machine
} && $conf->{machine
} =~ m/\.pxe$/) {
7133 sub qemu_use_old_bios_files
{
7134 my ($machine_type) = @_;
7136 return if !$machine_type;
7138 my $use_old_bios_files = undef;
7140 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7142 $use_old_bios_files = 1;
7144 my $kvmver = kvm_user_version
();
7145 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7146 # load new efi bios files on migration. So this hack is required to allow
7147 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7148 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7149 $use_old_bios_files = !qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 4);
7152 return ($use_old_bios_files, $machine_type);
7155 sub create_efidisk
($$$$$) {
7156 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7158 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7159 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7161 my $vars_size = PVE
::Tools
::convert_size
(-s
$ovmf_vars, 'b' => 'kb');
7162 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7163 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7165 my $path = PVE
::Storage
::path
($storecfg, $volid);
7167 run_command
(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7169 die "Copying EFI vars image failed: $@" if $@;
7171 return ($volid, $vars_size);
7174 sub vm_iothreads_list
{
7177 my $res = vm_mon_cmd
($vmid, 'query-iothreads');
7180 foreach my $iothread (@$res) {
7181 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7188 my ($conf, $drive) = @_;
7192 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7194 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7200 my $controller = int($drive->{index} / $maxdev);
7201 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single') ?
"virtioscsi" : "scsihw";
7203 return ($maxdev, $controller, $controller_prefix);
7206 sub add_hyperv_enlightenments
{
7207 my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7209 return if $winversion < 6;
7210 return if $bios && $bios eq 'ovmf' && $winversion < 8;
7212 if ($gpu_passthrough || defined($hv_vendor_id)) {
7213 $hv_vendor_id //= 'proxmox';
7214 push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7217 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3)) {
7218 push @$cpuFlags , 'hv_spinlocks=0x1fff';
7219 push @$cpuFlags , 'hv_vapic';
7220 push @$cpuFlags , 'hv_time';
7222 push @$cpuFlags , 'hv_spinlocks=0xffff';
7225 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 6)) {
7226 push @$cpuFlags , 'hv_reset';
7227 push @$cpuFlags , 'hv_vpindex';
7228 push @$cpuFlags , 'hv_runtime';
7231 if ($winversion >= 7) {
7232 push @$cpuFlags , 'hv_relaxed';
7234 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 12)) {
7235 push @$cpuFlags , 'hv_synic';
7236 push @$cpuFlags , 'hv_stimer';
7239 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 3, 1)) {
7240 push @$cpuFlags , 'hv_tlbflush';
7241 push @$cpuFlags , 'hv_ipi';
7242 # FIXME: AMD does not supports this currently, only add with special flag??
7243 #push @$cpuFlags , 'hv_evmcs';
7248 sub windows_version
{
7251 return 0 if !$ostype;
7255 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7257 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7259 } elsif ($ostype =~ m/^win(\d+)$/) {
7266 sub resolve_dst_disk_format
{
7267 my ($storecfg, $storeid, $src_volname, $format) = @_;
7268 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7271 # if no target format is specified, use the source disk format as hint
7273 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7274 $format = qemu_img_format
($scfg, $src_volname);
7280 # test if requested format is supported - else use default
7281 my $supported = grep { $_ eq $format } @$validFormats;
7282 $format = $defFormat if !$supported;
7286 sub resolve_first_disk
{
7288 my @disks = PVE
::QemuServer
::valid_drive_names
();
7290 foreach my $ds (reverse @disks) {
7291 next if !$conf->{$ds};
7292 my $disk = PVE
::QemuServer
::parse_drive
($ds, $conf->{$ds});
7293 next if PVE
::QemuServer
::drive_is_cdrom
($disk);
7300 my ($uuid, $uuid_str);
7301 UUID
::generate
($uuid);
7302 UUID
::unparse
($uuid, $uuid_str);
7306 sub generate_smbios1_uuid
{
7307 return "uuid=".generate_uuid
();
7313 vm_mon_cmd
($vmid, 'nbd-server-stop');
7316 # bash completion helper
7318 sub complete_backup_archives
{
7319 my ($cmdname, $pname, $cvalue) = @_;
7321 my $cfg = PVE
::Storage
::config
();
7325 if ($cvalue =~ m/^([^:]+):/) {
7329 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7332 foreach my $id (keys %$data) {
7333 foreach my $item (@{$data->{$id}}) {
7334 next if $item->{format
} !~ m/^vma\.(gz|lzo)$/;
7335 push @$res, $item->{volid
} if defined($item->{volid
});
7342 my $complete_vmid_full = sub {
7345 my $idlist = vmstatus
();
7349 foreach my $id (keys %$idlist) {
7350 my $d = $idlist->{$id};
7351 if (defined($running)) {
7352 next if $d->{template
};
7353 next if $running && $d->{status
} ne 'running';
7354 next if !$running && $d->{status
} eq 'running';
7363 return &$complete_vmid_full();
7366 sub complete_vmid_stopped
{
7367 return &$complete_vmid_full(0);
7370 sub complete_vmid_running
{
7371 return &$complete_vmid_full(1);
7374 sub complete_storage
{
7376 my $cfg = PVE
::Storage
::config
();
7377 my $ids = $cfg->{ids
};
7380 foreach my $sid (keys %$ids) {
7381 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7382 next if !$ids->{$sid}->{content
}->{images
};