1 package PVE
::QemuServer
;
23 use Storable
qw(dclone);
25 use PVE
::Exception
qw(raise raise_param_exc);
27 use PVE
::Tools
qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
28 use PVE
::JSONSchema
qw(get_standard_option);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
34 use PVE
::RPCEnvironment
;
35 use PVE
::GuestHelpers
;
36 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr);
37 use PVE
::QemuServer
::Memory
;
38 use PVE
::QemuServer
::USB
qw(parse_usb_device);
39 use PVE
::QemuServer
::Cloudinit
;
42 use Time
::HiRes
qw(gettimeofday);
43 use File
::Copy
qw(copy);
46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
49 "$EDK2_FW_BASE/OVMF_CODE.fd",
50 "$EDK2_FW_BASE/OVMF_VARS.fd"
53 "$EDK2_FW_BASE/AAVMF_CODE.fd",
54 "$EDK2_FW_BASE/AAVMF_VARS.fd"
58 my $qemu_snap_storage = { rbd
=> 1 };
60 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
64 # Note about locking: we use flock on the config file protect
65 # against concurent actions.
66 # Aditionaly, we have a 'lock' setting in the config file. This
67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
68 # allowed when such lock is set. But you can ignore this kind of
69 # lock with the --skiplock flag.
71 cfs_register_file
('/qemu-server/',
75 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
76 description
=> "Some command save/restore state from this location.",
82 PVE
::JSONSchema
::register_standard_option
('pve-qm-image-format', {
84 enum
=> [qw(raw cow qcow qed qcow2 vmdk cloop)],
85 description
=> "The drive's backing file's data format.",
89 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
90 description
=> "Specifies the Qemu machine type.",
92 pattern
=> '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
97 #no warnings 'redefine';
100 my ($controller, $vmid, $option, $value) = @_;
102 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
103 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
107 my $nodename = PVE
::INotify
::nodename
();
109 mkdir "/etc/pve/nodes/$nodename";
110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
113 my $var_run_tmpdir = "/var/run/qemu-server";
114 mkdir $var_run_tmpdir;
116 my $lock_dir = "/var/lock/qemu-server";
119 my $cpu_vendor_list = {
121 486 => 'GenuineIntel',
122 pentium
=> 'GenuineIntel',
123 pentium2
=> 'GenuineIntel',
124 pentium3
=> 'GenuineIntel',
125 coreduo
=> 'GenuineIntel',
126 core2duo
=> 'GenuineIntel',
127 Conroe
=> 'GenuineIntel',
128 Penryn
=> 'GenuineIntel',
129 Nehalem
=> 'GenuineIntel',
130 'Nehalem-IBRS' => 'GenuineIntel',
131 Westmere
=> 'GenuineIntel',
132 'Westmere-IBRS' => 'GenuineIntel',
133 SandyBridge
=> 'GenuineIntel',
134 'SandyBridge-IBRS' => 'GenuineIntel',
135 IvyBridge
=> 'GenuineIntel',
136 'IvyBridge-IBRS' => 'GenuineIntel',
137 Haswell
=> 'GenuineIntel',
138 'Haswell-IBRS' => 'GenuineIntel',
139 'Haswell-noTSX' => 'GenuineIntel',
140 'Haswell-noTSX-IBRS' => 'GenuineIntel',
141 Broadwell
=> 'GenuineIntel',
142 'Broadwell-IBRS' => 'GenuineIntel',
143 'Broadwell-noTSX' => 'GenuineIntel',
144 'Broadwell-noTSX-IBRS' => 'GenuineIntel',
145 'Skylake-Client' => 'GenuineIntel',
146 'Skylake-Client-IBRS' => 'GenuineIntel',
147 'Skylake-Server' => 'GenuineIntel',
148 'Skylake-Server-IBRS' => 'GenuineIntel',
151 athlon
=> 'AuthenticAMD',
152 phenom
=> 'AuthenticAMD',
153 Opteron_G1
=> 'AuthenticAMD',
154 Opteron_G2
=> 'AuthenticAMD',
155 Opteron_G3
=> 'AuthenticAMD',
156 Opteron_G4
=> 'AuthenticAMD',
157 Opteron_G5
=> 'AuthenticAMD',
158 EPYC
=> 'AuthenticAMD',
159 'EPYC-IBPB' => 'AuthenticAMD',
161 # generic types, use vendor from host node
170 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb|md-clear)/;
174 description
=> "Emulated CPU type.",
176 enum
=> [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
181 description
=> "Do not identify as a KVM virtual machine.",
188 pattern
=> qr/[a-zA-Z0-9]{1,12}/,
189 format_description
=> 'vendor-id',
190 description
=> 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
194 description
=> "List of additional CPU flags separated by ';'."
195 . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
196 . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb', 'md-clear'.",
197 format_description
=> '+FLAG[;-FLAG...]',
199 pattern
=> qr/$cpu_flag(;$cpu_flag)*/,
208 enum
=> [qw(i6300esb ib700)],
209 description
=> "Watchdog type to emulate.",
210 default => 'i6300esb',
215 enum
=> [qw(reset shutdown poweroff pause debug none)],
216 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
220 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
224 description
=> "Enable/disable Qemu GuestAgent.",
229 fstrim_cloned_disks
=> {
230 description
=> "Run fstrim after cloning/moving a disk.",
239 description
=> "Select the VGA type.",
244 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
247 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
259 description
=> "The size of the file in MB.",
263 pattern
=> '[a-zA-Z0-9\-]+',
265 format_description
=> 'string',
266 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
274 description
=> "Specifies whether a VM will be started during system bootup.",
280 description
=> "Automatic restart after crash (currently ignored).",
285 type
=> 'string', format
=> 'pve-hotplug-features',
286 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
287 default => 'network,disk,usb',
292 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
298 description
=> "Lock/unlock the VM.",
299 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
304 description
=> "Limit of CPU usage.",
305 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
313 description
=> "CPU weight for a VM.",
314 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
322 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
329 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
335 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
343 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
344 "It should not be necessary to set it.",
345 enum
=> PVE
::Tools
::kvmkeymaplist
(),
350 type
=> 'string', format
=> 'dns-name',
351 description
=> "Set a name for the VM. Only used on the configuration web interface.",
356 description
=> "SCSI controller model",
357 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
363 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
368 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
369 description
=> "Specify guest operating system.",
370 verbose_description
=> <<EODESC,
371 Specify guest operating system. This is used to enable special
372 optimization/features for specific operating systems:
375 other;; unspecified OS
376 wxp;; Microsoft Windows XP
377 w2k;; Microsoft Windows 2000
378 w2k3;; Microsoft Windows 2003
379 w2k8;; Microsoft Windows 2008
380 wvista;; Microsoft Windows Vista
381 win7;; Microsoft Windows 7
382 win8;; Microsoft Windows 8/2012/2012r2
383 win10;; Microsoft Windows 10/2016
384 l24;; Linux 2.4 Kernel
385 l26;; Linux 2.6/3.X Kernel
386 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
392 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
393 pattern
=> '[acdn]{1,4}',
398 type
=> 'string', format
=> 'pve-qm-bootdisk',
399 description
=> "Enable booting from specified disk.",
400 pattern
=> '(ide|sata|scsi|virtio)\d+',
405 description
=> "The number of CPUs. Please use option -sockets instead.",
412 description
=> "The number of CPU sockets.",
419 description
=> "The number of cores per socket.",
426 description
=> "Enable/disable NUMA.",
432 description
=> "Enable/disable hugepages memory.",
433 enum
=> [qw(any 2 1024)],
438 description
=> "Number of hotplugged vcpus.",
445 description
=> "Enable/disable ACPI.",
450 description
=> "Enable/disable Qemu GuestAgent and its properties.",
452 format
=> $agent_fmt,
457 description
=> "Enable/disable KVM hardware virtualization.",
463 description
=> "Enable/disable time drift fix.",
469 description
=> "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
474 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
478 type
=> 'string', format
=> $vga_fmt,
479 description
=> "Configure the VGA hardware.",
480 verbose_description
=> "Configure the VGA Hardware. If you want to use ".
481 "high resolution modes (>= 1280x1024x16) you may need to increase " .
482 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
483 "is 'std' for all OS types besides some Windows versions (XP and " .
484 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
485 "display server. For win* OS you can select how many independent " .
486 "displays you want, Linux guests can add displays them self.\n".
487 "You can also run without any graphic card, using a serial device as terminal.",
491 type
=> 'string', format
=> 'pve-qm-watchdog',
492 description
=> "Create a virtual hardware watchdog device.",
493 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled" .
494 " (by a guest action), the watchdog must be periodically polled " .
495 "by an agent inside the guest or else the watchdog will reset " .
496 "the guest (or execute the respective action specified)",
501 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
502 description
=> "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
503 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
506 startup
=> get_standard_option
('pve-startup-order'),
510 description
=> "Enable/disable Template.",
516 description
=> "Arbitrary arguments passed to kvm.",
517 verbose_description
=> <<EODESCR,
518 Arbitrary arguments passed to kvm, for example:
520 args: -no-reboot -no-hpet
522 NOTE: this option is for experts only.
529 description
=> "Enable/disable the USB tablet device.",
530 verbose_description
=> "Enable/disable the USB tablet device. This device is " .
531 "usually needed to allow absolute mouse positioning with VNC. " .
532 "Else the mouse runs out of sync with normal VNC clients. " .
533 "If you're running lots of console-only guests on one host, " .
534 "you may consider disabling this to save some context switches. " .
535 "This is turned off by default if you use spice (-vga=qxl).",
540 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
544 migrate_downtime
=> {
547 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
553 type
=> 'string', format
=> 'pve-qm-ide',
554 typetext
=> '<volume>',
555 description
=> "This is an alias for option -ide2",
559 description
=> "Emulated CPU type.",
563 parent
=> get_standard_option
('pve-snapshot-name', {
565 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
569 description
=> "Timestamp for snapshots.",
575 type
=> 'string', format
=> 'pve-volume-id',
576 description
=> "Reference to a volume which stores the VM state. This is used internally for snapshots.",
578 vmstatestorage
=> get_standard_option
('pve-storage-id', {
579 description
=> "Default storage for VM state volumes/files.",
582 runningmachine
=> get_standard_option
('pve-qemu-machine', {
583 description
=> "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
585 machine
=> get_standard_option
('pve-qemu-machine'),
587 description
=> "Virtual processor architecture. Defaults to the host.",
590 enum
=> [qw(x86_64 aarch64)],
593 description
=> "Specify SMBIOS type 1 fields.",
594 type
=> 'string', format
=> 'pve-qm-smbios1',
601 description
=> "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
607 enum
=> [ qw(seabios ovmf) ],
608 description
=> "Select BIOS implementation.",
609 default => 'seabios',
613 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
614 format_description
=> 'UUID',
615 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
616 verbose_description
=> "The VM generation ID (vmgenid) device exposes a".
617 " 128-bit integer value identifier to the guest OS. This allows to".
618 " notify the guest operating system when the virtual machine is".
619 " executed with a different configuration (e.g. snapshot execution".
620 " or creation from a template). The guest operating system notices".
621 " the change, and is then able to react as appropriate by marking".
622 " its copies of distributed databases as dirty, re-initializing its".
623 " random number generator, etc.\n".
624 "Note that auto-creation only works when done throug API/CLI create".
625 " or update methods, but not when manually editing the config file.",
626 default => "1 (autogenerated)",
631 format
=> 'pve-volume-id',
633 description
=> "Script that will be executed during various steps in the vms lifetime.",
637 format
=> $ivshmem_fmt,
638 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
647 description
=> 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
648 format
=> 'pve-volume-id',
649 format_description
=> 'volume',
654 description
=> 'Specify a custom file containing all network data passed to the VM via cloud-init.',
655 format
=> 'pve-volume-id',
656 format_description
=> 'volume',
661 description
=> 'Specify a custom file containing all user data passed to the VM via cloud-init.',
662 format
=> 'pve-volume-id',
663 format_description
=> 'volume',
666 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
668 my $confdesc_cloudinit = {
672 description
=> 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
673 enum
=> ['configdrive2', 'nocloud'],
678 description
=> "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
683 description
=> 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
688 description
=> 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
689 format
=> 'pve-qm-cicustom',
694 description
=> "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
698 type
=> 'string', format
=> 'address-list',
699 description
=> "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
704 format
=> 'urlencoded',
705 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
709 # what about other qemu settings ?
711 #machine => 'string',
724 ##soundhw => 'string',
726 while (my ($k, $v) = each %$confdesc) {
727 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
730 my $MAX_IDE_DISKS = 4;
731 my $MAX_SCSI_DISKS = 14;
732 my $MAX_VIRTIO_DISKS = 16;
733 my $MAX_SATA_DISKS = 6;
734 my $MAX_USB_DEVICES = 5;
736 my $MAX_UNUSED_DISKS = 256;
737 my $MAX_HOSTPCI_DEVICES = 4;
738 my $MAX_SERIAL_PORTS = 4;
739 my $MAX_PARALLEL_PORTS = 3;
745 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
746 description
=> "CPUs accessing this NUMA node.",
747 format_description
=> "id[-id];...",
751 description
=> "Amount of memory this NUMA node provides.",
756 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
757 description
=> "Host NUMA nodes to use.",
758 format_description
=> "id[-id];...",
763 enum
=> [qw(preferred bind interleave)],
764 description
=> "NUMA allocation policy.",
768 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
771 type
=> 'string', format
=> $numa_fmt,
772 description
=> "NUMA topology.",
774 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
776 for (my $i = 0; $i < $MAX_NUMA; $i++) {
777 $confdesc->{"numa$i"} = $numadesc;
780 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
781 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
782 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
783 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
785 my $net_fmt_bridge_descr = <<__EOD__;
786 Bridge to attach the network device to. The Proxmox VE standard bridge
789 If you do not specify a bridge, we create a kvm user (NATed) network
790 device, which provides DHCP and DNS services. The following addresses
797 The DHCP server assign addresses to the guest starting from 10.0.2.15.
801 macaddr
=> get_standard_option
('mac-addr', {
802 description
=> "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
806 description
=> "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
807 enum
=> $nic_model_list,
810 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
813 description
=> $net_fmt_bridge_descr,
814 format_description
=> 'bridge',
819 minimum
=> 0, maximum
=> 16,
820 description
=> 'Number of packet queues to be used on the device.',
826 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
831 minimum
=> 1, maximum
=> 4094,
832 description
=> 'VLAN tag to apply to packets on this interface.',
837 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
838 description
=> 'VLAN trunks to pass through this interface.',
839 format_description
=> 'vlanid[;vlanid...]',
844 description
=> 'Whether this interface should be protected by the firewall.',
849 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
856 type
=> 'string', format
=> $net_fmt,
857 description
=> "Specify network devices.",
860 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
865 format
=> 'pve-ipv4-config',
866 format_description
=> 'IPv4Format/CIDR',
867 description
=> 'IPv4 address in CIDR format.',
874 format_description
=> 'GatewayIPv4',
875 description
=> 'Default gateway for IPv4 traffic.',
881 format
=> 'pve-ipv6-config',
882 format_description
=> 'IPv6Format/CIDR',
883 description
=> 'IPv6 address in CIDR format.',
890 format_description
=> 'GatewayIPv6',
891 description
=> 'Default gateway for IPv6 traffic.',
896 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
899 type
=> 'string', format
=> 'pve-qm-ipconfig',
900 description
=> <<'EODESCR',
901 cloud-init: Specify IP addresses and gateways for the corresponding interface.
903 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
905 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
906 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
908 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
911 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
913 for (my $i = 0; $i < $MAX_NETS; $i++) {
914 $confdesc->{"net$i"} = $netdesc;
915 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
918 foreach my $key (keys %$confdesc_cloudinit) {
919 $confdesc->{$key} = $confdesc_cloudinit->{$key};
922 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
923 sub verify_volume_id_or_qm_path
{
924 my ($volid, $noerr) = @_;
926 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
930 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
931 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
933 return undef if $noerr;
941 my %drivedesc_base = (
942 volume
=> { alias
=> 'file' },
945 format
=> 'pve-volume-id-or-qm-path',
947 format_description
=> 'volume',
948 description
=> "The drive's backing volume.",
952 enum
=> [qw(cdrom disk)],
953 description
=> "The drive's media type.",
959 description
=> "Force the drive's physical geometry to have a specific cylinder count.",
964 description
=> "Force the drive's physical geometry to have a specific head count.",
969 description
=> "Force the drive's physical geometry to have a specific sector count.",
974 enum
=> [qw(none lba auto)],
975 description
=> "Force disk geometry bios translation mode.",
980 description
=> "Controls qemu's snapshot mode feature."
981 . " If activated, changes made to the disk are temporary and will"
982 . " be discarded when the VM is shutdown.",
987 enum
=> [qw(none writethrough writeback unsafe directsync)],
988 description
=> "The drive's cache mode",
991 format
=> get_standard_option
('pve-qm-image-format'),
994 format
=> 'disk-size',
995 format_description
=> 'DiskSize',
996 description
=> "Disk size. This is purely informational and has no effect.",
1001 description
=> "Whether the drive should be included when making backups.",
1006 description
=> 'Whether the drive should considered for replication jobs.',
1012 enum
=> [qw(ignore report stop)],
1013 description
=> 'Read error action.',
1018 enum
=> [qw(enospc ignore report stop)],
1019 description
=> 'Write error action.',
1024 enum
=> [qw(native threads)],
1025 description
=> 'AIO type to use.',
1030 enum
=> [qw(ignore on)],
1031 description
=> 'Controls whether to pass discard/trim requests to the underlying storage.',
1036 description
=> 'Controls whether to detect and try to optimize writes of zeroes.',
1041 format
=> 'urlencoded',
1042 format_description
=> 'serial',
1043 maxLength
=> 20*3, # *3 since it's %xx url enoded
1044 description
=> "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1049 description
=> 'Mark this locally-managed volume as available on all nodes',
1050 verbose_description
=> "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1056 my %iothread_fmt = ( iothread
=> {
1058 description
=> "Whether to use iothreads for this drive",
1065 format
=> 'urlencoded',
1066 format_description
=> 'model',
1067 maxLength
=> 40*3, # *3 since it's %xx url enoded
1068 description
=> "The drive's reported model name, url-encoded, up to 40 bytes long.",
1076 description
=> "Number of queues.",
1082 my %scsiblock_fmt = (
1085 description
=> "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1094 description
=> "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1102 pattern
=> qr/^(0x)[0-9a-fA-F]{16}/,
1103 format_description
=> 'wwn',
1104 description
=> "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1109 my $add_throttle_desc = sub {
1110 my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1113 format_description
=> $unit,
1114 description
=> "Maximum $what in $longunit.",
1117 $d->{minimum
} = $minimum if defined($minimum);
1118 $drivedesc_base{$key} = $d;
1120 # throughput: (leaky bucket)
1121 $add_throttle_desc->('bps', 'integer', 'r/w speed', 'bps', 'bytes per second');
1122 $add_throttle_desc->('bps_rd', 'integer', 'read speed', 'bps', 'bytes per second');
1123 $add_throttle_desc->('bps_wr', 'integer', 'write speed', 'bps', 'bytes per second');
1124 $add_throttle_desc->('mbps', 'number', 'r/w speed', 'mbps', 'megabytes per second');
1125 $add_throttle_desc->('mbps_rd', 'number', 'read speed', 'mbps', 'megabytes per second');
1126 $add_throttle_desc->('mbps_wr', 'number', 'write speed', 'mbps', 'megabytes per second');
1127 $add_throttle_desc->('iops', 'integer', 'r/w I/O', 'iops', 'operations per second');
1128 $add_throttle_desc->('iops_rd', 'integer', 'read I/O', 'iops', 'operations per second');
1129 $add_throttle_desc->('iops_wr', 'integer', 'write I/O', 'iops', 'operations per second');
1131 # pools: (pool of IO before throttling starts taking effect)
1132 $add_throttle_desc->('mbps_max', 'number', 'unthrottled r/w pool', 'mbps', 'megabytes per second');
1133 $add_throttle_desc->('mbps_rd_max', 'number', 'unthrottled read pool', 'mbps', 'megabytes per second');
1134 $add_throttle_desc->('mbps_wr_max', 'number', 'unthrottled write pool', 'mbps', 'megabytes per second');
1135 $add_throttle_desc->('iops_max', 'integer', 'unthrottled r/w I/O pool', 'iops', 'operations per second');
1136 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool', 'iops', 'operations per second');
1137 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1140 $add_throttle_desc->('bps_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1141 $add_throttle_desc->('bps_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1142 $add_throttle_desc->('bps_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1143 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1144 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1145 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1148 $drivedesc_base{'bps_rd_length'} = { alias
=> 'bps_rd_max_length' };
1149 $drivedesc_base{'bps_wr_length'} = { alias
=> 'bps_wr_max_length' };
1150 $drivedesc_base{'iops_rd_length'} = { alias
=> 'iops_rd_max_length' };
1151 $drivedesc_base{'iops_wr_length'} = { alias
=> 'iops_wr_max_length' };
1159 PVE
::JSONSchema
::register_format
("pve-qm-ide", $ide_fmt);
1163 type
=> 'string', format
=> $ide_fmt,
1164 description
=> "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1166 PVE
::JSONSchema
::register_standard_option
("pve-qm-ide", $idedesc);
1178 type
=> 'string', format
=> $scsi_fmt,
1179 description
=> "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1181 PVE
::JSONSchema
::register_standard_option
("pve-qm-scsi", $scsidesc);
1190 type
=> 'string', format
=> $sata_fmt,
1191 description
=> "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1193 PVE
::JSONSchema
::register_standard_option
("pve-qm-sata", $satadesc);
1201 type
=> 'string', format
=> $virtio_fmt,
1202 description
=> "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1204 PVE
::JSONSchema
::register_standard_option
("pve-qm-virtio", $virtiodesc);
1206 my $alldrive_fmt = {
1217 volume
=> { alias
=> 'file' },
1220 format
=> 'pve-volume-id-or-qm-path',
1222 format_description
=> 'volume',
1223 description
=> "The drive's backing volume.",
1225 format
=> get_standard_option
('pve-qm-image-format'),
1228 format
=> 'disk-size',
1229 format_description
=> 'DiskSize',
1230 description
=> "Disk size. This is purely informational and has no effect.",
1235 my $efidisk_desc = {
1237 type
=> 'string', format
=> $efidisk_fmt,
1238 description
=> "Configure a Disk for storing EFI vars",
1241 PVE
::JSONSchema
::register_standard_option
("pve-qm-efidisk", $efidisk_desc);
1246 type
=> 'string', format
=> 'pve-qm-usb-device',
1247 format_description
=> 'HOSTUSBDEVICE|spice',
1248 description
=> <<EODESCR,
1249 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1251 'bus-port(.port)*' (decimal numbers) or
1252 'vendor_id:product_id' (hexadeciaml numbers) or
1255 You can use the 'lsusb -t' command to list existing usb devices.
1257 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1259 The value 'spice' can be used to add a usb redirection devices for spice.
1265 description
=> "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1272 type
=> 'string', format
=> $usb_fmt,
1273 description
=> "Configure an USB device (n is 0 to 4).",
1275 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1277 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1282 pattern
=> qr/$PCIRE(;$PCIRE)*/,
1283 format_description
=> 'HOSTPCIID[;HOSTPCIID2...]',
1284 description
=> <<EODESCR,
1285 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1286 of PCI virtual functions of the host. HOSTPCIID syntax is:
1288 'bus:dev.func' (hexadecimal numbers)
1290 You can us the 'lspci' command to list existing PCI devices.
1295 description
=> "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1301 pattern
=> '[^,;]+',
1302 format_description
=> 'string',
1303 description
=> "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1308 description
=> "Choose the PCI-express bus (needs the 'q35' machine model).",
1314 description
=> "Enable vfio-vga device support.",
1320 format_description
=> 'string',
1321 pattern
=> '[^/\.:]+',
1323 description
=> <<EODESCR
1324 The type of mediated device to use.
1325 An instance of this type will be created on startup of the VM and
1326 will be cleaned up when the VM stops.
1330 PVE
::JSONSchema
::register_format
('pve-qm-hostpci', $hostpci_fmt);
1334 type
=> 'string', format
=> 'pve-qm-hostpci',
1335 description
=> "Map host PCI devices into guest.",
1336 verbose_description
=> <<EODESCR,
1337 Map host PCI devices into guest.
1339 NOTE: This option allows direct access to host hardware. So it is no longer
1340 possible to migrate such machines - use with special care.
1342 CAUTION: Experimental! User reported problems with this option.
1345 PVE
::JSONSchema
::register_standard_option
("pve-qm-hostpci", $hostpcidesc);
1350 pattern
=> '(/dev/.+|socket)',
1351 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1352 verbose_description
=> <<EODESCR,
1353 Create a serial device inside the VM (n is 0 to 3), and pass through a
1354 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1355 host side (use 'qm terminal' to open a terminal connection).
1357 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1359 CAUTION: Experimental! User reported problems with this option.
1366 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1367 description
=> "Map host parallel devices (n is 0 to 2).",
1368 verbose_description
=> <<EODESCR,
1369 Map host parallel devices (n is 0 to 2).
1371 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1373 CAUTION: Experimental! User reported problems with this option.
1377 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1378 $confdesc->{"parallel$i"} = $paralleldesc;
1381 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1382 $confdesc->{"serial$i"} = $serialdesc;
1385 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
1386 $confdesc->{"hostpci$i"} = $hostpcidesc;
1389 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) {
1390 $drivename_hash->{"ide$i"} = 1;
1391 $confdesc->{"ide$i"} = $idedesc;
1394 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) {
1395 $drivename_hash->{"sata$i"} = 1;
1396 $confdesc->{"sata$i"} = $satadesc;
1399 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) {
1400 $drivename_hash->{"scsi$i"} = 1;
1401 $confdesc->{"scsi$i"} = $scsidesc ;
1404 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) {
1405 $drivename_hash->{"virtio$i"} = 1;
1406 $confdesc->{"virtio$i"} = $virtiodesc;
1409 $drivename_hash->{efidisk0
} = 1;
1410 $confdesc->{efidisk0
} = $efidisk_desc;
1412 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1413 $confdesc->{"usb$i"} = $usbdesc;
1418 type
=> 'string', format
=> 'pve-volume-id',
1419 description
=> "Reference to unused volumes. This is used internally, and should not be modified manually.",
1422 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1423 $confdesc->{"unused$i"} = $unuseddesc;
1426 my $kvm_api_version = 0;
1429 return $kvm_api_version if $kvm_api_version;
1431 open my $fh, '<', '/dev/kvm'
1434 # 0xae00 => KVM_GET_API_VERSION
1435 $kvm_api_version = ioctl($fh, 0xae00, 0);
1437 return $kvm_api_version;
1440 my $kvm_user_version;
1442 sub kvm_user_version
{
1444 return $kvm_user_version if $kvm_user_version;
1446 $kvm_user_version = 'unknown';
1450 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1451 $kvm_user_version = $2;
1455 eval { run_command
("kvm -version", outfunc
=> $code); };
1458 return $kvm_user_version;
1462 sub kernel_has_vhost_net
{
1463 return -c
'/dev/vhost-net';
1466 sub valid_drive_names
{
1467 # order is important - used to autoselect boot disk
1468 return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1469 (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1470 (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1471 (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1475 sub is_valid_drivename
{
1478 return defined($drivename_hash->{$dev});
1483 return defined($confdesc->{$key});
1487 return $nic_model_list;
1490 sub os_list_description
{
1494 wxp
=> 'Windows XP',
1495 w2k
=> 'Windows 2000',
1496 w2k3
=>, 'Windows 2003',
1497 w2k8
=> 'Windows 2008',
1498 wvista
=> 'Windows Vista',
1499 win7
=> 'Windows 7',
1500 win8
=> 'Windows 8/2012',
1501 win10
=> 'Windows 10/2016',
1509 sub get_cdrom_path
{
1511 return $cdrom_path if $cdrom_path;
1513 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1514 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1515 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1519 my ($storecfg, $vmid, $cdrom) = @_;
1521 if ($cdrom eq 'cdrom') {
1522 return get_cdrom_path
();
1523 } elsif ($cdrom eq 'none') {
1525 } elsif ($cdrom =~ m
|^/|) {
1528 return PVE
::Storage
::path
($storecfg, $cdrom);
1532 # try to convert old style file names to volume IDs
1533 sub filename_to_volume_id
{
1534 my ($vmid, $file, $media) = @_;
1536 if (!($file eq 'none' || $file eq 'cdrom' ||
1537 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1539 return undef if $file =~ m
|/|;
1541 if ($media && $media eq 'cdrom') {
1542 $file = "local:iso/$file";
1544 $file = "local:$vmid/$file";
1551 sub verify_media_type
{
1552 my ($opt, $vtype, $media) = @_;
1557 if ($media eq 'disk') {
1559 } elsif ($media eq 'cdrom') {
1562 die "internal error";
1565 return if ($vtype eq $etype);
1567 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1570 sub cleanup_drive_path
{
1571 my ($opt, $storecfg, $drive) = @_;
1573 # try to convert filesystem paths to volume IDs
1575 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1576 ($drive->{file
} !~ m
|^/dev/.+|) &&
1577 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1578 ($drive->{file
} !~ m/^\d+$/)) {
1579 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1580 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1581 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1582 verify_media_type
($opt, $vtype, $drive->{media
});
1583 $drive->{file
} = $volid;
1586 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1589 sub parse_hotplug_features
{
1594 return $res if $data eq '0';
1596 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1598 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1599 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1602 die "invalid hotplug feature '$feature'\n";
1608 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1609 sub pve_verify_hotplug_features
{
1610 my ($value, $noerr) = @_;
1612 return $value if parse_hotplug_features
($value);
1614 return undef if $noerr;
1616 die "unable to parse hotplug option\n";
1619 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1620 # [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1621 # [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1622 # [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1623 # [,iothread=on][,serial=serial][,model=model]
1626 my ($key, $data) = @_;
1628 my ($interface, $index);
1630 if ($key =~ m/^([^\d]+)(\d+)$/) {
1637 my $desc = $key =~ /^unused\d+$/ ?
$alldrive_fmt
1638 : $confdesc->{$key}->{format
};
1640 warn "invalid drive key: $key\n";
1643 my $res = eval { PVE
::JSONSchema
::parse_property_string
($desc, $data) };
1644 return undef if !$res;
1645 $res->{interface
} = $interface;
1646 $res->{index} = $index;
1649 foreach my $opt (qw(bps bps_rd bps_wr)) {
1650 if (my $bps = defined(delete $res->{$opt})) {
1651 if (defined($res->{"m$opt"})) {
1652 warn "both $opt and m$opt specified\n";
1656 $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1660 # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1661 for my $requirement (
1662 [mbps_max
=> 'mbps'],
1663 [mbps_rd_max
=> 'mbps_rd'],
1664 [mbps_wr_max
=> 'mbps_wr'],
1665 [miops_max
=> 'miops'],
1666 [miops_rd_max
=> 'miops_rd'],
1667 [miops_wr_max
=> 'miops_wr'],
1668 [bps_max_length
=> 'mbps_max'],
1669 [bps_rd_max_length
=> 'mbps_rd_max'],
1670 [bps_wr_max_length
=> 'mbps_wr_max'],
1671 [iops_max_length
=> 'iops_max'],
1672 [iops_rd_max_length
=> 'iops_rd_max'],
1673 [iops_wr_max_length
=> 'iops_wr_max']) {
1674 my ($option, $requires) = @$requirement;
1675 if ($res->{$option} && !$res->{$requires}) {
1676 warn "$option requires $requires\n";
1681 return undef if $error;
1683 return undef if $res->{mbps_rd
} && $res->{mbps
};
1684 return undef if $res->{mbps_wr
} && $res->{mbps
};
1685 return undef if $res->{iops_rd
} && $res->{iops
};
1686 return undef if $res->{iops_wr
} && $res->{iops
};
1688 if ($res->{media
} && ($res->{media
} eq 'cdrom')) {
1689 return undef if $res->{snapshot
} || $res->{trans
} || $res->{format
};
1690 return undef if $res->{heads
} || $res->{secs
} || $res->{cyls
};
1691 return undef if $res->{interface
} eq 'virtio';
1694 if (my $size = $res->{size
}) {
1695 return undef if !defined($res->{size
} = PVE
::JSONSchema
::parse_size
($size));
1702 my ($vmid, $drive) = @_;
1703 my $data = { %$drive };
1704 delete $data->{$_} for qw(index interface);
1705 return PVE
::JSONSchema
::print_property_string
($data, $alldrive_fmt);
1709 my($fh, $noerr) = @_;
1712 my $SG_GET_VERSION_NUM = 0x2282;
1714 my $versionbuf = "\x00" x
8;
1715 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1717 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1720 my $version = unpack("I", $versionbuf);
1721 if ($version < 30000) {
1722 die "scsi generic interface too old\n" if !$noerr;
1726 my $buf = "\x00" x
36;
1727 my $sensebuf = "\x00" x
8;
1728 my $cmd = pack("C x3 C x1", 0x12, 36);
1730 # see /usr/include/scsi/sg.h
1731 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1733 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1734 length($sensebuf), 0, length($buf), $buf,
1735 $cmd, $sensebuf, 6000);
1737 $ret = ioctl($fh, $SG_IO, $packet);
1739 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1743 my @res = unpack($sg_io_hdr_t, $packet);
1744 if ($res[17] || $res[18]) {
1745 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1750 (my $byte0, my $byte1, $res->{vendor
},
1751 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1753 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1754 $res->{type
} = $byte0 & 31;
1762 my $fh = IO
::File-
>new("+<$path") || return undef;
1763 my $res = scsi_inquiry
($fh, 1);
1769 sub machine_type_is_q35
{
1772 return $conf->{machine
} && ($conf->{machine
} =~ m/q35/) ?
1 : 0;
1775 sub print_tabletdevice_full
{
1776 my ($conf, $arch) = @_;
1778 my $q35 = machine_type_is_q35
($conf);
1780 # we use uhci for old VMs because tablet driver was buggy in older qemu
1782 if (machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1788 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1791 sub print_keyboarddevice_full
{
1792 my ($conf, $arch, $machine) = @_;
1794 return undef if $arch ne 'aarch64';
1796 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1799 sub print_drivedevice_full
{
1800 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1805 if ($drive->{interface
} eq 'virtio') {
1806 my $pciaddr = print_pci_addr
("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1807 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1808 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread
};
1809 } elsif ($drive->{interface
} eq 'scsi') {
1811 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1812 my $unit = $drive->{index} % $maxdev;
1813 my $devicetype = 'hd';
1815 if (drive_is_cdrom
($drive)) {
1818 if ($drive->{file
} =~ m
|^/|) {
1819 $path = $drive->{file
};
1820 if (my $info = path_is_scsi
($path)) {
1821 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1822 $devicetype = 'block';
1823 } elsif ($info->{type
} == 1) { # tape
1824 $devicetype = 'generic';
1828 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1831 if($path =~ m/^iscsi\:\/\
//){
1832 $devicetype = 'generic';
1836 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1837 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1839 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1842 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1843 $device .= ",rotation_rate=1";
1845 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1847 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1848 my $maxdev = ($drive->{interface
} eq 'sata') ?
$MAX_SATA_DISKS : 2;
1849 my $controller = int($drive->{index} / $maxdev);
1850 my $unit = $drive->{index} % $maxdev;
1851 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1853 $device = "ide-$devicetype";
1854 if ($drive->{interface
} eq 'ide') {
1855 $device .= ",bus=ide.$controller,unit=$unit";
1857 $device .= ",bus=ahci$controller.$unit";
1859 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1861 if ($devicetype eq 'hd') {
1862 if (my $model = $drive->{model
}) {
1863 $model = URI
::Escape
::uri_unescape
($model);
1864 $device .= ",model=$model";
1866 if ($drive->{ssd
}) {
1867 $device .= ",rotation_rate=1";
1870 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1871 } elsif ($drive->{interface
} eq 'usb') {
1873 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1875 die "unsupported interface type";
1878 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1880 if (my $serial = $drive->{serial
}) {
1881 $serial = URI
::Escape
::uri_unescape
($serial);
1882 $device .= ",serial=$serial";
1889 sub get_initiator_name
{
1892 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return undef;
1893 while (defined(my $line = <$fh>)) {
1894 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1903 sub print_drive_full
{
1904 my ($storecfg, $vmid, $drive) = @_;
1907 my $volid = $drive->{file
};
1910 if (drive_is_cdrom
($drive)) {
1911 $path = get_iso_path
($storecfg, $vmid, $volid);
1913 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1915 $path = PVE
::Storage
::path
($storecfg, $volid);
1916 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1917 $format = qemu_img_format
($scfg, $volname);
1925 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1926 foreach my $o (@qemu_drive_options) {
1927 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1930 # snapshot only accepts on|off
1931 if (defined($drive->{snapshot
})) {
1932 my $v = $drive->{snapshot
} ?
'on' : 'off';
1933 $opts .= ",snapshot=$v";
1936 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1937 my ($dir, $qmpname) = @$type;
1938 if (my $v = $drive->{"mbps$dir"}) {
1939 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1941 if (my $v = $drive->{"mbps${dir}_max"}) {
1942 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1944 if (my $v = $drive->{"bps${dir}_max_length"}) {
1945 $opts .= ",throttling.bps$qmpname-max-length=$v";
1947 if (my $v = $drive->{"iops${dir}"}) {
1948 $opts .= ",throttling.iops$qmpname=$v";
1950 if (my $v = $drive->{"iops${dir}_max"}) {
1951 $opts .= ",throttling.iops$qmpname-max=$v";
1953 if (my $v = $drive->{"iops${dir}_max_length"}) {
1954 $opts .= ",throttling.iops$qmpname-max-length=$v";
1958 $opts .= ",format=$format" if $format && !$drive->{format
};
1960 my $cache_direct = 0;
1962 if (my $cache = $drive->{cache
}) {
1963 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1964 } elsif (!drive_is_cdrom
($drive)) {
1965 $opts .= ",cache=none";
1969 # aio native works only with O_DIRECT
1970 if (!$drive->{aio
}) {
1972 $opts .= ",aio=native";
1974 $opts .= ",aio=threads";
1978 if (!drive_is_cdrom
($drive)) {
1980 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1981 $detectzeroes = 'off';
1982 } elsif ($drive->{discard
}) {
1983 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1985 # This used to be our default with discard not being specified:
1986 $detectzeroes = 'on';
1988 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1991 my $pathinfo = $path ?
"file=$path," : '';
1993 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1996 sub print_netdevice_full
{
1997 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1999 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
2001 my $device = $net->{model
};
2002 if ($net->{model
} eq 'virtio') {
2003 $device = 'virtio-net-pci';
2006 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
2007 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2008 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
2009 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2010 my $vectors = $net->{queues
} * 2 + 2;
2011 $tmpstr .= ",vectors=$vectors,mq=on";
2013 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
2015 if ($use_old_bios_files) {
2017 if ($device eq 'virtio-net-pci') {
2018 $romfile = 'pxe-virtio.rom';
2019 } elsif ($device eq 'e1000') {
2020 $romfile = 'pxe-e1000.rom';
2021 } elsif ($device eq 'ne2k') {
2022 $romfile = 'pxe-ne2k_pci.rom';
2023 } elsif ($device eq 'pcnet') {
2024 $romfile = 'pxe-pcnet.rom';
2025 } elsif ($device eq 'rtl8139') {
2026 $romfile = 'pxe-rtl8139.rom';
2028 $tmpstr .= ",romfile=$romfile" if $romfile;
2034 sub print_netdev_full
{
2035 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2038 if ($netid =~ m/^net(\d+)$/) {
2042 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
2044 my $ifname = "tap${vmid}i$i";
2046 # kvm uses TUNSETIFF ioctl, and that limits ifname length
2047 die "interface name '$ifname' is too long (max 15 character)\n"
2048 if length($ifname) >= 16;
2050 my $vhostparam = '';
2051 if (is_native
($arch)) {
2052 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
2055 my $vmname = $conf->{name
} || "vm$vmid";
2058 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
2060 if ($net->{bridge
}) {
2061 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2063 $netdev = "type=user,id=$netid,hostname=$vmname";
2066 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
2072 sub print_cpu_device
{
2073 my ($conf, $id) = @_;
2075 my $kvm = $conf->{kvm
} // 1;
2076 my $cpu = $kvm ?
"kvm64" : "qemu64";
2077 if (my $cputype = $conf->{cpu
}) {
2078 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
2079 or die "Cannot parse cpu description: $cputype\n";
2080 $cpu = $cpuconf->{cputype
};
2083 my $cores = $conf->{cores
} || 1;
2085 my $current_core = ($id - 1) % $cores;
2086 my $current_socket = int(($id - 1 - $current_core)/$cores);
2088 return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2092 'cirrus' => 'cirrus-vga',
2094 'vmware' => 'vmware-svga',
2095 'virtio' => 'virtio-vga',
2098 sub print_vga_device
{
2099 my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2101 my $type = $vga_map->{$vga->{type
}};
2102 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2103 $type = 'virtio-gpu';
2105 my $vgamem_mb = $vga->{memory
};
2107 $type = $id ?
'qxl' : 'qxl-vga';
2109 die "no devicetype for $vga->{type}\n" if !$type;
2113 if ($vga->{type
} eq 'virtio') {
2114 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
2115 $memory = ",max_hostmem=$bytes";
2117 # from https://www.spice-space.org/multiple-monitors.html
2118 $memory = ",vgamem_mb=$vga->{memory}";
2119 my $ram = $vgamem_mb * 4;
2120 my $vram = $vgamem_mb * 2;
2121 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2123 $memory = ",vgamem_mb=$vga->{memory}";
2125 } elsif ($qxlnum && $id) {
2126 $memory = ",ram_size=67108864,vram_size=33554432";
2129 my $q35 = machine_type_is_q35
($conf);
2130 my $vgaid = "vga" . ($id // '');
2133 if ($q35 && $vgaid eq 'vga') {
2134 # the first display uses pcie.0 bus on q35 machines
2135 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
2137 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
2140 return "$type,id=${vgaid}${memory}${pciaddr}";
2143 sub drive_is_cloudinit
{
2145 return $drive->{file
} =~ m
@[:/]vm-\d
+-cloudinit
(?
:\
.$QEMU_FORMAT_RE)?
$@;
2148 sub drive_is_cdrom
{
2149 my ($drive, $exclude_cloudinit) = @_;
2151 return 0 if $exclude_cloudinit && drive_is_cloudinit
($drive);
2153 return $drive && $drive->{media
} && ($drive->{media
} eq 'cdrom');
2157 sub parse_number_sets
{
2160 foreach my $part (split(/;/, $set)) {
2161 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2162 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2163 push @$res, [ $1, $2 ];
2165 die "invalid range: $part\n";
2174 my $res = PVE
::JSONSchema
::parse_property_string
($numa_fmt, $data);
2175 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
2176 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
2183 return undef if !$value;
2185 my $res = PVE
::JSONSchema
::parse_property_string
($hostpci_fmt, $value);
2187 my @idlist = split(/;/, $res->{host
});
2188 delete $res->{host
};
2189 foreach my $id (@idlist) {
2190 if ($id =~ m/\./) { # full id 00:00.1
2191 push @{$res->{pciid
}}, {
2194 } else { # partial id 00:00
2195 $res->{pciid
} = PVE
::SysFSTools
::lspci
($id);
2201 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2205 my $res = eval { PVE
::JSONSchema
::parse_property_string
($net_fmt, $data) };
2210 if (!defined($res->{macaddr
})) {
2211 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
2212 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
2217 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2218 sub parse_ipconfig
{
2221 my $res = eval { PVE
::JSONSchema
::parse_property_string
($ipconfig_fmt, $data) };
2227 if ($res->{gw
} && !$res->{ip
}) {
2228 warn 'gateway specified without specifying an IP address';
2231 if ($res->{gw6
} && !$res->{ip6
}) {
2232 warn 'IPv6 gateway specified without specifying an IPv6 address';
2235 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
2236 warn 'gateway specified together with DHCP';
2239 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
2241 warn "IPv6 gateway specified together with $res->{ip6} address";
2245 if (!$res->{ip
} && !$res->{ip6
}) {
2246 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
2255 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
2258 sub add_random_macs
{
2259 my ($settings) = @_;
2261 foreach my $opt (keys %$settings) {
2262 next if $opt !~ m/^net(\d+)$/;
2263 my $net = parse_net
($settings->{$opt});
2265 $settings->{$opt} = print_net
($net);
2269 sub vm_is_volid_owner
{
2270 my ($storecfg, $vmid, $volid) = @_;
2272 if ($volid !~ m
|^/|) {
2274 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
2275 if ($owner && ($owner == $vmid)) {
2283 sub split_flagged_list
{
2284 my $text = shift || '';
2285 $text =~ s/[,;]/ /g;
2287 return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2290 sub join_flagged_list
{
2291 my ($how, $lst) = @_;
2292 join $how, map { $lst->{$_} . $_ } keys %$lst;
2295 sub vmconfig_delete_pending_option
{
2296 my ($conf, $key, $force) = @_;
2298 delete $conf->{pending
}->{$key};
2299 my $pending_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
2300 $pending_delete_hash->{$key} = $force ?
'!' : '';
2301 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
2304 sub vmconfig_undelete_pending_option
{
2305 my ($conf, $key) = @_;
2307 my $pending_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
2308 delete $pending_delete_hash->{$key};
2310 if (%$pending_delete_hash) {
2311 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
2313 delete $conf->{pending
}->{delete};
2317 sub vmconfig_register_unused_drive
{
2318 my ($storecfg, $vmid, $conf, $drive) = @_;
2320 if (drive_is_cloudinit
($drive)) {
2321 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2323 } elsif (!drive_is_cdrom
($drive)) {
2324 my $volid = $drive->{file
};
2325 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2326 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2331 sub vmconfig_cleanup_pending
{
2334 # remove pending changes when nothing changed
2336 foreach my $opt (keys %{$conf->{pending
}}) {
2337 if (defined($conf->{$opt}) && ($conf->{pending
}->{$opt} eq $conf->{$opt})) {
2339 delete $conf->{pending
}->{$opt};
2343 my $current_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
2344 my $pending_delete_hash = {};
2345 while (my ($opt, $force) = each %$current_delete_hash) {
2346 if (defined($conf->{$opt})) {
2347 $pending_delete_hash->{$opt} = $force;
2353 if (%$pending_delete_hash) {
2354 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
2356 delete $conf->{pending
}->{delete};
2362 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2366 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2367 format_description
=> 'UUID',
2368 description
=> "Set SMBIOS1 UUID.",
2373 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2374 format_description
=> 'Base64 encoded string',
2375 description
=> "Set SMBIOS1 version.",
2380 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2381 format_description
=> 'Base64 encoded string',
2382 description
=> "Set SMBIOS1 serial number.",
2387 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2388 format_description
=> 'Base64 encoded string',
2389 description
=> "Set SMBIOS1 manufacturer.",
2394 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2395 format_description
=> 'Base64 encoded string',
2396 description
=> "Set SMBIOS1 product ID.",
2401 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2402 format_description
=> 'Base64 encoded string',
2403 description
=> "Set SMBIOS1 SKU string.",
2408 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2409 format_description
=> 'Base64 encoded string',
2410 description
=> "Set SMBIOS1 family string.",
2415 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2423 my $res = eval { PVE
::JSONSchema
::parse_property_string
($smbios1_fmt, $data) };
2430 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2433 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2435 PVE
::JSONSchema
::register_format
('pve-qm-bootdisk', \
&verify_bootdisk
);
2436 sub verify_bootdisk
{
2437 my ($value, $noerr) = @_;
2439 return $value if is_valid_drivename
($value);
2441 return undef if $noerr;
2443 die "invalid boot disk '$value'\n";
2446 sub parse_watchdog
{
2449 return undef if !$value;
2451 my $res = eval { PVE
::JSONSchema
::parse_property_string
($watchdog_fmt, $value) };
2456 sub parse_guest_agent
{
2459 return {} if !defined($value->{agent
});
2461 my $res = eval { PVE
::JSONSchema
::parse_property_string
($agent_fmt, $value->{agent
}) };
2464 # if the agent is disabled ignore the other potentially set properties
2465 return {} if !$res->{enabled
};
2472 return {} if !$value;
2473 my $res = eval { PVE
::JSONSchema
::parse_property_string
($vga_fmt, $value) };
2478 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2479 sub verify_usb_device
{
2480 my ($value, $noerr) = @_;
2482 return $value if parse_usb_device
($value);
2484 return undef if $noerr;
2486 die "unable to parse usb device\n";
2489 # add JSON properties for create and set function
2490 sub json_config_properties
{
2493 foreach my $opt (keys %$confdesc) {
2494 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2495 $prop->{$opt} = $confdesc->{$opt};
2501 # return copy of $confdesc_cloudinit to generate documentation
2502 sub cloudinit_config_properties
{
2504 return dclone
($confdesc_cloudinit);
2508 my ($key, $value) = @_;
2510 die "unknown setting '$key'\n" if !$confdesc->{$key};
2512 my $type = $confdesc->{$key}->{type
};
2514 if (!defined($value)) {
2515 die "got undefined value\n";
2518 if ($value =~ m/[\n\r]/) {
2519 die "property contains a line feed\n";
2522 if ($type eq 'boolean') {
2523 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2524 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2525 die "type check ('boolean') failed - got '$value'\n";
2526 } elsif ($type eq 'integer') {
2527 return int($1) if $value =~ m/^(\d+)$/;
2528 die "type check ('integer') failed - got '$value'\n";
2529 } elsif ($type eq 'number') {
2530 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2531 die "type check ('number') failed - got '$value'\n";
2532 } elsif ($type eq 'string') {
2533 if (my $fmt = $confdesc->{$key}->{format
}) {
2534 PVE
::JSONSchema
::check_format
($fmt, $value);
2537 $value =~ s/^\"(.*)\"$/$1/;
2540 die "internal error"
2547 my $conf = PVE
::QemuConfig-
>config_file($vmid);
2548 utime undef, undef, $conf;
2552 my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2554 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
2556 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2558 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2560 if ($conf->{template
}) {
2561 # check if any base image is still used by a linked clone
2562 foreach_drive
($conf, sub {
2563 my ($ds, $drive) = @_;
2565 return if drive_is_cdrom
($drive);
2567 my $volid = $drive->{file
};
2569 return if !$volid || $volid =~ m
|^/|;
2571 die "base volume '$volid' is still in use by linked cloned\n"
2572 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2577 # only remove disks owned by this VM
2578 foreach_drive
($conf, sub {
2579 my ($ds, $drive) = @_;
2581 return if drive_is_cdrom
($drive, 1);
2583 my $volid = $drive->{file
};
2585 return if !$volid || $volid =~ m
|^/|;
2587 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2588 return if !$path || !$owner || ($owner != $vmid);
2591 PVE
::Storage
::vdisk_free
($storecfg, $volid);
2593 warn "Could not remove disk '$volid', check manually: $@" if $@;
2597 if ($keep_empty_config) {
2598 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
2603 # also remove unused disk
2605 my $dl = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2608 PVE
::Storage
::foreach_volid
($dl, sub {
2609 my ($volid, $sid, $volname, $d) = @_;
2610 PVE
::Storage
::vdisk_free
($storecfg, $volid);
2619 sub parse_vm_config
{
2620 my ($filename, $raw) = @_;
2622 return undef if !defined($raw);
2625 digest
=> Digest
::SHA
::sha1_hex
($raw),
2630 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2631 || die "got strange filename '$filename'";
2639 my @lines = split(/\n/, $raw);
2640 foreach my $line (@lines) {
2641 next if $line =~ m/^\s*$/;
2643 if ($line =~ m/^\[PENDING\]\s*$/i) {
2644 $section = 'pending';
2645 if (defined($descr)) {
2647 $conf->{description
} = $descr;
2650 $conf = $res->{$section} = {};
2653 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2655 if (defined($descr)) {
2657 $conf->{description
} = $descr;
2660 $conf = $res->{snapshots
}->{$section} = {};
2664 if ($line =~ m/^\#(.*)\s*$/) {
2665 $descr = '' if !defined($descr);
2666 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2670 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2671 $descr = '' if !defined($descr);
2672 $descr .= PVE
::Tools
::decode_text
($2);
2673 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2674 $conf->{snapstate
} = $1;
2675 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2678 $conf->{$key} = $value;
2679 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2681 if ($section eq 'pending') {
2682 $conf->{delete} = $value; # we parse this later
2684 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2686 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2689 eval { $value = check_type
($key, $value); };
2691 warn "vm $vmid - unable to parse value of '$key' - $@";
2693 $key = 'ide2' if $key eq 'cdrom';
2694 my $fmt = $confdesc->{$key}->{format
};
2695 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2696 my $v = parse_drive
($key, $value);
2697 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2698 $v->{file
} = $volid;
2699 $value = print_drive
($vmid, $v);
2701 warn "vm $vmid - unable to parse value of '$key'\n";
2706 $conf->{$key} = $value;
2711 if (defined($descr)) {
2713 $conf->{description
} = $descr;
2715 delete $res->{snapstate
}; # just to be sure
2720 sub write_vm_config
{
2721 my ($filename, $conf) = @_;
2723 delete $conf->{snapstate
}; # just to be sure
2725 if ($conf->{cdrom
}) {
2726 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2727 $conf->{ide2
} = $conf->{cdrom
};
2728 delete $conf->{cdrom
};
2731 # we do not use 'smp' any longer
2732 if ($conf->{sockets
}) {
2733 delete $conf->{smp
};
2734 } elsif ($conf->{smp
}) {
2735 $conf->{sockets
} = $conf->{smp
};
2736 delete $conf->{cores
};
2737 delete $conf->{smp
};
2740 my $used_volids = {};
2742 my $cleanup_config = sub {
2743 my ($cref, $pending, $snapname) = @_;
2745 foreach my $key (keys %$cref) {
2746 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2747 $key eq 'snapstate' || $key eq 'pending';
2748 my $value = $cref->{$key};
2749 if ($key eq 'delete') {
2750 die "propertry 'delete' is only allowed in [PENDING]\n"
2752 # fixme: check syntax?
2755 eval { $value = check_type
($key, $value); };
2756 die "unable to parse value of '$key' - $@" if $@;
2758 $cref->{$key} = $value;
2760 if (!$snapname && is_valid_drivename
($key)) {
2761 my $drive = parse_drive
($key, $value);
2762 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2767 &$cleanup_config($conf);
2769 &$cleanup_config($conf->{pending
}, 1);
2771 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2772 die "internal error" if $snapname eq 'pending';
2773 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2776 # remove 'unusedX' settings if we re-add a volume
2777 foreach my $key (keys %$conf) {
2778 my $value = $conf->{$key};
2779 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2780 delete $conf->{$key};
2784 my $generate_raw_config = sub {
2785 my ($conf, $pending) = @_;
2789 # add description as comment to top of file
2790 if (defined(my $descr = $conf->{description
})) {
2792 foreach my $cl (split(/\n/, $descr)) {
2793 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2796 $raw .= "#\n" if $pending;
2800 foreach my $key (sort keys %$conf) {
2801 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2802 $raw .= "$key: $conf->{$key}\n";
2807 my $raw = &$generate_raw_config($conf);
2809 if (scalar(keys %{$conf->{pending
}})){
2810 $raw .= "\n[PENDING]\n";
2811 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2814 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2815 $raw .= "\n[$snapname]\n";
2816 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2826 # we use static defaults from our JSON schema configuration
2827 foreach my $key (keys %$confdesc) {
2828 if (defined(my $default = $confdesc->{$key}->{default})) {
2829 $res->{$key} = $default;
2837 my $vmlist = PVE
::Cluster
::get_vmlist
();
2839 return $res if !$vmlist || !$vmlist->{ids
};
2840 my $ids = $vmlist->{ids
};
2842 foreach my $vmid (keys %$ids) {
2843 my $d = $ids->{$vmid};
2844 next if !$d->{node
} || $d->{node
} ne $nodename;
2845 next if !$d->{type
} || $d->{type
} ne 'qemu';
2846 $res->{$vmid}->{exists} = 1;
2851 # test if VM uses local resources (to prevent migration)
2852 sub check_local_resources
{
2853 my ($conf, $noerr) = @_;
2857 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2858 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2860 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2862 foreach my $k (keys %$conf) {
2863 next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2864 # sockets are safe: they will recreated be on the target side post-migrate
2865 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2866 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2869 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2874 # check if used storages are available on all nodes (use by migrate)
2875 sub check_storage_availability
{
2876 my ($storecfg, $conf, $node) = @_;
2878 foreach_drive
($conf, sub {
2879 my ($ds, $drive) = @_;
2881 my $volid = $drive->{file
};
2884 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2887 # check if storage is available on both nodes
2888 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2889 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2893 # list nodes where all VM images are available (used by has_feature API)
2895 my ($conf, $storecfg) = @_;
2897 my $nodelist = PVE
::Cluster
::get_nodelist
();
2898 my $nodehash = { map { $_ => 1 } @$nodelist };
2899 my $nodename = PVE
::INotify
::nodename
();
2901 foreach_drive
($conf, sub {
2902 my ($ds, $drive) = @_;
2904 my $volid = $drive->{file
};
2907 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2909 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2910 if ($scfg->{disable
}) {
2912 } elsif (my $avail = $scfg->{nodes
}) {
2913 foreach my $node (keys %$nodehash) {
2914 delete $nodehash->{$node} if !$avail->{$node};
2916 } elsif (!$scfg->{shared
}) {
2917 foreach my $node (keys %$nodehash) {
2918 delete $nodehash->{$node} if $node ne $nodename
2928 my ($pidfile, $pid) = @_;
2930 my $fh = IO
::File-
>new("/proc/$pid/cmdline", "r");
2934 return undef if !$line;
2935 my @param = split(/\0/, $line);
2937 my $cmd = $param[0];
2938 return if !$cmd || ($cmd !~ m
|kvm
$| && $cmd !~ m
@(?
:^|/)qemu-system-[^/]+$@);
2940 for (my $i = 0; $i < scalar (@param); $i++) {
2943 if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2944 my $p = $param[$i+1];
2945 return 1 if $p && ($p eq $pidfile);
2954 my ($vmid, $nocheck, $node) = @_;
2956 my $filename = PVE
::QemuConfig-
>config_file($vmid, $node);
2958 die "unable to find configuration file for VM $vmid - no such machine\n"
2959 if !$nocheck && ! -f
$filename;
2961 my $pidfile = pidfile_name
($vmid);
2963 if (my $fd = IO
::File-
>new("<$pidfile")) {
2968 my $mtime = $st->mtime;
2969 if ($mtime > time()) {
2970 warn "file '$filename' modified in future\n";
2973 if ($line =~ m/^(\d+)$/) {
2975 if (check_cmdline
($pidfile, $pid)) {
2976 if (my $pinfo = PVE
::ProcFSTools
::check_process_running
($pid)) {
2988 my $vzlist = config_list
();
2990 my $fd = IO
::Dir-
>new($var_run_tmpdir) || return $vzlist;
2992 while (defined(my $de = $fd->read)) {
2993 next if $de !~ m/^(\d+)\.pid$/;
2995 next if !defined($vzlist->{$vmid});
2996 if (my $pid = check_running
($vmid)) {
2997 $vzlist->{$vmid}->{pid
} = $pid;
3005 my ($storecfg, $conf) = @_;
3007 my $bootdisk = $conf->{bootdisk
};
3008 return undef if !$bootdisk;
3009 return undef if !is_valid_drivename
($bootdisk);
3011 return undef if !$conf->{$bootdisk};
3013 my $drive = parse_drive
($bootdisk, $conf->{$bootdisk});
3014 return undef if !defined($drive);
3016 return undef if drive_is_cdrom
($drive);
3018 my $volid = $drive->{file
};
3019 return undef if !$volid;
3021 return $drive->{size
};
3024 our $vmstatus_return_properties = {
3025 vmid
=> get_standard_option
('pve-vmid'),
3027 description
=> "Qemu process status.",
3029 enum
=> ['stopped', 'running'],
3032 description
=> "Maximum memory in bytes.",
3035 renderer
=> 'bytes',
3038 description
=> "Root disk size in bytes.",
3041 renderer
=> 'bytes',
3044 description
=> "VM name.",
3049 description
=> "Qemu QMP agent status.",
3054 description
=> "PID of running qemu process.",
3059 description
=> "Uptime.",
3062 renderer
=> 'duration',
3065 description
=> "Maximum usable CPUs.",
3070 description
=> "The current config lock, if any.",
3076 my $last_proc_pid_stat;
3078 # get VM status information
3079 # This must be fast and should not block ($full == false)
3080 # We only query KVM using QMP if $full == true (this can be slow)
3082 my ($opt_vmid, $full) = @_;
3086 my $storecfg = PVE
::Storage
::config
();
3088 my $list = vzlist
();
3089 my $defaults = load_defaults
();
3091 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
3093 my $cpucount = $cpuinfo->{cpus
} || 1;
3095 foreach my $vmid (keys %$list) {
3096 next if $opt_vmid && ($vmid ne $opt_vmid);
3098 my $cfspath = PVE
::QemuConfig-
>cfs_config_path($vmid);
3099 my $conf = PVE
::Cluster
::cfs_read_file
($cfspath) || {};
3101 my $d = { vmid
=> $vmid };
3102 $d->{pid
} = $list->{$vmid}->{pid
};
3104 # fixme: better status?
3105 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
3107 my $size = disksize
($storecfg, $conf);
3108 if (defined($size)) {
3109 $d->{disk
} = 0; # no info available
3110 $d->{maxdisk
} = $size;
3116 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
3117 * ($conf->{cores
} || $defaults->{cores
});
3118 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
3119 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
3121 $d->{name
} = $conf->{name
} || "VM $vmid";
3122 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
3123 : $defaults->{memory
}*(1024*1024);
3125 if ($conf->{balloon
}) {
3126 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
3127 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
3128 : $defaults->{shares
};
3139 $d->{diskwrite
} = 0;
3141 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
3143 $d->{serial
} = 1 if conf_has_serial
($conf);
3144 $d->{lock} = $conf->{lock} if $conf->{lock};
3149 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
3150 foreach my $dev (keys %$netdev) {
3151 next if $dev !~ m/^tap([1-9]\d*)i/;
3153 my $d = $res->{$vmid};
3156 $d->{netout
} += $netdev->{$dev}->{receive
};
3157 $d->{netin
} += $netdev->{$dev}->{transmit
};
3160 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
3161 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
3166 my $ctime = gettimeofday
;
3168 foreach my $vmid (keys %$list) {
3170 my $d = $res->{$vmid};
3171 my $pid = $d->{pid
};
3174 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
3175 next if !$pstat; # not running
3177 my $used = $pstat->{utime} + $pstat->{stime
};
3179 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
3181 if ($pstat->{vsize
}) {
3182 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
3185 my $old = $last_proc_pid_stat->{$pid};
3187 $last_proc_pid_stat->{$pid} = {
3195 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
3197 if ($dtime > 1000) {
3198 my $dutime = $used - $old->{used
};
3200 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
3201 $last_proc_pid_stat->{$pid} = {
3207 $d->{cpu
} = $old->{cpu
};
3211 return $res if !$full;
3213 my $qmpclient = PVE
::QMPClient-
>new();
3215 my $ballooncb = sub {
3216 my ($vmid, $resp) = @_;
3218 my $info = $resp->{'return'};
3219 return if !$info->{max_mem
};
3221 my $d = $res->{$vmid};
3223 # use memory assigned to VM
3224 $d->{maxmem
} = $info->{max_mem
};
3225 $d->{balloon
} = $info->{actual
};
3227 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
3228 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
3229 $d->{freemem
} = $info->{free_mem
};
3232 $d->{ballooninfo
} = $info;
3235 my $blockstatscb = sub {
3236 my ($vmid, $resp) = @_;
3237 my $data = $resp->{'return'} || [];
3238 my $totalrdbytes = 0;
3239 my $totalwrbytes = 0;
3241 for my $blockstat (@$data) {
3242 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
3243 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
3245 $blockstat->{device
} =~ s/drive-//;
3246 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
3248 $res->{$vmid}->{diskread
} = $totalrdbytes;
3249 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
3252 my $statuscb = sub {
3253 my ($vmid, $resp) = @_;
3255 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3256 # this fails if ballon driver is not loaded, so this must be
3257 # the last commnand (following command are aborted if this fails).
3258 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3260 my $status = 'unknown';
3261 if (!defined($status = $resp->{'return'}->{status
})) {
3262 warn "unable to get VM status\n";
3266 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3269 foreach my $vmid (keys %$list) {
3270 next if $opt_vmid && ($vmid ne $opt_vmid);
3271 next if !$res->{$vmid}->{pid
}; # not running
3272 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3275 $qmpclient->queue_execute(undef, 2);
3277 foreach my $vmid (keys %$list) {
3278 next if $opt_vmid && ($vmid ne $opt_vmid);
3279 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3286 my ($conf, $func, @param) = @_;
3288 foreach my $ds (valid_drive_names
()) {
3289 next if !defined($conf->{$ds});
3291 my $drive = parse_drive
($ds, $conf->{$ds});
3294 &$func($ds, $drive, @param);
3299 my ($conf, $func, @param) = @_;
3303 my $test_volid = sub {
3304 my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3308 $volhash->{$volid}->{cdrom
} //= 1;
3309 $volhash->{$volid}->{cdrom
} = 0 if !$is_cdrom;
3311 $volhash->{$volid}->{replicate
} //= 0;
3312 $volhash->{$volid}->{replicate
} = 1 if $replicate;
3314 $volhash->{$volid}->{shared
} //= 0;
3315 $volhash->{$volid}->{shared
} = 1 if $shared;
3317 $volhash->{$volid}->{referenced_in_config
} //= 0;
3318 $volhash->{$volid}->{referenced_in_config
} = 1 if !defined($snapname);
3320 $volhash->{$volid}->{referenced_in_snapshot
}->{$snapname} = 1
3321 if defined($snapname);
3324 foreach_drive
($conf, sub {
3325 my ($ds, $drive) = @_;
3326 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, undef);
3329 foreach my $snapname (keys %{$conf->{snapshots
}}) {
3330 my $snap = $conf->{snapshots
}->{$snapname};
3331 $test_volid->($snap->{vmstate
}, 0, 1, $snapname);
3332 foreach_drive
($snap, sub {
3333 my ($ds, $drive) = @_;
3334 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, $snapname);
3338 foreach my $volid (keys %$volhash) {
3339 &$func($volid, $volhash->{$volid}, @param);
3343 sub conf_has_serial
{
3346 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3347 if ($conf->{"serial$i"}) {
3355 sub vga_conf_has_spice
{
3358 my $vgaconf = parse_vga
($vga);
3359 my $vgatype = $vgaconf->{type
};
3360 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3365 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3366 sub get_host_arch
() {
3367 $host_arch = (POSIX
::uname
())[4] if !$host_arch;
3373 return get_host_arch
() eq $arch;
3376 my $default_machines = {
3381 sub get_basic_machine_info
{
3382 my ($conf, $forcemachine) = @_;
3384 my $arch = $conf->{arch
} // get_host_arch
();
3385 my $machine = $forcemachine || $conf->{machine
} || $default_machines->{$arch};
3386 return ($arch, $machine);
3389 sub get_ovmf_files
($) {
3392 my $ovmf = $OVMF->{$arch}
3393 or die "no OVMF images known for architecture '$arch'\n";
3399 aarch64
=> '/usr/bin/qemu-system-aarch64',
3400 x86_64
=> '/usr/bin/qemu-system-x86_64',
3402 sub get_command_for_arch
($) {
3404 return '/usr/bin/kvm' if is_native
($arch);
3406 my $cmd = $Arch2Qemu->{$arch}
3407 or die "don't know how to emulate architecture '$arch'\n";
3411 sub get_cpu_options
{
3412 my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3415 my $ostype = $conf->{ostype
};
3417 my $cpu = $kvm ?
"kvm64" : "qemu64";
3418 if ($arch eq 'aarch64') {
3419 $cpu = 'cortex-a57';
3422 if (my $cputype = $conf->{cpu
}) {
3423 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
3424 or die "Cannot parse cpu description: $cputype\n";
3425 $cpu = $cpuconf->{cputype
};
3426 $kvm_off = 1 if $cpuconf->{hidden
};
3427 $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3429 if (defined(my $flags = $cpuconf->{flags
})) {
3430 push @$cpuFlags, split(";", $flags);
3434 push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3436 push @$cpuFlags , '-x2apic'
3437 if $conf->{ostype
} && $conf->{ostype
} eq 'solaris';
3439 push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3441 push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3443 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3445 push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3446 push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3449 add_hyperv_enlightenments
($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios
}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3451 push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3453 push @$cpuFlags, 'kvm=off' if $kvm_off;
3455 if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3456 push @$cpuFlags, "vendor=${cpu_vendor}"
3457 if $cpu_vendor ne 'default';
3458 } elsif ($arch ne 'aarch64') {
3459 die "internal error"; # should not happen
3462 $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3464 return ('-cpu', $cpu);
3467 sub config_to_command
{
3468 my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3471 my $globalFlags = [];
3472 my $machineFlags = [];
3477 my $kvmver = kvm_user_version
();
3478 my $vernum = 0; # unknown
3479 my $ostype = $conf->{ostype
};
3480 my $winversion = windows_version
($ostype);
3481 my $kvm = $conf->{kvm
};
3483 my ($arch, $machine_type) = get_basic_machine_info
($conf, $forcemachine);
3484 $kvm //= 1 if is_native
($arch);
3487 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3488 if !defined kvm_version
();
3491 if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3492 $vernum = $1*1000000+$2*1000;
3493 } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3494 $vernum = $1*1000000+$2*1000+$3;
3497 die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3499 my $have_ovz = -f
'/proc/vz/vestat';
3501 my $q35 = machine_type_is_q35
($conf);
3502 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3503 my $use_old_bios_files = undef;
3504 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3506 my $cpuunits = defined($conf->{cpuunits
}) ?
3507 $conf->{cpuunits
} : $defaults->{cpuunits
};
3509 push @$cmd, get_command_for_arch
($arch);
3511 push @$cmd, '-id', $vmid;
3513 my $vmname = $conf->{name
} || "vm$vmid";
3515 push @$cmd, '-name', $vmname;
3519 my $qmpsocket = qmp_socket
($vmid);
3520 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3521 push @$cmd, '-mon', "chardev=qmp,mode=control";
3523 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 12)) {
3524 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3525 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3528 push @$cmd, '-pidfile' , pidfile_name
($vmid);
3530 push @$cmd, '-daemonize';
3532 if ($conf->{smbios1
}) {
3533 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3534 if ($smbios_conf->{base64
}) {
3535 # Do not pass base64 flag to qemu
3536 delete $smbios_conf->{base64
};
3537 my $smbios_string = "";
3538 foreach my $key (keys %$smbios_conf) {
3540 if ($key eq "uuid") {
3541 $value = $smbios_conf->{uuid
}
3543 $value = decode_base64
($smbios_conf->{$key});
3545 # qemu accepts any binary data, only commas need escaping by double comma
3547 $smbios_string .= "," . $key . "=" . $value if $value;
3549 push @$cmd, '-smbios', "type=1" . $smbios_string;
3551 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3555 if ($conf->{vmgenid
}) {
3556 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3559 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3560 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3561 die "uefi base image not found\n" if ! -f
$ovmf_code;
3565 if (my $efidisk = $conf->{efidisk0
}) {
3566 my $d = PVE
::JSONSchema
::parse_property_string
($efidisk_fmt, $efidisk);
3567 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3568 $format = $d->{format
};
3570 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3571 if (!defined($format)) {
3572 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3573 $format = qemu_img_format
($scfg, $volname);
3577 die "efidisk format must be specified\n"
3578 if !defined($format);
3581 warn "no efidisk configured! Using temporary efivars disk.\n";
3582 $path = "/tmp/$vmid-ovmf.fd";
3583 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3587 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3588 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3592 # add usb controllers
3593 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
($conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3594 push @$devices, @usbcontrollers if @usbcontrollers;
3595 my $vga = parse_vga
($conf->{vga
});
3597 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3598 $vga->{type
} = 'qxl' if $qxlnum;
3600 if (!$vga->{type
}) {
3601 if ($arch eq 'aarch64') {
3602 $vga->{type
} = 'virtio';
3603 } elsif (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 9)) {
3604 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3606 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3610 # enable absolute mouse coordinates (needed by vnc)
3612 if (defined($conf->{tablet
})) {
3613 $tablet = $conf->{tablet
};
3615 $tablet = $defaults->{tablet
};
3616 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3617 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3621 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3622 my $kbd = print_keyboarddevice_full
($conf, $arch);
3623 push @$devices, '-device', $kbd if defined($kbd);
3627 my $gpu_passthrough;
3630 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
3631 my $d = parse_hostpci
($conf->{"hostpci$i"});
3634 my $pcie = $d->{pcie
};
3636 die "q35 machine model is not enabled" if !$q35;
3637 # win7 wants to have the pcie devices directly on the pcie bus
3638 # instead of in the root port
3639 if ($winversion == 7) {
3640 $pciaddr = print_pcie_addr
("hostpci${i}bus0");
3642 $pciaddr = print_pcie_addr
("hostpci$i");
3645 $pciaddr = print_pci_addr
("hostpci$i", $bridges, $arch, $machine_type);
3648 my $rombar = defined($d->{rombar
}) && !$d->{rombar
} ?
',rombar=0' : '';
3649 my $romfile = $d->{romfile
};
3652 if ($d->{'x-vga'}) {
3653 $xvga = ',x-vga=on';
3655 $vga->{type
} = 'none' if !defined($conf->{vga
});
3656 $gpu_passthrough = 1;
3658 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3662 my $pcidevices = $d->{pciid
};
3663 my $multifunction = 1 if @$pcidevices > 1;
3665 if ($d->{mdev
} && scalar(@$pcidevices) == 1) {
3666 my $id = $pcidevices->[0]->{id
};
3667 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $i);
3668 $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3669 } elsif ($d->{mdev
}) {
3670 warn "ignoring mediated device with multifunction device\n";
3674 foreach my $pcidevice (@$pcidevices) {
3676 my $id = "hostpci$i";
3677 $id .= ".$j" if $multifunction;
3678 my $addr = $pciaddr;
3679 $addr .= ".$j" if $multifunction;
3680 my $devicestr = "vfio-pci";
3682 $devicestr .= ",sysfsdev=$sysfspath";
3684 $devicestr .= ",host=$pcidevice->{id}";
3686 $devicestr .= ",id=$id$addr";
3689 $devicestr .= "$rombar$xvga";
3690 $devicestr .= ",multifunction=on" if $multifunction;
3691 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3694 push @$devices, '-device', $devicestr;
3700 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
($conf, $usbdesc->{format
}, $MAX_USB_DEVICES);
3701 push @$devices, @usbdevices if @usbdevices;
3703 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3704 if (my $path = $conf->{"serial$i"}) {
3705 if ($path eq 'socket') {
3706 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3707 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3708 # On aarch64, serial0 is the UART device. Qemu only allows
3709 # connecting UART devices via the '-serial' command line, as
3710 # the device has a fixed slot on the hardware...
3711 if ($arch eq 'aarch64' && $i == 0) {
3712 push @$devices, '-serial', "chardev:serial$i";
3714 push @$devices, '-device', "isa-serial,chardev=serial$i";
3717 die "no such serial device\n" if ! -c
$path;
3718 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3719 push @$devices, '-device', "isa-serial,chardev=serial$i";
3725 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3726 if (my $path = $conf->{"parallel$i"}) {
3727 die "no such parallel device\n" if ! -c
$path;
3728 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3729 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3730 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3736 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3737 $sockets = $conf->{sockets
} if $conf->{sockets
};
3739 my $cores = $conf->{cores
} || 1;
3741 my $maxcpus = $sockets * $cores;
3743 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3745 my $allowed_vcpus = $cpuinfo->{cpus
};
3747 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3748 if ($allowed_vcpus < $maxcpus);
3750 if($hotplug_features->{cpu
} && qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 7)) {
3752 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3753 for (my $i = 2; $i <= $vcpus; $i++) {
3754 my $cpustr = print_cpu_device
($conf,$i);
3755 push @$cmd, '-device', $cpustr;
3760 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3762 push @$cmd, '-nodefaults';
3764 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
3766 my $bootindex_hash = {};
3768 foreach my $o (split(//, $bootorder)) {
3769 $bootindex_hash->{$o} = $i*100;
3773 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3775 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3777 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3779 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3780 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3781 my $socket = vnc_socket
($vmid);
3782 push @$cmd, '-vnc', "unix:$socket,password";
3784 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3785 push @$cmd, '-nographic';
3789 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3791 my $useLocaltime = $conf->{localtime};
3793 if ($winversion >= 5) { # windows
3794 $useLocaltime = 1 if !defined($conf->{localtime});
3796 # use time drift fix when acpi is enabled
3797 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3798 $tdf = 1 if !defined($conf->{tdf
});
3802 if ($winversion >= 6) {
3803 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3804 push @$cmd, '-no-hpet';
3807 push @$rtcFlags, 'driftfix=slew' if $tdf;
3810 push @$machineFlags, 'accel=tcg';
3813 if ($machine_type) {
3814 push @$machineFlags, "type=${machine_type}";
3817 if ($conf->{startdate
}) {
3818 push @$rtcFlags, "base=$conf->{startdate}";
3819 } elsif ($useLocaltime) {
3820 push @$rtcFlags, 'base=localtime';
3823 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3825 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3827 push @$cmd, '-S' if $conf->{freeze
};
3829 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3832 #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3833 #push @$cmd, '-soundhw', 'es1370';
3834 #push @$cmd, '-soundhw', $soundhw if $soundhw;
3836 if (parse_guest_agent
($conf)->{enabled
}) {
3837 my $qgasocket = qmp_socket
($vmid, 1);
3838 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3839 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3840 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3841 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3849 for(my $i = 1; $i < $qxlnum; $i++){
3850 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3853 # assume other OS works like Linux
3854 my ($ram, $vram) = ("134217728", "67108864");
3855 if ($vga->{memory
}) {
3856 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3857 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3859 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3860 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3864 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3866 my $nodename = PVE
::INotify
::nodename
();
3867 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3868 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3869 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3870 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3871 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3873 push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3875 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3876 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3877 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3880 # enable balloon by default, unless explicitly disabled
3881 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3882 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3883 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3886 if ($conf->{watchdog
}) {
3887 my $wdopts = parse_watchdog
($conf->{watchdog
});
3888 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3889 my $watchdog = $wdopts->{model
} || 'i6300esb';
3890 push @$devices, '-device', "$watchdog$pciaddr";
3891 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3895 my $scsicontroller = {};
3896 my $ahcicontroller = {};
3897 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3899 # Add iscsi initiator name if available
3900 if (my $initiator = get_initiator_name
()) {
3901 push @$devices, '-iscsi', "initiator-name=$initiator";
3904 foreach_drive
($conf, sub {
3905 my ($ds, $drive) = @_;
3907 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3908 push @$vollist, $drive->{file
};
3911 # ignore efidisk here, already added in bios/fw handling code above
3912 return if $drive->{interface
} eq 'efidisk';
3914 $use_virtio = 1 if $ds =~ m/^virtio/;
3916 if (drive_is_cdrom
($drive)) {
3917 if ($bootindex_hash->{d
}) {
3918 $drive->{bootindex
} = $bootindex_hash->{d
};
3919 $bootindex_hash->{d
} += 1;
3922 if ($bootindex_hash->{c
}) {
3923 $drive->{bootindex
} = $bootindex_hash->{c
} if $conf->{bootdisk
} && ($conf->{bootdisk
} eq $ds);
3924 $bootindex_hash->{c
} += 1;
3928 if($drive->{interface
} eq 'virtio'){
3929 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3932 if ($drive->{interface
} eq 'scsi') {
3934 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3936 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3937 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3940 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3941 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3942 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3943 } elsif ($drive->{iothread
}) {
3944 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3948 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3949 $queues = ",num_queues=$drive->{queues}";
3952 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3953 $scsicontroller->{$controller}=1;
3956 if ($drive->{interface
} eq 'sata') {
3957 my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3958 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3959 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3960 $ahcicontroller->{$controller}=1;
3963 my $drive_cmd = print_drive_full
($storecfg, $vmid, $drive);
3964 push @$devices, '-drive',$drive_cmd;
3965 push @$devices, '-device', print_drivedevice_full
($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3968 for (my $i = 0; $i < $MAX_NETS; $i++) {
3969 next if !$conf->{"net$i"};
3970 my $d = parse_net
($conf->{"net$i"});
3973 $use_virtio = 1 if $d->{model
} eq 'virtio';
3975 if ($bootindex_hash->{n
}) {
3976 $d->{bootindex
} = $bootindex_hash->{n
};
3977 $bootindex_hash->{n
} += 1;
3980 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, "net$i");
3981 push @$devices, '-netdev', $netdevfull;
3983 my $netdevicefull = print_netdevice_full
($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3984 push @$devices, '-device', $netdevicefull;
3987 if ($conf->{ivshmem
}) {
3988 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3992 $bus = print_pcie_addr
("ivshmem");
3994 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3997 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3998 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4000 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4001 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4006 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3)) {
4011 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4013 while (my ($k, $v) = each %$bridges) {
4014 $pciaddr = print_pci_addr
("pci.$k", undef, $arch, $machine_type);
4015 unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4019 push @$cmd, @$devices;
4020 push @$cmd, '-rtc', join(',', @$rtcFlags)
4021 if scalar(@$rtcFlags);
4022 push @$cmd, '-machine', join(',', @$machineFlags)
4023 if scalar(@$machineFlags);
4024 push @$cmd, '-global', join(',', @$globalFlags)
4025 if scalar(@$globalFlags);
4027 if (my $vmstate = $conf->{vmstate
}) {
4028 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4029 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
4030 push @$cmd, '-loadstate', $statepath;
4034 if ($conf->{args
}) {
4035 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4039 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
4044 return "${var_run_tmpdir}/$vmid.vnc";
4050 my $res = vm_mon_cmd
($vmid, 'query-spice');
4052 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4056 my ($vmid, $qga, $name) = @_;
4057 my $sockettype = $qga ?
'qga' : 'qmp';
4058 my $ext = $name ?
'-'.$name : '';
4059 return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4064 return "${var_run_tmpdir}/$vmid.pid";
4067 sub vm_devices_list
{
4070 my $res = vm_mon_cmd
($vmid, 'query-pci');
4071 my $devices_to_check = [];
4073 foreach my $pcibus (@$res) {
4074 push @$devices_to_check, @{$pcibus->{devices
}},
4077 while (@$devices_to_check) {
4079 for my $d (@$devices_to_check) {
4080 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4081 next if !$d->{'pci_bridge'};
4083 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4084 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4086 $devices_to_check = $to_check;
4089 my $resblock = vm_mon_cmd
($vmid, 'query-block');
4090 foreach my $block (@$resblock) {
4091 if($block->{device
} =~ m/^drive-(\S+)/){
4096 my $resmice = vm_mon_cmd
($vmid, 'query-mice');
4097 foreach my $mice (@$resmice) {
4098 if ($mice->{name
} eq 'QEMU HID Tablet') {
4099 $devices->{tablet
} = 1;
4104 # for usb devices there is no query-usb
4105 # but we can iterate over the entries in
4106 # qom-list path=/machine/peripheral
4107 my $resperipheral = vm_mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4108 foreach my $per (@$resperipheral) {
4109 if ($per->{name
} =~ m/^usb\d+$/) {
4110 $devices->{$per->{name
}} = 1;
4118 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4120 my $q35 = machine_type_is_q35
($conf);
4122 my $devices_list = vm_devices_list
($vmid);
4123 return 1 if defined($devices_list->{$deviceid});
4125 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4127 if ($deviceid eq 'tablet') {
4129 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4131 } elsif ($deviceid eq 'keyboard') {
4133 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4135 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4137 die "usb hotplug currently not reliable\n";
4138 # since we can't reliably hot unplug all added usb devices
4139 # and usb passthrough disables live migration
4140 # we disable usb hotplugging for now
4141 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device));
4143 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4145 qemu_iothread_add
($vmid, $deviceid, $device);
4147 qemu_driveadd
($storecfg, $vmid, $device);
4148 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4150 qemu_deviceadd
($vmid, $devicefull);
4151 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4153 eval { qemu_drivedel
($vmid, $deviceid); };
4158 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4161 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4162 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4163 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4165 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4167 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4168 qemu_iothread_add
($vmid, $deviceid, $device);
4169 $devicefull .= ",iothread=iothread-$deviceid";
4172 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4173 $devicefull .= ",num_queues=$device->{queues}";
4176 qemu_deviceadd
($vmid, $devicefull);
4177 qemu_deviceaddverify
($vmid, $deviceid);
4179 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4181 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4182 qemu_driveadd
($storecfg, $vmid, $device);
4184 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4185 eval { qemu_deviceadd
($vmid, $devicefull); };
4187 eval { qemu_drivedel
($vmid, $deviceid); };
4192 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4194 return undef if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4196 my $machine_type = PVE
::QemuServer
::qemu_machine_pxe
($vmid, $conf);
4197 my $use_old_bios_files = undef;
4198 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4200 my $netdevicefull = print_netdevice_full
($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4201 qemu_deviceadd
($vmid, $netdevicefull);
4203 qemu_deviceaddverify
($vmid, $deviceid);
4204 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4207 eval { qemu_netdevdel
($vmid, $deviceid); };
4212 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4215 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4216 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4218 qemu_deviceadd
($vmid, $devicefull);
4219 qemu_deviceaddverify
($vmid, $deviceid);
4222 die "can't hotplug device '$deviceid'\n";
4228 # fixme: this should raise exceptions on error!
4229 sub vm_deviceunplug
{
4230 my ($vmid, $conf, $deviceid) = @_;
4232 my $devices_list = vm_devices_list
($vmid);
4233 return 1 if !defined($devices_list->{$deviceid});
4235 die "can't unplug bootdisk" if $conf->{bootdisk
} && $conf->{bootdisk
} eq $deviceid;
4237 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4239 qemu_devicedel
($vmid, $deviceid);
4241 } elsif ($deviceid =~ m/^usb\d+$/) {
4243 die "usb hotplug currently not reliable\n";
4244 # when unplugging usb devices this way,
4245 # there may be remaining usb controllers/hubs
4246 # so we disable it for now
4247 qemu_devicedel
($vmid, $deviceid);
4248 qemu_devicedelverify
($vmid, $deviceid);
4250 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4252 qemu_devicedel
($vmid, $deviceid);
4253 qemu_devicedelverify
($vmid, $deviceid);
4254 qemu_drivedel
($vmid, $deviceid);
4255 qemu_iothread_del
($conf, $vmid, $deviceid);
4257 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4259 qemu_devicedel
($vmid, $deviceid);
4260 qemu_devicedelverify
($vmid, $deviceid);
4261 qemu_iothread_del
($conf, $vmid, $deviceid);
4263 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4265 qemu_devicedel
($vmid, $deviceid);
4266 qemu_drivedel
($vmid, $deviceid);
4267 qemu_deletescsihw
($conf, $vmid, $deviceid);
4269 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4271 qemu_devicedel
($vmid, $deviceid);
4272 qemu_devicedelverify
($vmid, $deviceid);
4273 qemu_netdevdel
($vmid, $deviceid);
4276 die "can't unplug device '$deviceid'\n";
4282 sub qemu_deviceadd
{
4283 my ($vmid, $devicefull) = @_;
4285 $devicefull = "driver=".$devicefull;
4286 my %options = split(/[=,]/, $devicefull);
4288 vm_mon_cmd
($vmid, "device_add" , %options);
4291 sub qemu_devicedel
{
4292 my ($vmid, $deviceid) = @_;
4294 my $ret = vm_mon_cmd
($vmid, "device_del", id
=> $deviceid);
4297 sub qemu_iothread_add
{
4298 my($vmid, $deviceid, $device) = @_;
4300 if ($device->{iothread
}) {
4301 my $iothreads = vm_iothreads_list
($vmid);
4302 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4306 sub qemu_iothread_del
{
4307 my($conf, $vmid, $deviceid) = @_;
4309 my $confid = $deviceid;
4310 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4311 $confid = 'scsi' . $1;
4313 my $device = parse_drive
($confid, $conf->{$confid});
4314 if ($device->{iothread
}) {
4315 my $iothreads = vm_iothreads_list
($vmid);
4316 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4320 sub qemu_objectadd
{
4321 my($vmid, $objectid, $qomtype) = @_;
4323 vm_mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4328 sub qemu_objectdel
{
4329 my($vmid, $objectid) = @_;
4331 vm_mon_cmd
($vmid, "object-del", id
=> $objectid);
4337 my ($storecfg, $vmid, $device) = @_;
4339 my $drive = print_drive_full
($storecfg, $vmid, $device);
4340 $drive =~ s/\\/\\\\/g;
4341 my $ret = vm_human_monitor_command
($vmid, "drive_add auto \"$drive\"");
4343 # If the command succeeds qemu prints: "OK
"
4344 return 1 if $ret =~ m/OK/s;
4346 die "adding drive failed
: $ret\n";
4350 my($vmid, $deviceid) = @_;
4352 my $ret = vm_human_monitor_command($vmid, "drive_del drive-
$deviceid");
4355 return 1 if $ret eq "";
4357 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4358 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4360 die "deleting drive
$deviceid failed
: $ret\n";
4363 sub qemu_deviceaddverify {
4364 my ($vmid, $deviceid) = @_;
4366 for (my $i = 0; $i <= 5; $i++) {
4367 my $devices_list = vm_devices_list($vmid);
4368 return 1 if defined($devices_list->{$deviceid});
4372 die "error on hotplug device
'$deviceid'\n";
4376 sub qemu_devicedelverify {
4377 my ($vmid, $deviceid) = @_;
4379 # need to verify that the device is correctly removed as device_del
4380 # is async and empty return is not reliable
4382 for (my $i = 0; $i <= 5; $i++) {
4383 my $devices_list = vm_devices_list($vmid);
4384 return 1 if !defined($devices_list->{$deviceid});
4388 die "error on hot-unplugging device
'$deviceid'\n";
4391 sub qemu_findorcreatescsihw {
4392 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4394 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4396 my $scsihwid="$controller_prefix$controller";
4397 my $devices_list = vm_devices_list($vmid);
4399 if(!defined($devices_list->{$scsihwid})) {
4400 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4406 sub qemu_deletescsihw {
4407 my ($conf, $vmid, $opt) = @_;
4409 my $device = parse_drive($opt, $conf->{$opt});
4411 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4412 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4416 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4418 my $devices_list = vm_devices_list($vmid);
4419 foreach my $opt (keys %{$devices_list}) {
4420 if (PVE::QemuServer::is_valid_drivename($opt)) {
4421 my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4422 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4428 my $scsihwid="scsihw
$controller";
4430 vm_deviceunplug($vmid, $conf, $scsihwid);
4435 sub qemu_add_pci_bridge {
4436 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4442 print_pci_addr($device, $bridges, $arch, $machine_type);
4444 while (my ($k, $v) = each %$bridges) {
4447 return 1 if !defined($bridgeid) || $bridgeid < 1;
4449 my $bridge = "pci
.$bridgeid";
4450 my $devices_list = vm_devices_list($vmid);
4452 if (!defined($devices_list->{$bridge})) {
4453 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4459 sub qemu_set_link_status {
4460 my ($vmid, $device, $up) = @_;
4462 vm_mon_cmd($vmid, "set_link
", name => $device,
4463 up => $up ? JSON::true : JSON::false);
4466 sub qemu_netdevadd {
4467 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4469 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4470 my %options = split(/[=,]/, $netdev);
4472 vm_mon_cmd($vmid, "netdev_add
", %options);
4476 sub qemu_netdevdel {
4477 my ($vmid, $deviceid) = @_;
4479 vm_mon_cmd($vmid, "netdev_del
", id => $deviceid);
4482 sub qemu_usb_hotplug {
4483 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4487 # remove the old one first
4488 vm_deviceunplug($vmid, $conf, $deviceid);
4490 # check if xhci controller is necessary and available
4491 if ($device->{usb3}) {
4493 my $devicelist = vm_devices_list($vmid);
4495 if (!$devicelist->{xhci}) {
4496 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4497 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4500 my $d = parse_usb_device($device->{host});
4501 $d->{usb3} = $device->{usb3};
4504 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4507 sub qemu_cpu_hotplug {
4508 my ($vmid, $conf, $vcpus) = @_;
4510 my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4513 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4514 $sockets = $conf->{sockets} if $conf->{sockets};
4515 my $cores = $conf->{cores} || 1;
4516 my $maxcpus = $sockets * $cores;
4518 $vcpus = $maxcpus if !$vcpus;
4520 die "you can
't add more vcpus than maxcpus\n"
4521 if $vcpus > $maxcpus;
4523 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4525 if ($vcpus < $currentvcpus) {
4527 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4529 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4530 qemu_devicedel($vmid, "cpu$i");
4532 my $currentrunningvcpus = undef;
4534 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4535 last if scalar(@{$currentrunningvcpus}) == $i-1;
4536 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4540 #update conf after each succesfull cpu unplug
4541 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4542 PVE::QemuConfig->write_config($vmid, $conf);
4545 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4551 my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4552 die "vcpus in running vm does not match its configuration\n"
4553 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4555 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4557 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4558 my $cpustr = print_cpu_device($conf, $i);
4559 qemu_deviceadd($vmid, $cpustr);
4562 my $currentrunningvcpus = undef;
4564 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4565 last if scalar(@{$currentrunningvcpus}) == $i;
4566 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4570 #update conf after each succesfull cpu hotplug
4571 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4572 PVE::QemuConfig->write_config($vmid, $conf);
4576 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4577 vm_mon_cmd($vmid, "cpu-add", id => int($i));
4582 sub qemu_block_set_io_throttle {
4583 my ($vmid, $deviceid,
4584 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4585 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4586 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4587 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4589 return if !check_running($vmid) ;
4591 vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4593 bps_rd => int($bps_rd),
4594 bps_wr => int($bps_wr),
4596 iops_rd => int($iops_rd),
4597 iops_wr => int($iops_wr),
4598 bps_max => int($bps_max),
4599 bps_rd_max => int($bps_rd_max),
4600 bps_wr_max => int($bps_wr_max),
4601 iops_max => int($iops_max),
4602 iops_rd_max => int($iops_rd_max),
4603 iops_wr_max => int($iops_wr_max),
4604 bps_max_length => int($bps_max_length),
4605 bps_rd_max_length => int($bps_rd_max_length),
4606 bps_wr_max_length => int($bps_wr_max_length),
4607 iops_max_length => int($iops_max_length),
4608 iops_rd_max_length => int($iops_rd_max_length),
4609 iops_wr_max_length => int($iops_wr_max_length),
4614 # old code, only used to shutdown old VM after update
4616 my ($fh, $timeout) = @_;
4618 my $sel = new IO::Select;
4625 while (scalar (@ready = $sel->can_read($timeout))) {
4627 if ($count = $fh->sysread($buf, 8192)) {
4628 if ($buf =~ /^(.*)\(qemu\) $/s) {
4635 if (!defined($count)) {
4642 die "monitor read timeout\n" if !scalar(@ready);
4647 sub qemu_block_resize {
4648 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4650 my $running = check_running($vmid);
4652 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4654 return if !$running;
4656 vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4660 sub qemu_volume_snapshot {
4661 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4663 my $running = check_running($vmid);
4665 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4666 vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4668 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4672 sub qemu_volume_snapshot_delete {
4673 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4675 my $running = check_running($vmid);
4680 my $conf = PVE::QemuConfig->load_config($vmid);
4681 foreach_drive($conf, sub {
4682 my ($ds, $drive) = @_;
4683 $running = 1 if $drive->{file} eq $volid;
4687 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4688 vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4690 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4694 sub set_migration_caps {
4700 "auto-converge" => 1,
4702 "x-rdma-pin-all" => 0,
4707 my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4709 for my $supported_capability (@$supported_capabilities) {
4711 capability => $supported_capability->{capability},
4712 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4716 vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4719 my $fast_plug_option = {
4727 'vmstatestorage
' => 1,
4731 # hotplug changes in [PENDING]
4732 # $selection hash can be used to only apply specified options, for
4733 # example: { cores => 1 } (only apply changed 'cores
')
4734 # $errors ref is used to return error messages
4735 sub vmconfig_hotplug_pending {
4736 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4738 my $defaults = load_defaults();
4739 my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4741 # commit values which do not have any impact on running VM first
4742 # Note: those option cannot raise errors, we we do not care about
4743 # $selection and always apply them.
4745 my $add_error = sub {
4746 my ($opt, $msg) = @_;
4747 $errors->{$opt} = "hotplug problem - $msg";
4751 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4752 if ($fast_plug_option->{$opt}) {
4753 $conf->{$opt} = $conf->{pending}->{$opt};
4754 delete $conf->{pending}->{$opt};
4760 PVE::QemuConfig->write_config($vmid, $conf);
4761 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4764 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4766 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4767 while (my ($opt, $force) = each %$pending_delete_hash) {
4768 next if $selection && !$selection->{$opt};
4770 if ($opt eq 'hotplug
') {
4771 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4772 } elsif ($opt eq 'tablet
') {
4773 die "skip\n" if !$hotplug_features->{usb};
4774 if ($defaults->{tablet}) {
4775 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4776 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4777 if $arch eq 'aarch64
';
4779 vm_deviceunplug($vmid, $conf, 'tablet
');
4780 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4782 } elsif ($opt =~ m/^usb\d+/) {
4784 # since we cannot reliably hot unplug usb devices
4785 # we are disabling it
4786 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4787 vm_deviceunplug($vmid, $conf, $opt);
4788 } elsif ($opt eq 'vcpus
') {
4789 die "skip\n" if !$hotplug_features->{cpu};
4790 qemu_cpu_hotplug($vmid, $conf, undef);
4791 } elsif ($opt eq 'balloon
') {
4792 # enable balloon device is not hotpluggable
4793 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4794 # here we reset the ballooning value to memory
4795 my $balloon = $conf->{memory} || $defaults->{memory};
4796 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4797 } elsif ($fast_plug_option->{$opt}) {
4799 } elsif ($opt =~ m/^net(\d+)$/) {
4800 die "skip\n" if !$hotplug_features->{network};
4801 vm_deviceunplug($vmid, $conf, $opt);
4802 } elsif (is_valid_drivename($opt)) {
4803 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4804 vm_deviceunplug($vmid, $conf, $opt);
4805 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4806 } elsif ($opt =~ m/^memory$/) {
4807 die "skip\n" if !$hotplug_features->{memory};
4808 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4809 } elsif ($opt eq 'cpuunits
') {
4810 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4811 } elsif ($opt eq 'cpulimit
') {
4812 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4818 &$add_error($opt, $err) if $err ne "skip\n";
4820 # save new config if hotplug was successful
4821 delete $conf->{$opt};
4822 vmconfig_undelete_pending_option($conf, $opt);
4823 PVE::QemuConfig->write_config($vmid, $conf);
4824 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4828 my $apply_pending_cloudinit;
4829 $apply_pending_cloudinit = sub {
4830 my ($key, $value) = @_;
4831 $apply_pending_cloudinit = sub {}; # once is enough
4833 my @cloudinit_opts = keys %$confdesc_cloudinit;
4834 foreach my $opt (keys %{$conf->{pending}}) {
4835 next if !grep { $_ eq $opt } @cloudinit_opts;
4836 $conf->{$opt} = delete $conf->{pending}->{$opt};
4839 my $new_conf = { %$conf };
4840 $new_conf->{$key} = $value;
4841 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4844 foreach my $opt (keys %{$conf->{pending}}) {
4845 next if $selection && !$selection->{$opt};
4846 my $value = $conf->{pending}->{$opt};
4848 if ($opt eq 'hotplug
') {
4849 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4850 } elsif ($opt eq 'tablet
') {
4851 die "skip\n" if !$hotplug_features->{usb};
4853 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4854 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4855 if $arch eq 'aarch64
';
4856 } elsif ($value == 0) {
4857 vm_deviceunplug($vmid, $conf, 'tablet
');
4858 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4860 } elsif ($opt =~ m/^usb\d+$/) {
4862 # since we cannot reliably hot unplug usb devices
4863 # we are disabling it
4864 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4865 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4866 die "skip\n" if !$d;
4867 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4868 } elsif ($opt eq 'vcpus
') {
4869 die "skip\n" if !$hotplug_features->{cpu};
4870 qemu_cpu_hotplug($vmid, $conf, $value);
4871 } elsif ($opt eq 'balloon
') {
4872 # enable/disable balloning device is not hotpluggable
4873 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4874 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4875 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4877 # allow manual ballooning if shares is set to zero
4878 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4879 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4880 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4882 } elsif ($opt =~ m/^net(\d+)$/) {
4883 # some changes can be done without hotplug
4884 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4885 $vmid, $opt, $value, $arch, $machine_type);
4886 } elsif (is_valid_drivename($opt)) {
4887 # some changes can be done without hotplug
4888 my $drive = parse_drive($opt, $value);
4889 if (drive_is_cloudinit($drive)) {
4890 &$apply_pending_cloudinit($opt, $value);
4892 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4893 $vmid, $opt, $value, 1, $arch, $machine_type);
4894 } elsif ($opt =~ m/^memory$/) { #dimms
4895 die "skip\n" if !$hotplug_features->{memory};
4896 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4897 } elsif ($opt eq 'cpuunits
') {
4898 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4899 } elsif ($opt eq 'cpulimit
') {
4900 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4901 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4903 die "skip\n"; # skip non-hot-pluggable options
4907 &$add_error($opt, $err) if $err ne "skip\n";
4909 # save new config if hotplug was successful
4910 $conf->{$opt} = $value;
4911 delete $conf->{pending}->{$opt};
4912 PVE::QemuConfig->write_config($vmid, $conf);
4913 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4918 sub try_deallocate_drive {
4919 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4921 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4922 my $volid = $drive->{file};
4923 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4924 my $sid = PVE::Storage::parse_volume_id($volid);
4925 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4927 # check if the disk is really unused
4928 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4929 if is_volume_in_use($storecfg, $conf, $key, $volid);
4930 PVE::Storage::vdisk_free($storecfg, $volid);
4933 # If vm is not owner of this disk remove from config
4941 sub vmconfig_delete_or_detach_drive {
4942 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4944 my $drive = parse_drive($opt, $conf->{$opt});
4946 my $rpcenv = PVE::RPCEnvironment::get();
4947 my $authuser = $rpcenv->get_user();
4950 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4951 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4953 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4957 sub vmconfig_apply_pending {
4958 my ($vmid, $conf, $storecfg) = @_;
4962 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4963 while (my ($opt, $force) = each %$pending_delete_hash) {
4964 die "internal error" if $opt =~ m/^unused/;
4965 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4966 if (!defined($conf->{$opt})) {
4967 vmconfig_undelete_pending_option($conf, $opt);
4968 PVE::QemuConfig->write_config($vmid, $conf);
4969 } elsif (is_valid_drivename($opt)) {
4970 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4971 vmconfig_undelete_pending_option($conf, $opt);
4972 delete $conf->{$opt};
4973 PVE::QemuConfig->write_config($vmid, $conf);
4975 vmconfig_undelete_pending_option($conf, $opt);
4976 delete $conf->{$opt};
4977 PVE::QemuConfig->write_config($vmid, $conf);
4981 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4983 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4984 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4986 if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4987 # skip if nothing changed
4988 } elsif (is_valid_drivename($opt)) {
4989 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4990 if defined($conf->{$opt});
4991 $conf->{$opt} = $conf->{pending}->{$opt};
4993 $conf->{$opt} = $conf->{pending}->{$opt};
4996 delete $conf->{pending}->{$opt};
4997 PVE::QemuConfig->write_config($vmid, $conf);
5001 my $safe_num_ne = sub {
5004 return 0 if !defined($a) && !defined($b);
5005 return 1 if !defined($a);
5006 return 1 if !defined($b);
5011 my $safe_string_ne = sub {
5014 return 0 if !defined($a) && !defined($b);
5015 return 1 if !defined($a);
5016 return 1 if !defined($b);
5021 sub vmconfig_update_net {
5022 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5024 my $newnet = parse_net($value);
5026 if ($conf->{$opt}) {
5027 my $oldnet = parse_net($conf->{$opt});
5029 if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5030 &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5031 &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5032 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5034 # for non online change, we try to hot-unplug
5035 die "skip\n" if !$hotplug;
5036 vm_deviceunplug($vmid, $conf, $opt);
5039 die "internal error" if $opt !~ m/net(\d+)/;
5040 my $iface = "tap${vmid}i$1";
5042 if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5043 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5044 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5045 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5046 PVE::Network::tap_unplug($iface);
5047 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5048 } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5049 # Rate can be applied on its own but any change above needs to
5050 # include the rate in tap_plug since OVS resets everything.
5051 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5054 if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5055 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5063 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5069 sub vmconfig_update_disk {
5070 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5072 # fixme: do we need force?
5074 my $drive = parse_drive($opt, $value);
5076 if ($conf->{$opt}) {
5078 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
5080 my $media = $drive->{media} || 'disk
';
5081 my $oldmedia = $old_drive->{media} || 'disk
';
5082 die "unable to change media type\n" if $media ne $oldmedia;
5084 if (!drive_is_cdrom($old_drive)) {
5086 if ($drive->{file} ne $old_drive->{file}) {
5088 die "skip\n" if !$hotplug;
5090 # unplug and register as unused
5091 vm_deviceunplug($vmid, $conf, $opt);
5092 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5095 # update existing disk
5097 # skip non hotpluggable value
5098 if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5099 &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5100 &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5101 &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5106 if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5107 &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5108 &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5109 &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5110 &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5111 &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5112 &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5113 &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5114 &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5115 &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5116 &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5117 &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5118 &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5119 &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5120 &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5121 &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5122 &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5123 &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5125 qemu_block_set_io_throttle($vmid,"drive-$opt",
5126 ($drive->{mbps} || 0)*1024*1024,
5127 ($drive->{mbps_rd} || 0)*1024*1024,
5128 ($drive->{mbps_wr} || 0)*1024*1024,
5129 $drive->{iops} || 0,
5130 $drive->{iops_rd} || 0,
5131 $drive->{iops_wr} || 0,
5132 ($drive->{mbps_max} || 0)*1024*1024,
5133 ($drive->{mbps_rd_max} || 0)*1024*1024,
5134 ($drive->{mbps_wr_max} || 0)*1024*1024,
5135 $drive->{iops_max} || 0,
5136 $drive->{iops_rd_max} || 0,
5137 $drive->{iops_wr_max} || 0,
5138 $drive->{bps_max_length} || 1,
5139 $drive->{bps_rd_max_length} || 1,
5140 $drive->{bps_wr_max_length} || 1,
5141 $drive->{iops_max_length} || 1,
5142 $drive->{iops_rd_max_length} || 1,
5143 $drive->{iops_wr_max_length} || 1);
5152 if ($drive->{file} eq 'none
') {
5153 vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5154 if (drive_is_cloudinit($old_drive)) {
5155 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5158 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5159 vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5160 vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5168 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5170 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5171 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5175 my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5176 $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5178 PVE::QemuConfig->lock_config($vmid, sub {
5179 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5181 die "you can't start a vm
if it
's a template\n" if PVE::QemuConfig->is_template($conf);
5183 my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended
');
5185 PVE::QemuConfig->check_lock($conf)
5186 if !($skiplock || $is_suspended);
5188 die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5190 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5191 vmconfig_apply_pending($vmid, $conf, $storecfg);
5192 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5195 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5197 my $defaults = load_defaults();
5199 # set environment variable useful inside network script
5200 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5202 my $local_volumes = {};
5204 if ($targetstorage) {
5205 foreach_drive($conf, sub {
5206 my ($ds, $drive) = @_;
5208 return if drive_is_cdrom($drive);
5210 my $volid = $drive->{file};
5214 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5216 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5217 return if $scfg->{shared};
5218 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5223 foreach my $opt (sort keys %$local_volumes) {
5225 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5226 my $drive = parse_drive($opt, $conf->{$opt});
5228 #if remote storage is specified, use default format
5229 if ($targetstorage && $targetstorage ne "1") {
5230 $storeid = $targetstorage;
5231 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5232 $format = $defFormat;
5234 #else we use same format than original
5235 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5236 $format = qemu_img_format($scfg, $volid);
5239 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5240 my $newdrive = $drive;
5241 $newdrive->{format} = $format;
5242 $newdrive->{file} = $newvolid;
5243 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5244 $local_volumes->{$opt} = $drivestr;
5245 #pass drive to conf for command line
5246 $conf->{$opt} = $drivestr;
5250 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
5252 if ($is_suspended) {
5253 # enforce machine type on suspended vm to ensure HW compatibility
5254 $forcemachine = $conf->{runningmachine};
5255 print "Resuming suspended VM\n";
5258 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5260 my $migrate_port = 0;
5263 if ($statefile eq 'tcp
') {
5264 my $localip = "localhost";
5265 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5266 my $nodename = PVE::INotify::nodename();
5268 if (!defined($migration_type)) {
5269 if (defined($datacenterconf->{migration}->{type})) {
5270 $migration_type = $datacenterconf->{migration}->{type};
5272 $migration_type = 'secure
';
5276 if ($migration_type eq 'insecure
') {
5277 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5278 if ($migrate_network_addr) {
5279 $localip = $migrate_network_addr;
5281 $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5284 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5287 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5288 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5289 $migrate_uri = "tcp:${localip}:${migrate_port}";
5290 push @$cmd, '-incoming
', $migrate_uri;
5293 } elsif ($statefile eq 'unix
') {
5294 # should be default for secure migrations as a ssh TCP forward
5295 # tunnel is not deterministic reliable ready and fails regurarly
5296 # to set up in time, so use UNIX socket forwards
5297 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5298 unlink $socket_addr;
5300 $migrate_uri = "unix:$socket_addr";
5302 push @$cmd, '-incoming
', $migrate_uri;
5306 push @$cmd, '-loadstate
', $statefile;
5313 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
5314 my $d = parse_hostpci($conf->{"hostpci$i"});
5316 my $pcidevices = $d->{pciid};
5317 foreach my $pcidevice (@$pcidevices) {
5318 my $pciid = $pcidevice->{id};
5320 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5321 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5322 die "no pci device info for device '$pciid'\n" if !$info;
5325 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5326 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5328 die "can't unbind
/bind pci group to vfio
'$pciid'\n"
5329 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5330 die "can
't reset pci device '$pciid'\n"
5331 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5336 PVE::Storage::activate_volumes($storecfg, $vollist);
5338 if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
5340 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5341 outfunc => sub {}, errfunc => sub {});
5345 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5346 : $defaults->{cpuunits};
5348 my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5349 my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5352 Slice => 'qemu
.slice
',
5354 CPUShares => $cpuunits
5357 if (my $cpulimit = $conf->{cpulimit}) {
5358 $properties{CPUQuota} = int($cpulimit * 100);
5360 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5362 my $run_qemu = sub {
5363 PVE::Tools::run_fork sub {
5364 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5365 run_command($cmd, %run_params);
5369 if ($conf->{hugepages}) {
5372 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5373 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5375 PVE::QemuServer::Memory::hugepages_mount();
5376 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5378 eval { $run_qemu->() };
5380 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5384 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5386 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5389 eval { $run_qemu->() };
5393 # deactivate volumes if start fails
5394 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5395 die "start failed: $err";
5398 print "migration listens on $migrate_uri\n" if $migrate_uri;
5400 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5401 eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5405 #start nbd server for storage migration
5406 if ($targetstorage) {
5407 my $nodename = PVE::INotify::nodename();
5408 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5409 my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5410 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5411 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5413 vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet
', data => { host => "${localip}", port => "${migrate_port}" } } );
5415 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5417 foreach my $opt (sort keys %$local_volumes) {
5418 my $volid = $local_volumes->{$opt};
5419 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5420 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5421 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5425 if ($migratedfrom) {
5427 set_migration_caps($vmid);
5432 print "spice listens on port $spice_port\n";
5433 if ($spice_ticket) {
5434 vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice
', password => $spice_ticket);
5435 vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice
', time => "+30");
5440 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5441 if !$statefile && $conf->{balloon};
5443 foreach my $opt (keys %$conf) {
5444 next if $opt !~ m/^net\d+$/;
5445 my $nicconf = parse_net($conf->{$opt});
5446 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5450 vm_mon_cmd_nocheck($vmid, 'qom-set
',
5451 path => "machine/peripheral/balloon0",
5452 property => "guest-stats-polling-interval",
5453 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5455 if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5456 print "Resumed VM, removing state\n";
5457 delete $conf->@{qw(lock vmstate runningmachine)};
5458 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5459 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5460 PVE
::QemuConfig-
>write_config($vmid, $conf);
5463 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5468 my ($vmid, $execute, %params) = @_;
5470 my $cmd = { execute
=> $execute, arguments
=> \
%params };
5471 vm_qmp_command
($vmid, $cmd);
5474 sub vm_mon_cmd_nocheck
{
5475 my ($vmid, $execute, %params) = @_;
5477 my $cmd = { execute
=> $execute, arguments
=> \
%params };
5478 vm_qmp_command
($vmid, $cmd, 1);
5481 sub vm_qmp_command
{
5482 my ($vmid, $cmd, $nocheck) = @_;
5487 if ($cmd->{arguments
} && $cmd->{arguments
}->{timeout
}) {
5488 $timeout = $cmd->{arguments
}->{timeout
};
5489 delete $cmd->{arguments
}->{timeout
};
5493 die "VM $vmid not running\n" if !check_running
($vmid, $nocheck);
5494 my $sname = qmp_socket
($vmid);
5495 if (-e
$sname) { # test if VM is reasonambe new and supports qmp/qga
5496 my $qmpclient = PVE
::QMPClient-
>new();
5498 $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5500 die "unable to open monitor socket\n";
5504 syslog
("err", "VM $vmid qmp command failed - $err");
5511 sub vm_human_monitor_command
{
5512 my ($vmid, $cmdline) = @_;
5517 execute
=> 'human-monitor-command',
5518 arguments
=> { 'command-line' => $cmdline},
5521 return vm_qmp_command
($vmid, $cmd);
5524 sub vm_commandline
{
5525 my ($storecfg, $vmid, $snapname) = @_;
5527 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5530 my $snapshot = $conf->{snapshots
}->{$snapname};
5531 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5533 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5538 my $defaults = load_defaults
();
5540 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults);
5542 return PVE
::Tools
::cmd2string
($cmd);
5546 my ($vmid, $skiplock) = @_;
5548 PVE
::QemuConfig-
>lock_config($vmid, sub {
5550 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5552 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5554 vm_mon_cmd
($vmid, "system_reset");
5558 sub get_vm_volumes
{
5562 foreach_volid
($conf, sub {
5563 my ($volid, $attr) = @_;
5565 return if $volid =~ m
|^/|;
5567 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5570 push @$vollist, $volid;
5576 sub vm_stop_cleanup
{
5577 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5582 my $vollist = get_vm_volumes
($conf);
5583 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5586 foreach my $ext (qw(mon qmp pid vnc qga)) {
5587 unlink "/var/run/qemu-server/${vmid}.$ext";
5590 if ($conf->{ivshmem
}) {
5591 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5592 # just delete it for now, VMs which have this already open do not
5593 # are affected, but new VMs will get a separated one. If this
5594 # becomes an issue we either add some sort of ref-counting or just
5595 # add a "don't delete on stop" flag to the ivshmem format.
5596 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5599 foreach my $key (keys %$conf) {
5600 next if $key !~ m/^hostpci(\d+)$/;
5601 my $hostpciindex = $1;
5602 my $d = parse_hostpci
($conf->{$key});
5603 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5605 foreach my $pci (@{$d->{pciid
}}) {
5606 my $pciid = $pci->{id
};
5607 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5611 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5613 warn $@ if $@; # avoid errors - just warn
5616 # Note: use $nockeck to skip tests if VM configuration file exists.
5617 # We need that when migration VMs to other nodes (files already moved)
5618 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5620 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5622 $force = 1 if !defined($force) && !$shutdown;
5625 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5626 kill 15, $pid if $pid;
5627 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5628 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5632 PVE
::QemuConfig-
>lock_config($vmid, sub {
5634 my $pid = check_running
($vmid, $nocheck);
5639 $conf = PVE
::QemuConfig-
>load_config($vmid);
5640 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5641 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5642 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5643 $timeout = $opts->{down
} if $opts->{down
};
5645 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5648 $timeout = 60 if !defined($timeout);
5652 if (defined($conf) && parse_guest_agent
($conf)->{enabled
}) {
5653 vm_qmp_command
($vmid, { execute
=> "guest-shutdown" }, $nocheck);
5655 vm_qmp_command
($vmid, { execute
=> "system_powerdown" }, $nocheck);
5658 vm_qmp_command
($vmid, { execute
=> "quit" }, $nocheck);
5665 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5670 if ($count >= $timeout) {
5672 warn "VM still running - terminating now with SIGTERM\n";
5675 die "VM quit/powerdown failed - got timeout\n";
5678 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5683 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5686 die "VM quit/powerdown failed\n";
5694 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5699 if ($count >= $timeout) {
5700 warn "VM still running - terminating now with SIGKILL\n";
5705 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5710 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5717 PVE
::QemuConfig-
>lock_config($vmid, sub {
5719 $conf = PVE
::QemuConfig-
>load_config($vmid);
5721 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5722 PVE
::QemuConfig-
>check_lock($conf)
5723 if !($skiplock || $is_backing_up);
5725 die "cannot suspend to disk during backup\n"
5726 if $is_backing_up && $includestate;
5728 if ($includestate) {
5729 $conf->{lock} = 'suspending';
5730 my $date = strftime
("%Y-%m-%d", localtime(time()));
5731 $storecfg = PVE
::Storage
::config
();
5732 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5733 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5734 PVE
::QemuConfig-
>write_config($vmid, $conf);
5736 vm_mon_cmd
($vmid, "stop");
5740 if ($includestate) {
5742 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5745 vm_mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5747 my $state = vm_mon_cmd_nocheck
($vmid, "query-savevm");
5748 if (!$state->{status
}) {
5749 die "savevm not active\n";
5750 } elsif ($state->{status
} eq 'active') {
5753 } elsif ($state->{status
} eq 'completed') {
5754 print "State saved, quitting\n";
5756 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5757 die "query-savevm failed with error '$state->{error}'\n"
5759 die "query-savevm returned status '$state->{status}'\n";
5765 PVE
::QemuConfig-
>lock_config($vmid, sub {
5766 $conf = PVE
::QemuConfig-
>load_config($vmid);
5768 # cleanup, but leave suspending lock, to indicate something went wrong
5770 vm_mon_cmd
($vmid, "savevm-end");
5771 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5772 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5773 delete $conf->@{qw(vmstate runningmachine)};
5774 PVE
::QemuConfig-
>write_config($vmid, $conf);
5780 die "lock changed unexpectedly\n"
5781 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5783 vm_qmp_command
($vmid, { execute
=> "quit" });
5784 $conf->{lock} = 'suspended';
5785 PVE
::QemuConfig-
>write_config($vmid, $conf);
5791 my ($vmid, $skiplock, $nocheck) = @_;
5793 PVE
::QemuConfig-
>lock_config($vmid, sub {
5794 my $vm_mon_cmd = $nocheck ? \
&vm_mon_cmd_nocheck
: \
&vm_mon_cmd
;
5795 my $res = $vm_mon_cmd->($vmid, 'query-status');
5796 my $resume_cmd = 'cont';
5798 if ($res->{status
} && $res->{status
} eq 'suspended') {
5799 $resume_cmd = 'system_wakeup';
5804 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5806 PVE
::QemuConfig-
>check_lock($conf)
5807 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5810 $vm_mon_cmd->($vmid, $resume_cmd);
5815 my ($vmid, $skiplock, $key) = @_;
5817 PVE
::QemuConfig-
>lock_config($vmid, sub {
5819 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5821 # there is no qmp command, so we use the human monitor command
5822 vm_human_monitor_command
($vmid, "sendkey $key");
5827 my ($storecfg, $vmid, $skiplock) = @_;
5829 PVE
::QemuConfig-
>lock_config($vmid, sub {
5831 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5833 if (!check_running
($vmid)) {
5834 destroy_vm
($storecfg, $vmid, undef, $skiplock);
5836 die "VM $vmid is running - destroy failed\n";
5841 # vzdump restore implementaion
5843 sub tar_archive_read_firstfile
{
5844 my $archive = shift;
5846 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5848 # try to detect archive type first
5849 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5850 die "unable to open file '$archive'\n";
5851 my $firstfile = <$fh>;
5855 die "ERROR: archive contaions no data\n" if !$firstfile;
5861 sub tar_restore_cleanup
{
5862 my ($storecfg, $statfile) = @_;
5864 print STDERR
"starting cleanup\n";
5866 if (my $fd = IO
::File-
>new($statfile, "r")) {
5867 while (defined(my $line = <$fd>)) {
5868 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5871 if ($volid =~ m
|^/|) {
5872 unlink $volid || die 'unlink failed\n';
5874 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5876 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5878 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5880 print STDERR
"unable to parse line in statfile - $line";
5887 sub restore_archive
{
5888 my ($archive, $vmid, $user, $opts) = @_;
5890 my $format = $opts->{format
};
5893 if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5894 $format = 'tar' if !$format;
5896 } elsif ($archive =~ m/\.tar$/) {
5897 $format = 'tar' if !$format;
5898 } elsif ($archive =~ m/.tar.lzo$/) {
5899 $format = 'tar' if !$format;
5901 } elsif ($archive =~ m/\.vma$/) {
5902 $format = 'vma' if !$format;
5903 } elsif ($archive =~ m/\.vma\.gz$/) {
5904 $format = 'vma' if !$format;
5906 } elsif ($archive =~ m/\.vma\.lzo$/) {
5907 $format = 'vma' if !$format;
5910 $format = 'vma' if !$format; # default
5913 # try to detect archive format
5914 if ($format eq 'tar') {
5915 return restore_tar_archive
($archive, $vmid, $user, $opts);
5917 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5921 sub restore_update_config_line
{
5922 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5924 return if $line =~ m/^\#qmdump\#/;
5925 return if $line =~ m/^\#vzdump\#/;
5926 return if $line =~ m/^lock:/;
5927 return if $line =~ m/^unused\d+:/;
5928 return if $line =~ m/^parent:/;
5930 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5931 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5932 # try to convert old 1.X settings
5933 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5934 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5935 my ($model, $macaddr) = split(/\=/, $devconfig);
5936 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5939 bridge
=> "vmbr$ind",
5940 macaddr
=> $macaddr,
5942 my $netstr = print_net
($net);
5944 print $outfd "net$cookie->{netcount}: $netstr\n";
5945 $cookie->{netcount
}++;
5947 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5948 my ($id, $netstr) = ($1, $2);
5949 my $net = parse_net
($netstr);
5950 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5951 $netstr = print_net
($net);
5952 print $outfd "$id: $netstr\n";
5953 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5956 my $di = parse_drive
($virtdev, $value);
5957 if (defined($di->{backup
}) && !$di->{backup
}) {
5958 print $outfd "#$line";
5959 } elsif ($map->{$virtdev}) {
5960 delete $di->{format
}; # format can change on restore
5961 $di->{file
} = $map->{$virtdev};
5962 $value = print_drive
($vmid, $di);
5963 print $outfd "$virtdev: $value\n";
5967 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5969 if ($vmgenid ne '0') {
5970 # always generate a new vmgenid if there was a valid one setup
5971 $vmgenid = generate_uuid
();
5973 print $outfd "vmgenid: $vmgenid\n";
5974 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5975 my ($uuid, $uuid_str);
5976 UUID
::generate
($uuid);
5977 UUID
::unparse
($uuid, $uuid_str);
5978 my $smbios1 = parse_smbios1
($2);
5979 $smbios1->{uuid
} = $uuid_str;
5980 print $outfd $1.print_smbios1
($smbios1)."\n";
5987 my ($cfg, $vmid) = @_;
5989 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5991 my $volid_hash = {};
5992 foreach my $storeid (keys %$info) {
5993 foreach my $item (@{$info->{$storeid}}) {
5994 next if !($item->{volid
} && $item->{size
});
5995 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5996 $volid_hash->{$item->{volid
}} = $item;
6003 sub is_volume_in_use
{
6004 my ($storecfg, $conf, $skip_drive, $volid) = @_;
6006 my $path = PVE
::Storage
::path
($storecfg, $volid);
6008 my $scan_config = sub {
6009 my ($cref, $snapname) = @_;
6011 foreach my $key (keys %$cref) {
6012 my $value = $cref->{$key};
6013 if (is_valid_drivename
($key)) {
6014 next if $skip_drive && $key eq $skip_drive;
6015 my $drive = parse_drive
($key, $value);
6016 next if !$drive || !$drive->{file
} || drive_is_cdrom
($drive);
6017 return 1 if $volid eq $drive->{file
};
6018 if ($drive->{file
} =~ m!^/!) {
6019 return 1 if $drive->{file
} eq $path;
6021 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
}, 1);
6023 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid, 1);
6025 return 1 if $path eq PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
6033 return 1 if &$scan_config($conf);
6037 foreach my $snapname (keys %{$conf->{snapshots
}}) {
6038 return 1 if &$scan_config($conf->{snapshots
}->{$snapname}, $snapname);
6044 sub update_disksize
{
6045 my ($vmid, $conf, $volid_hash) = @_;
6048 my $prefix = "VM $vmid:";
6050 # used and unused disks
6051 my $referenced = {};
6053 # Note: it is allowed to define multiple storages with same path (alias), so
6054 # we need to check both 'volid' and real 'path' (two different volid can point
6055 # to the same path).
6057 my $referencedpath = {};
6060 foreach my $opt (keys %$conf) {
6061 if (is_valid_drivename
($opt)) {
6062 my $drive = parse_drive
($opt, $conf->{$opt});
6063 my $volid = $drive->{file
};
6066 $referenced->{$volid} = 1;
6067 if ($volid_hash->{$volid} &&
6068 (my $path = $volid_hash->{$volid}->{path
})) {
6069 $referencedpath->{$path} = 1;
6072 next if drive_is_cdrom
($drive);
6073 next if !$volid_hash->{$volid};
6075 $drive->{size
} = $volid_hash->{$volid}->{size
};
6076 my $new = print_drive
($vmid, $drive);
6077 if ($new ne $conf->{$opt}) {
6079 $conf->{$opt} = $new;
6080 print "$prefix update disk '$opt' information.\n";
6085 # remove 'unusedX' entry if volume is used
6086 foreach my $opt (keys %$conf) {
6087 next if $opt !~ m/^unused\d+$/;
6088 my $volid = $conf->{$opt};
6089 my $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6090 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6091 print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6093 delete $conf->{$opt};
6096 $referenced->{$volid} = 1;
6097 $referencedpath->{$path} = 1 if $path;
6100 foreach my $volid (sort keys %$volid_hash) {
6101 next if $volid =~ m/vm-$vmid-state-/;
6102 next if $referenced->{$volid};
6103 my $path = $volid_hash->{$volid}->{path
};
6104 next if !$path; # just to be sure
6105 next if $referencedpath->{$path};
6107 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6108 print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6109 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6116 my ($vmid, $nolock, $dryrun) = @_;
6118 my $cfg = PVE
::Storage
::config
();
6120 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6121 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6122 foreach my $stor (keys %{$cfg->{ids
}}) {
6123 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
6126 print "rescan volumes...\n";
6127 my $volid_hash = scan_volids
($cfg, $vmid);
6129 my $updatefn = sub {
6132 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6134 PVE
::QemuConfig-
>check_lock($conf);
6137 foreach my $volid (keys %$volid_hash) {
6138 my $info = $volid_hash->{$volid};
6139 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6142 my $changes = update_disksize
($vmid, $conf, $vm_volids);
6144 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6147 if (defined($vmid)) {
6151 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6154 my $vmlist = config_list
();
6155 foreach my $vmid (keys %$vmlist) {
6159 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6165 sub restore_vma_archive
{
6166 my ($archive, $vmid, $user, $opts, $comp) = @_;
6168 my $readfrom = $archive;
6170 my $cfg = PVE
::Storage
::config
();
6172 my $bwlimit = $opts->{bwlimit
};
6174 my $dbg_cmdstring = '';
6175 my $add_pipe = sub {
6177 push @$commands, $cmd;
6178 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6179 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6184 if ($archive eq '-') {
6187 # If we use a backup from a PVE defined storage we also consider that
6188 # storage's rate limit:
6189 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6190 if (defined($volid)) {
6191 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6192 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6194 print STDERR
"applying read rate limit: $readlimit\n";
6195 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6196 $add_pipe->($cstream);
6203 if ($comp eq 'gzip') {
6204 $cmd = ['zcat', $readfrom];
6205 } elsif ($comp eq 'lzop') {
6206 $cmd = ['lzop', '-d', '-c', $readfrom];
6208 die "unknown compression method '$comp'\n";
6213 my $tmpdir = "/var/tmp/vzdumptmp$$";
6216 # disable interrupts (always do cleanups)
6220 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6222 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6223 POSIX
::mkfifo
($mapfifo, 0600);
6226 my $openfifo = sub {
6227 open($fifofh, '>', $mapfifo) || die $!;
6230 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6237 my $rpcenv = PVE
::RPCEnvironment
::get
();
6239 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6240 my $tmpfn = "$conffile.$$.tmp";
6242 # Note: $oldconf is undef if VM does not exists
6243 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6244 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6248 my $print_devmap = sub {
6249 my $virtdev_hash = {};
6251 my $cfgfn = "$tmpdir/qemu-server.conf";
6253 # we can read the config - that is already extracted
6254 my $fh = IO
::File-
>new($cfgfn, "r") ||
6255 "unable to read qemu-server.conf - $!\n";
6257 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6259 my $pve_firewall_dir = '/etc/pve/firewall';
6260 mkdir $pve_firewall_dir; # make sure the dir exists
6261 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6264 while (defined(my $line = <$fh>)) {
6265 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6266 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6267 die "archive does not contain data for drive '$virtdev'\n"
6268 if !$devinfo->{$devname};
6269 if (defined($opts->{storage
})) {
6270 $storeid = $opts->{storage
} || 'local';
6271 } elsif (!$storeid) {
6274 $format = 'raw' if !$format;
6275 $devinfo->{$devname}->{devname
} = $devname;
6276 $devinfo->{$devname}->{virtdev
} = $virtdev;
6277 $devinfo->{$devname}->{format
} = $format;
6278 $devinfo->{$devname}->{storeid
} = $storeid;
6280 # check permission on storage
6281 my $pool = $opts->{pool
}; # todo: do we need that?
6282 if ($user ne 'root@pam') {
6283 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6286 $storage_limits{$storeid} = $bwlimit;
6288 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6289 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6291 my $drive = parse_drive
($virtdev, $2);
6292 if (drive_is_cloudinit
($drive)) {
6293 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6294 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
6295 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6299 storeid
=> $opts->{storage
} // $storeid,
6300 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6301 file
=> $drive->{file
}, # to make drive_is_cloudinit check possible
6302 name
=> "vm-$vmid-cloudinit",
6305 $virtdev_hash->{$virtdev} = $d;
6310 foreach my $key (keys %storage_limits) {
6311 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6313 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6314 $storage_limits{$key} = $limit * 1024;
6317 foreach my $devname (keys %$devinfo) {
6318 die "found no device mapping information for device '$devname'\n"
6319 if !$devinfo->{$devname}->{virtdev
};
6322 # create empty/temp config
6324 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6325 foreach_drive
($oldconf, sub {
6326 my ($ds, $drive) = @_;
6328 return if !$drive->{is_cloudinit
} && drive_is_cdrom
($drive);
6330 my $volid = $drive->{file
};
6331 return if !$volid || $volid =~ m
|^/|;
6333 my ($path, $owner) = PVE
::Storage
::path
($cfg, $volid);
6334 return if !$path || !$owner || ($owner != $vmid);
6336 # Note: only delete disk we want to restore
6337 # other volumes will become unused
6338 if ($virtdev_hash->{$ds}) {
6339 eval { PVE
::Storage
::vdisk_free
($cfg, $volid); };
6346 # delete vmstate files, after the restore we have no snapshots anymore
6347 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
6348 my $snap = $oldconf->{snapshots
}->{$snapname};
6349 if ($snap->{vmstate
}) {
6350 eval { PVE
::Storage
::vdisk_free
($cfg, $snap->{vmstate
}); };
6359 foreach my $virtdev (sort keys %$virtdev_hash) {
6360 my $d = $virtdev_hash->{$virtdev};
6361 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6362 my $storeid = $d->{storeid
};
6363 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
6366 if (my $limit = $storage_limits{$storeid}) {
6367 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6370 # test if requested format is supported
6371 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($cfg, $storeid);
6372 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6373 $d->{format
} = $defFormat if !$supported;
6376 if ($d->{is_cloudinit
}) {
6378 $name .= ".$d->{format}" if $d->{format
} ne 'raw';
6381 my $volid = PVE
::Storage
::vdisk_alloc
($cfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6382 print STDERR
"new volume ID is '$volid'\n";
6383 $d->{volid
} = $volid;
6385 PVE
::Storage
::activate_volumes
($cfg, [$volid]);
6387 my $write_zeros = 1;
6388 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6392 if (!$d->{is_cloudinit
}) {
6393 my $path = PVE
::Storage
::path
($cfg, $volid);
6395 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6397 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6399 $map->{$virtdev} = $volid;
6402 $fh->seek(0, 0) || die "seek failed - $!\n";
6404 my $outfd = new IO
::File
($tmpfn, "w") ||
6405 die "unable to write config for VM $vmid\n";
6407 my $cookie = { netcount
=> 0 };
6408 while (defined(my $line = <$fh>)) {
6409 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6422 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6423 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6425 $oldtimeout = alarm($timeout);
6432 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6433 my ($dev_id, $size, $devname) = ($1, $2, $3);
6434 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6435 } elsif ($line =~ m/^CTIME: /) {
6436 # we correctly received the vma config, so we can disable
6437 # the timeout now for disk allocation (set to 10 minutes, so
6438 # that we always timeout if something goes wrong)
6441 print $fifofh "done\n";
6442 my $tmp = $oldtimeout || 0;
6443 $oldtimeout = undef;
6449 print "restore vma archive: $dbg_cmdstring\n";
6450 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6454 alarm($oldtimeout) if $oldtimeout;
6457 foreach my $devname (keys %$devinfo) {
6458 my $volid = $devinfo->{$devname}->{volid
};
6459 push @$vollist, $volid if $volid;
6462 PVE
::Storage
::deactivate_volumes
($cfg, $vollist);
6470 foreach my $devname (keys %$devinfo) {
6471 my $volid = $devinfo->{$devname}->{volid
};
6474 if ($volid =~ m
|^/|) {
6475 unlink $volid || die 'unlink failed\n';
6477 PVE
::Storage
::vdisk_free
($cfg, $volid);
6479 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6481 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6488 rename($tmpfn, $conffile) ||
6489 die "unable to commit configuration file '$conffile'\n";
6491 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6493 eval { rescan
($vmid, 1); };
6497 sub restore_tar_archive
{
6498 my ($archive, $vmid, $user, $opts) = @_;
6500 if ($archive ne '-') {
6501 my $firstfile = tar_archive_read_firstfile
($archive);
6502 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6503 if $firstfile ne 'qemu-server.conf';
6506 my $storecfg = PVE
::Storage
::config
();
6508 # destroy existing data - keep empty config
6509 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6510 destroy_vm
($storecfg, $vmid, 1) if -f
$vmcfgfn;
6512 my $tocmd = "/usr/lib/qemu-server/qmextract";
6514 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6515 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6516 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6517 $tocmd .= ' --info' if $opts->{info
};
6519 # tar option "xf" does not autodetect compression when read from STDIN,
6520 # so we pipe to zcat
6521 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6522 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6524 my $tmpdir = "/var/tmp/vzdumptmp$$";
6527 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6528 local $ENV{VZDUMP_VMID
} = $vmid;
6529 local $ENV{VZDUMP_USER
} = $user;
6531 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6532 my $tmpfn = "$conffile.$$.tmp";
6534 # disable interrupts (always do cleanups)
6538 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6546 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6548 if ($archive eq '-') {
6549 print "extracting archive from STDIN\n";
6550 run_command
($cmd, input
=> "<&STDIN");
6552 print "extracting archive '$archive'\n";
6556 return if $opts->{info
};
6560 my $statfile = "$tmpdir/qmrestore.stat";
6561 if (my $fd = IO
::File-
>new($statfile, "r")) {
6562 while (defined (my $line = <$fd>)) {
6563 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6564 $map->{$1} = $2 if $1;
6566 print STDERR
"unable to parse line in statfile - $line\n";
6572 my $confsrc = "$tmpdir/qemu-server.conf";
6574 my $srcfd = new IO
::File
($confsrc, "r") ||
6575 die "unable to open file '$confsrc'\n";
6577 my $outfd = new IO
::File
($tmpfn, "w") ||
6578 die "unable to write config for VM $vmid\n";
6580 my $cookie = { netcount
=> 0 };
6581 while (defined (my $line = <$srcfd>)) {
6582 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6594 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6601 rename $tmpfn, $conffile ||
6602 die "unable to commit configuration file '$conffile'\n";
6604 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6606 eval { rescan
($vmid, 1); };
6610 sub foreach_storage_used_by_vm
{
6611 my ($conf, $func) = @_;
6615 foreach_drive
($conf, sub {
6616 my ($ds, $drive) = @_;
6617 return if drive_is_cdrom
($drive);
6619 my $volid = $drive->{file
};
6621 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6622 $sidhash->{$sid} = $sid if $sid;
6625 foreach my $sid (sort keys %$sidhash) {
6630 sub do_snapshots_with_qemu
{
6631 my ($storecfg, $volid) = @_;
6633 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6634 my $scfg = $storecfg->{ids
}->{$storage_name};
6636 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6640 if ($volid =~ m/\.(qcow2|qed)$/){
6647 sub qga_check_running
{
6648 my ($vmid, $nowarn) = @_;
6650 eval { vm_mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6652 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6658 sub template_create
{
6659 my ($vmid, $conf, $disk) = @_;
6661 my $storecfg = PVE
::Storage
::config
();
6663 foreach_drive
($conf, sub {
6664 my ($ds, $drive) = @_;
6666 return if drive_is_cdrom
($drive);
6667 return if $disk && $ds ne $disk;
6669 my $volid = $drive->{file
};
6670 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6672 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6673 $drive->{file
} = $voliddst;
6674 $conf->{$ds} = print_drive
($vmid, $drive);
6675 PVE
::QemuConfig-
>write_config($vmid, $conf);
6679 sub convert_iscsi_path
{
6682 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6687 my $initiator_name = get_initiator_name
();
6689 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6690 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6693 die "cannot convert iscsi path '$path', unkown format\n";
6696 sub qemu_img_convert
{
6697 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6699 my $storecfg = PVE
::Storage
::config
();
6700 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6701 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6703 if ($src_storeid && $dst_storeid) {
6705 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6707 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6708 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6710 my $src_format = qemu_img_format
($src_scfg, $src_volname);
6711 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6713 my $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6714 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6716 my $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6717 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6720 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6721 push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6722 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6723 push @$cmd, '-T', 'none' if $src_scfg->{type
} eq 'zfspool';
6725 if ($src_is_iscsi) {
6726 push @$cmd, '--image-opts';
6727 $src_path = convert_iscsi_path
($src_path);
6729 push @$cmd, '-f', $src_format;
6732 if ($dst_is_iscsi) {
6733 push @$cmd, '--target-image-opts';
6734 $dst_path = convert_iscsi_path
($dst_path);
6736 push @$cmd, '-O', $dst_format;
6739 push @$cmd, $src_path;
6741 if (!$dst_is_iscsi && $is_zero_initialized) {
6742 push @$cmd, "zeroinit:$dst_path";
6744 push @$cmd, $dst_path;
6749 if($line =~ m/\((\S+)\/100\
%\)/){
6751 my $transferred = int($size * $percent / 100);
6752 my $remaining = $size - $transferred;
6754 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6759 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6761 die "copy failed: $err" if $err;
6765 sub qemu_img_format
{
6766 my ($scfg, $volname) = @_;
6768 if ($scfg->{path
} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6775 sub qemu_drive_mirror
{
6776 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6778 $jobs = {} if !$jobs;
6782 $jobs->{"drive-$drive"} = {};
6784 if ($dst_volid =~ /^nbd:/) {
6785 $qemu_target = $dst_volid;
6788 my $storecfg = PVE
::Storage
::config
();
6789 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6791 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6793 $format = qemu_img_format
($dst_scfg, $dst_volname);
6795 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6797 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6800 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6801 $opts->{format
} = $format if $format;
6803 if (defined($bwlimit)) {
6804 $opts->{speed
} = $bwlimit * 1024;
6805 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6807 print "drive mirror is starting for drive-$drive\n";
6810 # if a job already runs for this device we get an error, catch it for cleanup
6811 eval { vm_mon_cmd
($vmid, "drive-mirror", %$opts); };
6813 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6815 die "mirroring error: $err\n";
6818 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6821 sub qemu_drive_mirror_monitor
{
6822 my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6825 my $err_complete = 0;
6828 die "storage migration timed out\n" if $err_complete > 300;
6830 my $stats = vm_mon_cmd
($vmid, "query-block-jobs");
6832 my $running_mirror_jobs = {};
6833 foreach my $stat (@$stats) {
6834 next if $stat->{type
} ne 'mirror';
6835 $running_mirror_jobs->{$stat->{device
}} = $stat;
6838 my $readycounter = 0;
6840 foreach my $job (keys %$jobs) {
6842 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6843 print "$job : finished\n";
6844 delete $jobs->{$job};
6848 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6850 my $busy = $running_mirror_jobs->{$job}->{busy
};
6851 my $ready = $running_mirror_jobs->{$job}->{ready
};
6852 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6853 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6854 my $remaining = $total - $transferred;
6855 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6857 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6860 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6863 last if scalar(keys %$jobs) == 0;
6865 if ($readycounter == scalar(keys %$jobs)) {
6866 print "all mirroring jobs are ready \n";
6867 last if $skipcomplete; #do the complete later
6869 if ($vmiddst && $vmiddst != $vmid) {
6870 my $agent_running = $qga && qga_check_running
($vmid);
6871 if ($agent_running) {
6872 print "freeze filesystem\n";
6873 eval { PVE
::QemuServer
::vm_mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6875 print "suspend vm\n";
6876 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6879 # if we clone a disk for a new target vm, we don't switch the disk
6880 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6882 if ($agent_running) {
6883 print "unfreeze filesystem\n";
6884 eval { PVE
::QemuServer
::vm_mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6886 print "resume vm\n";
6887 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6893 foreach my $job (keys %$jobs) {
6894 # try to switch the disk if source and destination are on the same guest
6895 print "$job: Completing block job...\n";
6897 eval { vm_mon_cmd
($vmid, "block-job-complete", device
=> $job) };
6898 if ($@ =~ m/cannot be completed/) {
6899 print "$job: Block job cannot be completed, try again.\n";
6902 print "$job: Completed successfully.\n";
6903 $jobs->{$job}->{complete
} = 1;
6914 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6915 die "mirroring error: $err";
6920 sub qemu_blockjobs_cancel
{
6921 my ($vmid, $jobs) = @_;
6923 foreach my $job (keys %$jobs) {
6924 print "$job: Cancelling block job\n";
6925 eval { vm_mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6926 $jobs->{$job}->{cancel
} = 1;
6930 my $stats = vm_mon_cmd
($vmid, "query-block-jobs");
6932 my $running_jobs = {};
6933 foreach my $stat (@$stats) {
6934 $running_jobs->{$stat->{device
}} = $stat;
6937 foreach my $job (keys %$jobs) {
6939 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6940 print "$job: Done.\n";
6941 delete $jobs->{$job};
6945 last if scalar(keys %$jobs) == 0;
6952 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6953 $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6958 print "create linked clone of drive $drivename ($drive->{file})\n";
6959 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6960 push @$newvollist, $newvolid;
6963 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6964 $storeid = $storage if $storage;
6966 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6967 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
6969 print "create full clone of drive $drivename ($drive->{file})\n";
6971 if (drive_is_cloudinit
($drive)) {
6972 $name = "vm-$newvmid-cloudinit";
6974 # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
6975 if ($dst_format ne 'raw') {
6976 $name .= ".$dst_format";
6979 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6980 push @$newvollist, $newvolid;
6982 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
6984 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6985 if (!$running || $snapname) {
6986 # TODO: handle bwlimits
6987 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
6990 my $kvmver = get_running_qemu_version
($vmid);
6991 if (!qemu_machine_feature_enabled
(undef, $kvmver, 2, 7)) {
6992 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6993 if $drive->{iothread
};
6996 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7000 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
7003 $disk->{format
} = undef;
7004 $disk->{file
} = $newvolid;
7005 $disk->{size
} = $size;
7010 # this only works if VM is running
7011 sub get_current_qemu_machine
{
7014 my $cmd = { execute
=> 'query-machines', arguments
=> {} };
7015 my $res = vm_qmp_command
($vmid, $cmd);
7017 my ($current, $default);
7018 foreach my $e (@$res) {
7019 $default = $e->{name
} if $e->{'is-default'};
7020 $current = $e->{name
} if $e->{'is-current'};
7023 # fallback to the default machine if current is not supported by qemu
7024 return $current || $default || 'pc';
7027 sub get_running_qemu_version
{
7029 my $cmd = { execute
=> 'query-version', arguments
=> {} };
7030 my $res = vm_qmp_command
($vmid, $cmd);
7031 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7034 sub qemu_machine_feature_enabled
{
7035 my ($machine, $kvmver, $version_major, $version_minor) = @_;
7040 if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7042 $current_major = $3;
7043 $current_minor = $4;
7045 } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7047 $current_major = $1;
7048 $current_minor = $2;
7051 return 1 if $current_major > $version_major ||
7052 ($current_major == $version_major &&
7053 $current_minor >= $version_minor);
7056 sub qemu_machine_pxe
{
7057 my ($vmid, $conf, $machine) = @_;
7059 $machine = PVE
::QemuServer
::get_current_qemu_machine
($vmid) if !$machine;
7061 if ($conf->{machine
} && $conf->{machine
} =~ m/\.pxe$/) {
7068 sub qemu_use_old_bios_files
{
7069 my ($machine_type) = @_;
7071 return if !$machine_type;
7073 my $use_old_bios_files = undef;
7075 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7077 $use_old_bios_files = 1;
7079 my $kvmver = kvm_user_version
();
7080 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7081 # load new efi bios files on migration. So this hack is required to allow
7082 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7083 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7084 $use_old_bios_files = !qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 4);
7087 return ($use_old_bios_files, $machine_type);
7090 sub create_efidisk
($$$$$) {
7091 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7093 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7094 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7096 my $vars_size = PVE
::Tools
::convert_size
(-s
$ovmf_vars, 'b' => 'kb');
7097 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7098 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7100 my $path = PVE
::Storage
::path
($storecfg, $volid);
7102 run_command
(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7104 die "Copying EFI vars image failed: $@" if $@;
7106 return ($volid, $vars_size);
7109 sub vm_iothreads_list
{
7112 my $res = vm_mon_cmd
($vmid, 'query-iothreads');
7115 foreach my $iothread (@$res) {
7116 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7123 my ($conf, $drive) = @_;
7127 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7129 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7135 my $controller = int($drive->{index} / $maxdev);
7136 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single') ?
"virtioscsi" : "scsihw";
7138 return ($maxdev, $controller, $controller_prefix);
7141 sub add_hyperv_enlightenments
{
7142 my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7144 return if $winversion < 6;
7145 return if $bios && $bios eq 'ovmf' && $winversion < 8;
7147 if ($gpu_passthrough || defined($hv_vendor_id)) {
7148 $hv_vendor_id //= 'proxmox';
7149 push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7152 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3)) {
7153 push @$cpuFlags , 'hv_spinlocks=0x1fff';
7154 push @$cpuFlags , 'hv_vapic';
7155 push @$cpuFlags , 'hv_time';
7157 push @$cpuFlags , 'hv_spinlocks=0xffff';
7160 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 6)) {
7161 push @$cpuFlags , 'hv_reset';
7162 push @$cpuFlags , 'hv_vpindex';
7163 push @$cpuFlags , 'hv_runtime';
7166 if ($winversion >= 7) {
7167 push @$cpuFlags , 'hv_relaxed';
7169 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 12)) {
7170 push @$cpuFlags , 'hv_synic';
7171 push @$cpuFlags , 'hv_stimer';
7176 sub windows_version
{
7179 return 0 if !$ostype;
7183 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7185 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7187 } elsif ($ostype =~ m/^win(\d+)$/) {
7194 sub resolve_dst_disk_format
{
7195 my ($storecfg, $storeid, $src_volname, $format) = @_;
7196 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7199 # if no target format is specified, use the source disk format as hint
7201 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7202 $format = qemu_img_format
($scfg, $src_volname);
7208 # test if requested format is supported - else use default
7209 my $supported = grep { $_ eq $format } @$validFormats;
7210 $format = $defFormat if !$supported;
7214 sub resolve_first_disk
{
7216 my @disks = PVE
::QemuServer
::valid_drive_names
();
7218 foreach my $ds (reverse @disks) {
7219 next if !$conf->{$ds};
7220 my $disk = PVE
::QemuServer
::parse_drive
($ds, $conf->{$ds});
7221 next if PVE
::QemuServer
::drive_is_cdrom
($disk);
7228 my ($uuid, $uuid_str);
7229 UUID
::generate
($uuid);
7230 UUID
::unparse
($uuid, $uuid_str);
7234 sub generate_smbios1_uuid
{
7235 return "uuid=".generate_uuid
();
7241 vm_mon_cmd
($vmid, 'nbd-server-stop');
7244 # bash completion helper
7246 sub complete_backup_archives
{
7247 my ($cmdname, $pname, $cvalue) = @_;
7249 my $cfg = PVE
::Storage
::config
();
7253 if ($cvalue =~ m/^([^:]+):/) {
7257 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7260 foreach my $id (keys %$data) {
7261 foreach my $item (@{$data->{$id}}) {
7262 next if $item->{format
} !~ m/^vma\.(gz|lzo)$/;
7263 push @$res, $item->{volid
} if defined($item->{volid
});
7270 my $complete_vmid_full = sub {
7273 my $idlist = vmstatus
();
7277 foreach my $id (keys %$idlist) {
7278 my $d = $idlist->{$id};
7279 if (defined($running)) {
7280 next if $d->{template
};
7281 next if $running && $d->{status
} ne 'running';
7282 next if !$running && $d->{status
} eq 'running';
7291 return &$complete_vmid_full();
7294 sub complete_vmid_stopped
{
7295 return &$complete_vmid_full(0);
7298 sub complete_vmid_running
{
7299 return &$complete_vmid_full(1);
7302 sub complete_storage
{
7304 my $cfg = PVE
::Storage
::config
();
7305 my $ids = $cfg->{ids
};
7308 foreach my $sid (keys %$ids) {
7309 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7310 next if !$ids->{$sid}->{content
}->{images
};