1 package PVE
::QemuServer
;
23 use Storable
qw(dclone);
25 use PVE
::Exception
qw(raise raise_param_exc);
27 use PVE
::Tools
qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
28 use PVE
::JSONSchema
qw(get_standard_option);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
34 use PVE
::RPCEnvironment
;
35 use PVE
::GuestHelpers
;
36 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr);
37 use PVE
::QemuServer
::Memory
;
38 use PVE
::QemuServer
::USB
qw(parse_usb_device);
39 use PVE
::QemuServer
::Cloudinit
;
42 use Time
::HiRes
qw(gettimeofday);
43 use File
::Copy
qw(copy);
46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
49 "$EDK2_FW_BASE/OVMF_CODE.fd",
50 "$EDK2_FW_BASE/OVMF_VARS.fd"
53 "$EDK2_FW_BASE/AAVMF_CODE.fd",
54 "$EDK2_FW_BASE/AAVMF_VARS.fd"
58 my $qemu_snap_storage = { rbd
=> 1 };
60 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
64 # Note about locking: we use flock on the config file protect
65 # against concurent actions.
66 # Aditionaly, we have a 'lock' setting in the config file. This
67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
68 # allowed when such lock is set. But you can ignore this kind of
69 # lock with the --skiplock flag.
71 cfs_register_file
('/qemu-server/',
75 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
76 description
=> "Some command save/restore state from this location.",
82 PVE
::JSONSchema
::register_standard_option
('pve-qm-image-format', {
84 enum
=> [qw(raw cow qcow qed qcow2 vmdk cloop)],
85 description
=> "The drive's backing file's data format.",
89 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
90 description
=> "Specifies the Qemu machine type.",
92 pattern
=> '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
97 #no warnings 'redefine';
100 my ($controller, $vmid, $option, $value) = @_;
102 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
103 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
107 my $nodename = PVE
::INotify
::nodename
();
109 mkdir "/etc/pve/nodes/$nodename";
110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
113 my $var_run_tmpdir = "/var/run/qemu-server";
114 mkdir $var_run_tmpdir;
116 my $lock_dir = "/var/lock/qemu-server";
119 my $cpu_vendor_list = {
121 486 => 'GenuineIntel',
122 pentium
=> 'GenuineIntel',
123 pentium2
=> 'GenuineIntel',
124 pentium3
=> 'GenuineIntel',
125 coreduo
=> 'GenuineIntel',
126 core2duo
=> 'GenuineIntel',
127 Conroe
=> 'GenuineIntel',
128 Penryn
=> 'GenuineIntel',
129 Nehalem
=> 'GenuineIntel',
130 'Nehalem-IBRS' => 'GenuineIntel',
131 Westmere
=> 'GenuineIntel',
132 'Westmere-IBRS' => 'GenuineIntel',
133 SandyBridge
=> 'GenuineIntel',
134 'SandyBridge-IBRS' => 'GenuineIntel',
135 IvyBridge
=> 'GenuineIntel',
136 'IvyBridge-IBRS' => 'GenuineIntel',
137 Haswell
=> 'GenuineIntel',
138 'Haswell-IBRS' => 'GenuineIntel',
139 'Haswell-noTSX' => 'GenuineIntel',
140 'Haswell-noTSX-IBRS' => 'GenuineIntel',
141 Broadwell
=> 'GenuineIntel',
142 'Broadwell-IBRS' => 'GenuineIntel',
143 'Broadwell-noTSX' => 'GenuineIntel',
144 'Broadwell-noTSX-IBRS' => 'GenuineIntel',
145 'Skylake-Client' => 'GenuineIntel',
146 'Skylake-Client-IBRS' => 'GenuineIntel',
147 'Skylake-Server' => 'GenuineIntel',
148 'Skylake-Server-IBRS' => 'GenuineIntel',
151 athlon
=> 'AuthenticAMD',
152 phenom
=> 'AuthenticAMD',
153 Opteron_G1
=> 'AuthenticAMD',
154 Opteron_G2
=> 'AuthenticAMD',
155 Opteron_G3
=> 'AuthenticAMD',
156 Opteron_G4
=> 'AuthenticAMD',
157 Opteron_G5
=> 'AuthenticAMD',
158 EPYC
=> 'AuthenticAMD',
159 'EPYC-IBPB' => 'AuthenticAMD',
161 # generic types, use vendor from host node
170 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb|md-clear)/;
174 description
=> "Emulated CPU type.",
176 enum
=> [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
181 description
=> "Do not identify as a KVM virtual machine.",
188 pattern
=> qr/[a-zA-Z0-9]{1,12}/,
189 format_description
=> 'vendor-id',
190 description
=> 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
194 description
=> "List of additional CPU flags separated by ';'."
195 . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
196 . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb', 'md-clear'.",
197 format_description
=> '+FLAG[;-FLAG...]',
199 pattern
=> qr/$cpu_flag(;$cpu_flag)*/,
208 enum
=> [qw(i6300esb ib700)],
209 description
=> "Watchdog type to emulate.",
210 default => 'i6300esb',
215 enum
=> [qw(reset shutdown poweroff pause debug none)],
216 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
220 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
224 description
=> "Enable/disable Qemu GuestAgent.",
229 fstrim_cloned_disks
=> {
230 description
=> "Run fstrim after cloning/moving a disk.",
239 description
=> "Select the VGA type.",
244 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
247 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
259 description
=> "The size of the file in MB.",
263 pattern
=> '[a-zA-Z0-9\-]+',
265 format_description
=> 'string',
266 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
274 description
=> "Specifies whether a VM will be started during system bootup.",
280 description
=> "Automatic restart after crash (currently ignored).",
285 type
=> 'string', format
=> 'pve-hotplug-features',
286 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
287 default => 'network,disk,usb',
292 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
298 description
=> "Lock/unlock the VM.",
299 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
304 description
=> "Limit of CPU usage.",
305 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
313 description
=> "CPU weight for a VM.",
314 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
322 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
329 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
335 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
343 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
344 "It should not be necessary to set it.",
345 enum
=> PVE
::Tools
::kvmkeymaplist
(),
350 type
=> 'string', format
=> 'dns-name',
351 description
=> "Set a name for the VM. Only used on the configuration web interface.",
356 description
=> "SCSI controller model",
357 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
363 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
368 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
369 description
=> "Specify guest operating system.",
370 verbose_description
=> <<EODESC,
371 Specify guest operating system. This is used to enable special
372 optimization/features for specific operating systems:
375 other;; unspecified OS
376 wxp;; Microsoft Windows XP
377 w2k;; Microsoft Windows 2000
378 w2k3;; Microsoft Windows 2003
379 w2k8;; Microsoft Windows 2008
380 wvista;; Microsoft Windows Vista
381 win7;; Microsoft Windows 7
382 win8;; Microsoft Windows 8/2012/2012r2
383 win10;; Microsoft Windows 10/2016
384 l24;; Linux 2.4 Kernel
385 l26;; Linux 2.6/3.X Kernel
386 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
392 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
393 pattern
=> '[acdn]{1,4}',
398 type
=> 'string', format
=> 'pve-qm-bootdisk',
399 description
=> "Enable booting from specified disk.",
400 pattern
=> '(ide|sata|scsi|virtio)\d+',
405 description
=> "The number of CPUs. Please use option -sockets instead.",
412 description
=> "The number of CPU sockets.",
419 description
=> "The number of cores per socket.",
426 description
=> "Enable/disable NUMA.",
432 description
=> "Enable/disable hugepages memory.",
433 enum
=> [qw(any 2 1024)],
438 description
=> "Number of hotplugged vcpus.",
445 description
=> "Enable/disable ACPI.",
450 description
=> "Enable/disable Qemu GuestAgent and its properties.",
452 format
=> $agent_fmt,
457 description
=> "Enable/disable KVM hardware virtualization.",
463 description
=> "Enable/disable time drift fix.",
469 description
=> "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
474 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
478 type
=> 'string', format
=> $vga_fmt,
479 description
=> "Configure the VGA hardware.",
480 verbose_description
=> "Configure the VGA Hardware. If you want to use ".
481 "high resolution modes (>= 1280x1024x16) you may need to increase " .
482 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
483 "is 'std' for all OS types besides some Windows versions (XP and " .
484 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
485 "display server. For win* OS you can select how many independent " .
486 "displays you want, Linux guests can add displays them self.\n".
487 "You can also run without any graphic card, using a serial device as terminal.",
491 type
=> 'string', format
=> 'pve-qm-watchdog',
492 description
=> "Create a virtual hardware watchdog device.",
493 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled" .
494 " (by a guest action), the watchdog must be periodically polled " .
495 "by an agent inside the guest or else the watchdog will reset " .
496 "the guest (or execute the respective action specified)",
501 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
502 description
=> "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
503 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
506 startup
=> get_standard_option
('pve-startup-order'),
510 description
=> "Enable/disable Template.",
516 description
=> "Arbitrary arguments passed to kvm.",
517 verbose_description
=> <<EODESCR,
518 Arbitrary arguments passed to kvm, for example:
520 args: -no-reboot -no-hpet
522 NOTE: this option is for experts only.
529 description
=> "Enable/disable the USB tablet device.",
530 verbose_description
=> "Enable/disable the USB tablet device. This device is " .
531 "usually needed to allow absolute mouse positioning with VNC. " .
532 "Else the mouse runs out of sync with normal VNC clients. " .
533 "If you're running lots of console-only guests on one host, " .
534 "you may consider disabling this to save some context switches. " .
535 "This is turned off by default if you use spice (-vga=qxl).",
540 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
544 migrate_downtime
=> {
547 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
553 type
=> 'string', format
=> 'pve-qm-ide',
554 typetext
=> '<volume>',
555 description
=> "This is an alias for option -ide2",
559 description
=> "Emulated CPU type.",
563 parent
=> get_standard_option
('pve-snapshot-name', {
565 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
569 description
=> "Timestamp for snapshots.",
575 type
=> 'string', format
=> 'pve-volume-id',
576 description
=> "Reference to a volume which stores the VM state. This is used internally for snapshots.",
578 vmstatestorage
=> get_standard_option
('pve-storage-id', {
579 description
=> "Default storage for VM state volumes/files.",
582 runningmachine
=> get_standard_option
('pve-qemu-machine', {
583 description
=> "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
585 machine
=> get_standard_option
('pve-qemu-machine'),
587 description
=> "Virtual processor architecture. Defaults to the host.",
590 enum
=> [qw(x86_64 aarch64)],
593 description
=> "Specify SMBIOS type 1 fields.",
594 type
=> 'string', format
=> 'pve-qm-smbios1',
601 description
=> "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
607 enum
=> [ qw(seabios ovmf) ],
608 description
=> "Select BIOS implementation.",
609 default => 'seabios',
613 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
614 format_description
=> 'UUID',
615 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
616 verbose_description
=> "The VM generation ID (vmgenid) device exposes a".
617 " 128-bit integer value identifier to the guest OS. This allows to".
618 " notify the guest operating system when the virtual machine is".
619 " executed with a different configuration (e.g. snapshot execution".
620 " or creation from a template). The guest operating system notices".
621 " the change, and is then able to react as appropriate by marking".
622 " its copies of distributed databases as dirty, re-initializing its".
623 " random number generator, etc.\n".
624 "Note that auto-creation only works when done throug API/CLI create".
625 " or update methods, but not when manually editing the config file.",
626 default => "1 (autogenerated)",
631 format
=> 'pve-volume-id',
633 description
=> "Script that will be executed during various steps in the vms lifetime.",
637 format
=> $ivshmem_fmt,
638 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
647 description
=> 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
648 format
=> 'pve-volume-id',
649 format_description
=> 'volume',
654 description
=> 'Specify a custom file containing all network data passed to the VM via cloud-init.',
655 format
=> 'pve-volume-id',
656 format_description
=> 'volume',
661 description
=> 'Specify a custom file containing all user data passed to the VM via cloud-init.',
662 format
=> 'pve-volume-id',
663 format_description
=> 'volume',
666 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
668 my $confdesc_cloudinit = {
672 description
=> 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
673 enum
=> ['configdrive2', 'nocloud'],
678 description
=> "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
683 description
=> 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
688 description
=> 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
689 format
=> 'pve-qm-cicustom',
694 description
=> "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
698 type
=> 'string', format
=> 'address-list',
699 description
=> "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
704 format
=> 'urlencoded',
705 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
709 # what about other qemu settings ?
711 #machine => 'string',
724 ##soundhw => 'string',
726 while (my ($k, $v) = each %$confdesc) {
727 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
730 my $MAX_IDE_DISKS = 4;
731 my $MAX_SCSI_DISKS = 14;
732 my $MAX_VIRTIO_DISKS = 16;
733 my $MAX_SATA_DISKS = 6;
734 my $MAX_USB_DEVICES = 5;
736 my $MAX_UNUSED_DISKS = 256;
737 my $MAX_HOSTPCI_DEVICES = 4;
738 my $MAX_SERIAL_PORTS = 4;
739 my $MAX_PARALLEL_PORTS = 3;
745 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
746 description
=> "CPUs accessing this NUMA node.",
747 format_description
=> "id[-id];...",
751 description
=> "Amount of memory this NUMA node provides.",
756 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
757 description
=> "Host NUMA nodes to use.",
758 format_description
=> "id[-id];...",
763 enum
=> [qw(preferred bind interleave)],
764 description
=> "NUMA allocation policy.",
768 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
771 type
=> 'string', format
=> $numa_fmt,
772 description
=> "NUMA topology.",
774 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
776 for (my $i = 0; $i < $MAX_NUMA; $i++) {
777 $confdesc->{"numa$i"} = $numadesc;
780 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
781 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
782 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
783 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
785 my $net_fmt_bridge_descr = <<__EOD__;
786 Bridge to attach the network device to. The Proxmox VE standard bridge
789 If you do not specify a bridge, we create a kvm user (NATed) network
790 device, which provides DHCP and DNS services. The following addresses
797 The DHCP server assign addresses to the guest starting from 10.0.2.15.
801 macaddr
=> get_standard_option
('mac-addr', {
802 description
=> "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
806 description
=> "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
807 enum
=> $nic_model_list,
810 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
813 description
=> $net_fmt_bridge_descr,
814 format_description
=> 'bridge',
819 minimum
=> 0, maximum
=> 16,
820 description
=> 'Number of packet queues to be used on the device.',
826 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
831 minimum
=> 1, maximum
=> 4094,
832 description
=> 'VLAN tag to apply to packets on this interface.',
837 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
838 description
=> 'VLAN trunks to pass through this interface.',
839 format_description
=> 'vlanid[;vlanid...]',
844 description
=> 'Whether this interface should be protected by the firewall.',
849 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
856 type
=> 'string', format
=> $net_fmt,
857 description
=> "Specify network devices.",
860 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
865 format
=> 'pve-ipv4-config',
866 format_description
=> 'IPv4Format/CIDR',
867 description
=> 'IPv4 address in CIDR format.',
874 format_description
=> 'GatewayIPv4',
875 description
=> 'Default gateway for IPv4 traffic.',
881 format
=> 'pve-ipv6-config',
882 format_description
=> 'IPv6Format/CIDR',
883 description
=> 'IPv6 address in CIDR format.',
890 format_description
=> 'GatewayIPv6',
891 description
=> 'Default gateway for IPv6 traffic.',
896 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
899 type
=> 'string', format
=> 'pve-qm-ipconfig',
900 description
=> <<'EODESCR',
901 cloud-init: Specify IP addresses and gateways for the corresponding interface.
903 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
905 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
906 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
908 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
911 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
913 for (my $i = 0; $i < $MAX_NETS; $i++) {
914 $confdesc->{"net$i"} = $netdesc;
915 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
918 foreach my $key (keys %$confdesc_cloudinit) {
919 $confdesc->{$key} = $confdesc_cloudinit->{$key};
922 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
923 sub verify_volume_id_or_qm_path
{
924 my ($volid, $noerr) = @_;
926 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
930 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
931 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
933 return undef if $noerr;
941 my %drivedesc_base = (
942 volume
=> { alias
=> 'file' },
945 format
=> 'pve-volume-id-or-qm-path',
947 format_description
=> 'volume',
948 description
=> "The drive's backing volume.",
952 enum
=> [qw(cdrom disk)],
953 description
=> "The drive's media type.",
959 description
=> "Force the drive's physical geometry to have a specific cylinder count.",
964 description
=> "Force the drive's physical geometry to have a specific head count.",
969 description
=> "Force the drive's physical geometry to have a specific sector count.",
974 enum
=> [qw(none lba auto)],
975 description
=> "Force disk geometry bios translation mode.",
980 description
=> "Controls qemu's snapshot mode feature."
981 . " If activated, changes made to the disk are temporary and will"
982 . " be discarded when the VM is shutdown.",
987 enum
=> [qw(none writethrough writeback unsafe directsync)],
988 description
=> "The drive's cache mode",
991 format
=> get_standard_option
('pve-qm-image-format'),
994 format
=> 'disk-size',
995 format_description
=> 'DiskSize',
996 description
=> "Disk size. This is purely informational and has no effect.",
1001 description
=> "Whether the drive should be included when making backups.",
1006 description
=> 'Whether the drive should considered for replication jobs.',
1012 enum
=> [qw(ignore report stop)],
1013 description
=> 'Read error action.',
1018 enum
=> [qw(enospc ignore report stop)],
1019 description
=> 'Write error action.',
1024 enum
=> [qw(native threads)],
1025 description
=> 'AIO type to use.',
1030 enum
=> [qw(ignore on)],
1031 description
=> 'Controls whether to pass discard/trim requests to the underlying storage.',
1036 description
=> 'Controls whether to detect and try to optimize writes of zeroes.',
1041 format
=> 'urlencoded',
1042 format_description
=> 'serial',
1043 maxLength
=> 20*3, # *3 since it's %xx url enoded
1044 description
=> "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1049 description
=> 'Mark this locally-managed volume as available on all nodes',
1050 verbose_description
=> "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1056 my %iothread_fmt = ( iothread
=> {
1058 description
=> "Whether to use iothreads for this drive",
1065 format
=> 'urlencoded',
1066 format_description
=> 'model',
1067 maxLength
=> 40*3, # *3 since it's %xx url enoded
1068 description
=> "The drive's reported model name, url-encoded, up to 40 bytes long.",
1076 description
=> "Number of queues.",
1082 my %scsiblock_fmt = (
1085 description
=> "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1094 description
=> "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1102 pattern
=> qr/^(0x)[0-9a-fA-F]{16}/,
1103 format_description
=> 'wwn',
1104 description
=> "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1109 my $add_throttle_desc = sub {
1110 my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1113 format_description
=> $unit,
1114 description
=> "Maximum $what in $longunit.",
1117 $d->{minimum
} = $minimum if defined($minimum);
1118 $drivedesc_base{$key} = $d;
1120 # throughput: (leaky bucket)
1121 $add_throttle_desc->('bps', 'integer', 'r/w speed', 'bps', 'bytes per second');
1122 $add_throttle_desc->('bps_rd', 'integer', 'read speed', 'bps', 'bytes per second');
1123 $add_throttle_desc->('bps_wr', 'integer', 'write speed', 'bps', 'bytes per second');
1124 $add_throttle_desc->('mbps', 'number', 'r/w speed', 'mbps', 'megabytes per second');
1125 $add_throttle_desc->('mbps_rd', 'number', 'read speed', 'mbps', 'megabytes per second');
1126 $add_throttle_desc->('mbps_wr', 'number', 'write speed', 'mbps', 'megabytes per second');
1127 $add_throttle_desc->('iops', 'integer', 'r/w I/O', 'iops', 'operations per second');
1128 $add_throttle_desc->('iops_rd', 'integer', 'read I/O', 'iops', 'operations per second');
1129 $add_throttle_desc->('iops_wr', 'integer', 'write I/O', 'iops', 'operations per second');
1131 # pools: (pool of IO before throttling starts taking effect)
1132 $add_throttle_desc->('mbps_max', 'number', 'unthrottled r/w pool', 'mbps', 'megabytes per second');
1133 $add_throttle_desc->('mbps_rd_max', 'number', 'unthrottled read pool', 'mbps', 'megabytes per second');
1134 $add_throttle_desc->('mbps_wr_max', 'number', 'unthrottled write pool', 'mbps', 'megabytes per second');
1135 $add_throttle_desc->('iops_max', 'integer', 'unthrottled r/w I/O pool', 'iops', 'operations per second');
1136 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool', 'iops', 'operations per second');
1137 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1140 $add_throttle_desc->('bps_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1141 $add_throttle_desc->('bps_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1142 $add_throttle_desc->('bps_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1143 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1144 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1145 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1148 $drivedesc_base{'bps_rd_length'} = { alias
=> 'bps_rd_max_length' };
1149 $drivedesc_base{'bps_wr_length'} = { alias
=> 'bps_wr_max_length' };
1150 $drivedesc_base{'iops_rd_length'} = { alias
=> 'iops_rd_max_length' };
1151 $drivedesc_base{'iops_wr_length'} = { alias
=> 'iops_wr_max_length' };
1159 PVE
::JSONSchema
::register_format
("pve-qm-ide", $ide_fmt);
1163 type
=> 'string', format
=> $ide_fmt,
1164 description
=> "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1166 PVE
::JSONSchema
::register_standard_option
("pve-qm-ide", $idedesc);
1178 type
=> 'string', format
=> $scsi_fmt,
1179 description
=> "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1181 PVE
::JSONSchema
::register_standard_option
("pve-qm-scsi", $scsidesc);
1190 type
=> 'string', format
=> $sata_fmt,
1191 description
=> "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1193 PVE
::JSONSchema
::register_standard_option
("pve-qm-sata", $satadesc);
1201 type
=> 'string', format
=> $virtio_fmt,
1202 description
=> "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1204 PVE
::JSONSchema
::register_standard_option
("pve-qm-virtio", $virtiodesc);
1206 my $alldrive_fmt = {
1217 volume
=> { alias
=> 'file' },
1220 format
=> 'pve-volume-id-or-qm-path',
1222 format_description
=> 'volume',
1223 description
=> "The drive's backing volume.",
1225 format
=> get_standard_option
('pve-qm-image-format'),
1228 format
=> 'disk-size',
1229 format_description
=> 'DiskSize',
1230 description
=> "Disk size. This is purely informational and has no effect.",
1235 my $efidisk_desc = {
1237 type
=> 'string', format
=> $efidisk_fmt,
1238 description
=> "Configure a Disk for storing EFI vars",
1241 PVE
::JSONSchema
::register_standard_option
("pve-qm-efidisk", $efidisk_desc);
1246 type
=> 'string', format
=> 'pve-qm-usb-device',
1247 format_description
=> 'HOSTUSBDEVICE|spice',
1248 description
=> <<EODESCR,
1249 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1251 'bus-port(.port)*' (decimal numbers) or
1252 'vendor_id:product_id' (hexadeciaml numbers) or
1255 You can use the 'lsusb -t' command to list existing usb devices.
1257 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1259 The value 'spice' can be used to add a usb redirection devices for spice.
1265 description
=> "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1272 type
=> 'string', format
=> $usb_fmt,
1273 description
=> "Configure an USB device (n is 0 to 4).",
1275 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1277 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1282 pattern
=> qr/$PCIRE(;$PCIRE)*/,
1283 format_description
=> 'HOSTPCIID[;HOSTPCIID2...]',
1284 description
=> <<EODESCR,
1285 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1286 of PCI virtual functions of the host. HOSTPCIID syntax is:
1288 'bus:dev.func' (hexadecimal numbers)
1290 You can us the 'lspci' command to list existing PCI devices.
1295 description
=> "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1301 pattern
=> '[^,;]+',
1302 format_description
=> 'string',
1303 description
=> "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1308 description
=> "Choose the PCI-express bus (needs the 'q35' machine model).",
1314 description
=> "Enable vfio-vga device support.",
1320 format_description
=> 'string',
1321 pattern
=> '[^/\.:]+',
1323 description
=> <<EODESCR
1324 The type of mediated device to use.
1325 An instance of this type will be created on startup of the VM and
1326 will be cleaned up when the VM stops.
1330 PVE
::JSONSchema
::register_format
('pve-qm-hostpci', $hostpci_fmt);
1334 type
=> 'string', format
=> 'pve-qm-hostpci',
1335 description
=> "Map host PCI devices into guest.",
1336 verbose_description
=> <<EODESCR,
1337 Map host PCI devices into guest.
1339 NOTE: This option allows direct access to host hardware. So it is no longer
1340 possible to migrate such machines - use with special care.
1342 CAUTION: Experimental! User reported problems with this option.
1345 PVE
::JSONSchema
::register_standard_option
("pve-qm-hostpci", $hostpcidesc);
1350 pattern
=> '(/dev/.+|socket)',
1351 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1352 verbose_description
=> <<EODESCR,
1353 Create a serial device inside the VM (n is 0 to 3), and pass through a
1354 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1355 host side (use 'qm terminal' to open a terminal connection).
1357 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1359 CAUTION: Experimental! User reported problems with this option.
1366 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1367 description
=> "Map host parallel devices (n is 0 to 2).",
1368 verbose_description
=> <<EODESCR,
1369 Map host parallel devices (n is 0 to 2).
1371 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1373 CAUTION: Experimental! User reported problems with this option.
1377 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1378 $confdesc->{"parallel$i"} = $paralleldesc;
1381 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1382 $confdesc->{"serial$i"} = $serialdesc;
1385 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
1386 $confdesc->{"hostpci$i"} = $hostpcidesc;
1389 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) {
1390 $drivename_hash->{"ide$i"} = 1;
1391 $confdesc->{"ide$i"} = $idedesc;
1394 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) {
1395 $drivename_hash->{"sata$i"} = 1;
1396 $confdesc->{"sata$i"} = $satadesc;
1399 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) {
1400 $drivename_hash->{"scsi$i"} = 1;
1401 $confdesc->{"scsi$i"} = $scsidesc ;
1404 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) {
1405 $drivename_hash->{"virtio$i"} = 1;
1406 $confdesc->{"virtio$i"} = $virtiodesc;
1409 $drivename_hash->{efidisk0
} = 1;
1410 $confdesc->{efidisk0
} = $efidisk_desc;
1412 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1413 $confdesc->{"usb$i"} = $usbdesc;
1418 type
=> 'string', format
=> 'pve-volume-id',
1419 description
=> "Reference to unused volumes. This is used internally, and should not be modified manually.",
1422 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1423 $confdesc->{"unused$i"} = $unuseddesc;
1426 my $kvm_api_version = 0;
1429 return $kvm_api_version if $kvm_api_version;
1431 open my $fh, '<', '/dev/kvm'
1434 # 0xae00 => KVM_GET_API_VERSION
1435 $kvm_api_version = ioctl($fh, 0xae00, 0);
1437 return $kvm_api_version;
1440 my $kvm_user_version;
1442 sub kvm_user_version
{
1444 return $kvm_user_version if $kvm_user_version;
1446 $kvm_user_version = 'unknown';
1450 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1451 $kvm_user_version = $2;
1455 eval { run_command
("kvm -version", outfunc
=> $code); };
1458 return $kvm_user_version;
1462 sub kernel_has_vhost_net
{
1463 return -c
'/dev/vhost-net';
1466 sub valid_drive_names
{
1467 # order is important - used to autoselect boot disk
1468 return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1469 (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1470 (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1471 (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1475 sub is_valid_drivename
{
1478 return defined($drivename_hash->{$dev});
1483 return defined($confdesc->{$key});
1487 return $nic_model_list;
1490 sub os_list_description
{
1494 wxp
=> 'Windows XP',
1495 w2k
=> 'Windows 2000',
1496 w2k3
=>, 'Windows 2003',
1497 w2k8
=> 'Windows 2008',
1498 wvista
=> 'Windows Vista',
1499 win7
=> 'Windows 7',
1500 win8
=> 'Windows 8/2012',
1501 win10
=> 'Windows 10/2016',
1509 sub get_cdrom_path
{
1511 return $cdrom_path if $cdrom_path;
1513 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1514 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1515 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1519 my ($storecfg, $vmid, $cdrom) = @_;
1521 if ($cdrom eq 'cdrom') {
1522 return get_cdrom_path
();
1523 } elsif ($cdrom eq 'none') {
1525 } elsif ($cdrom =~ m
|^/|) {
1528 return PVE
::Storage
::path
($storecfg, $cdrom);
1532 # try to convert old style file names to volume IDs
1533 sub filename_to_volume_id
{
1534 my ($vmid, $file, $media) = @_;
1536 if (!($file eq 'none' || $file eq 'cdrom' ||
1537 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1539 return undef if $file =~ m
|/|;
1541 if ($media && $media eq 'cdrom') {
1542 $file = "local:iso/$file";
1544 $file = "local:$vmid/$file";
1551 sub verify_media_type
{
1552 my ($opt, $vtype, $media) = @_;
1557 if ($media eq 'disk') {
1559 } elsif ($media eq 'cdrom') {
1562 die "internal error";
1565 return if ($vtype eq $etype);
1567 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1570 sub cleanup_drive_path
{
1571 my ($opt, $storecfg, $drive) = @_;
1573 # try to convert filesystem paths to volume IDs
1575 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1576 ($drive->{file
} !~ m
|^/dev/.+|) &&
1577 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1578 ($drive->{file
} !~ m/^\d+$/)) {
1579 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1580 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1581 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1582 verify_media_type
($opt, $vtype, $drive->{media
});
1583 $drive->{file
} = $volid;
1586 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1589 sub parse_hotplug_features
{
1594 return $res if $data eq '0';
1596 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1598 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1599 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1602 die "invalid hotplug feature '$feature'\n";
1608 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1609 sub pve_verify_hotplug_features
{
1610 my ($value, $noerr) = @_;
1612 return $value if parse_hotplug_features
($value);
1614 return undef if $noerr;
1616 die "unable to parse hotplug option\n";
1619 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1620 # [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1621 # [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1622 # [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1623 # [,iothread=on][,serial=serial][,model=model]
1626 my ($key, $data) = @_;
1628 my ($interface, $index);
1630 if ($key =~ m/^([^\d]+)(\d+)$/) {
1637 my $desc = $key =~ /^unused\d+$/ ?
$alldrive_fmt
1638 : $confdesc->{$key}->{format
};
1640 warn "invalid drive key: $key\n";
1643 my $res = eval { PVE
::JSONSchema
::parse_property_string
($desc, $data) };
1644 return undef if !$res;
1645 $res->{interface
} = $interface;
1646 $res->{index} = $index;
1649 foreach my $opt (qw(bps bps_rd bps_wr)) {
1650 if (my $bps = defined(delete $res->{$opt})) {
1651 if (defined($res->{"m$opt"})) {
1652 warn "both $opt and m$opt specified\n";
1656 $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1660 # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1661 for my $requirement (
1662 [mbps_max
=> 'mbps'],
1663 [mbps_rd_max
=> 'mbps_rd'],
1664 [mbps_wr_max
=> 'mbps_wr'],
1665 [miops_max
=> 'miops'],
1666 [miops_rd_max
=> 'miops_rd'],
1667 [miops_wr_max
=> 'miops_wr'],
1668 [bps_max_length
=> 'mbps_max'],
1669 [bps_rd_max_length
=> 'mbps_rd_max'],
1670 [bps_wr_max_length
=> 'mbps_wr_max'],
1671 [iops_max_length
=> 'iops_max'],
1672 [iops_rd_max_length
=> 'iops_rd_max'],
1673 [iops_wr_max_length
=> 'iops_wr_max']) {
1674 my ($option, $requires) = @$requirement;
1675 if ($res->{$option} && !$res->{$requires}) {
1676 warn "$option requires $requires\n";
1681 return undef if $error;
1683 return undef if $res->{mbps_rd
} && $res->{mbps
};
1684 return undef if $res->{mbps_wr
} && $res->{mbps
};
1685 return undef if $res->{iops_rd
} && $res->{iops
};
1686 return undef if $res->{iops_wr
} && $res->{iops
};
1688 if ($res->{media
} && ($res->{media
} eq 'cdrom')) {
1689 return undef if $res->{snapshot
} || $res->{trans
} || $res->{format
};
1690 return undef if $res->{heads
} || $res->{secs
} || $res->{cyls
};
1691 return undef if $res->{interface
} eq 'virtio';
1694 if (my $size = $res->{size
}) {
1695 return undef if !defined($res->{size
} = PVE
::JSONSchema
::parse_size
($size));
1702 my ($vmid, $drive) = @_;
1703 my $data = { %$drive };
1704 delete $data->{$_} for qw(index interface);
1705 return PVE
::JSONSchema
::print_property_string
($data, $alldrive_fmt);
1709 my($fh, $noerr) = @_;
1712 my $SG_GET_VERSION_NUM = 0x2282;
1714 my $versionbuf = "\x00" x
8;
1715 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1717 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1720 my $version = unpack("I", $versionbuf);
1721 if ($version < 30000) {
1722 die "scsi generic interface too old\n" if !$noerr;
1726 my $buf = "\x00" x
36;
1727 my $sensebuf = "\x00" x
8;
1728 my $cmd = pack("C x3 C x1", 0x12, 36);
1730 # see /usr/include/scsi/sg.h
1731 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1733 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1734 length($sensebuf), 0, length($buf), $buf,
1735 $cmd, $sensebuf, 6000);
1737 $ret = ioctl($fh, $SG_IO, $packet);
1739 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1743 my @res = unpack($sg_io_hdr_t, $packet);
1744 if ($res[17] || $res[18]) {
1745 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1750 (my $byte0, my $byte1, $res->{vendor
},
1751 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1753 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1754 $res->{type
} = $byte0 & 31;
1762 my $fh = IO
::File-
>new("+<$path") || return undef;
1763 my $res = scsi_inquiry
($fh, 1);
1769 sub machine_type_is_q35
{
1772 return $conf->{machine
} && ($conf->{machine
} =~ m/q35/) ?
1 : 0;
1775 sub print_tabletdevice_full
{
1776 my ($conf, $arch) = @_;
1778 my $q35 = machine_type_is_q35
($conf);
1780 # we use uhci for old VMs because tablet driver was buggy in older qemu
1782 if (machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1788 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1791 sub print_keyboarddevice_full
{
1792 my ($conf, $arch, $machine) = @_;
1794 return undef if $arch ne 'aarch64';
1796 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1799 sub print_drivedevice_full
{
1800 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1805 if ($drive->{interface
} eq 'virtio') {
1806 my $pciaddr = print_pci_addr
("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1807 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1808 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread
};
1809 } elsif ($drive->{interface
} eq 'scsi') {
1811 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1812 my $unit = $drive->{index} % $maxdev;
1813 my $devicetype = 'hd';
1815 if (drive_is_cdrom
($drive)) {
1818 if ($drive->{file
} =~ m
|^/|) {
1819 $path = $drive->{file
};
1820 if (my $info = path_is_scsi
($path)) {
1821 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1822 $devicetype = 'block';
1823 } elsif ($info->{type
} == 1) { # tape
1824 $devicetype = 'generic';
1828 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1831 if($path =~ m/^iscsi\:\/\
//){
1832 $devicetype = 'generic';
1836 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1837 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1839 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1842 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1843 $device .= ",rotation_rate=1";
1845 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1847 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1848 my $maxdev = ($drive->{interface
} eq 'sata') ?
$MAX_SATA_DISKS : 2;
1849 my $controller = int($drive->{index} / $maxdev);
1850 my $unit = $drive->{index} % $maxdev;
1851 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1853 $device = "ide-$devicetype";
1854 if ($drive->{interface
} eq 'ide') {
1855 $device .= ",bus=ide.$controller,unit=$unit";
1857 $device .= ",bus=ahci$controller.$unit";
1859 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1861 if ($devicetype eq 'hd') {
1862 if (my $model = $drive->{model
}) {
1863 $model = URI
::Escape
::uri_unescape
($model);
1864 $device .= ",model=$model";
1866 if ($drive->{ssd
}) {
1867 $device .= ",rotation_rate=1";
1870 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1871 } elsif ($drive->{interface
} eq 'usb') {
1873 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1875 die "unsupported interface type";
1878 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1880 if (my $serial = $drive->{serial
}) {
1881 $serial = URI
::Escape
::uri_unescape
($serial);
1882 $device .= ",serial=$serial";
1889 sub get_initiator_name
{
1892 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return undef;
1893 while (defined(my $line = <$fh>)) {
1894 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1903 sub print_drive_full
{
1904 my ($storecfg, $vmid, $drive) = @_;
1907 my $volid = $drive->{file
};
1910 if (drive_is_cdrom
($drive)) {
1911 $path = get_iso_path
($storecfg, $vmid, $volid);
1913 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1915 $path = PVE
::Storage
::path
($storecfg, $volid);
1916 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1917 $format = qemu_img_format
($scfg, $volname);
1925 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1926 foreach my $o (@qemu_drive_options) {
1927 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1930 # snapshot only accepts on|off
1931 if (defined($drive->{snapshot
})) {
1932 my $v = $drive->{snapshot
} ?
'on' : 'off';
1933 $opts .= ",snapshot=$v";
1936 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1937 my ($dir, $qmpname) = @$type;
1938 if (my $v = $drive->{"mbps$dir"}) {
1939 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1941 if (my $v = $drive->{"mbps${dir}_max"}) {
1942 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1944 if (my $v = $drive->{"bps${dir}_max_length"}) {
1945 $opts .= ",throttling.bps$qmpname-max-length=$v";
1947 if (my $v = $drive->{"iops${dir}"}) {
1948 $opts .= ",throttling.iops$qmpname=$v";
1950 if (my $v = $drive->{"iops${dir}_max"}) {
1951 $opts .= ",throttling.iops$qmpname-max=$v";
1953 if (my $v = $drive->{"iops${dir}_max_length"}) {
1954 $opts .= ",throttling.iops$qmpname-max-length=$v";
1958 $opts .= ",format=$format" if $format && !$drive->{format
};
1960 my $cache_direct = 0;
1962 if (my $cache = $drive->{cache
}) {
1963 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1964 } elsif (!drive_is_cdrom
($drive)) {
1965 $opts .= ",cache=none";
1969 # aio native works only with O_DIRECT
1970 if (!$drive->{aio
}) {
1972 $opts .= ",aio=native";
1974 $opts .= ",aio=threads";
1978 if (!drive_is_cdrom
($drive)) {
1980 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1981 $detectzeroes = 'off';
1982 } elsif ($drive->{discard
}) {
1983 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1985 # This used to be our default with discard not being specified:
1986 $detectzeroes = 'on';
1988 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1991 my $pathinfo = $path ?
"file=$path," : '';
1993 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1996 sub print_netdevice_full
{
1997 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1999 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
2001 my $device = $net->{model
};
2002 if ($net->{model
} eq 'virtio') {
2003 $device = 'virtio-net-pci';
2006 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
2007 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2008 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
2009 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2010 my $vectors = $net->{queues
} * 2 + 2;
2011 $tmpstr .= ",vectors=$vectors,mq=on";
2013 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
2015 if ($use_old_bios_files) {
2017 if ($device eq 'virtio-net-pci') {
2018 $romfile = 'pxe-virtio.rom';
2019 } elsif ($device eq 'e1000') {
2020 $romfile = 'pxe-e1000.rom';
2021 } elsif ($device eq 'ne2k') {
2022 $romfile = 'pxe-ne2k_pci.rom';
2023 } elsif ($device eq 'pcnet') {
2024 $romfile = 'pxe-pcnet.rom';
2025 } elsif ($device eq 'rtl8139') {
2026 $romfile = 'pxe-rtl8139.rom';
2028 $tmpstr .= ",romfile=$romfile" if $romfile;
2034 sub print_netdev_full
{
2035 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2038 if ($netid =~ m/^net(\d+)$/) {
2042 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
2044 my $ifname = "tap${vmid}i$i";
2046 # kvm uses TUNSETIFF ioctl, and that limits ifname length
2047 die "interface name '$ifname' is too long (max 15 character)\n"
2048 if length($ifname) >= 16;
2050 my $vhostparam = '';
2051 if (is_native
($arch)) {
2052 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
2055 my $vmname = $conf->{name
} || "vm$vmid";
2058 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
2060 if ($net->{bridge
}) {
2061 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2063 $netdev = "type=user,id=$netid,hostname=$vmname";
2066 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
2072 sub print_cpu_device
{
2073 my ($conf, $id) = @_;
2075 my $kvm = $conf->{kvm
} // 1;
2076 my $cpu = $kvm ?
"kvm64" : "qemu64";
2077 if (my $cputype = $conf->{cpu
}) {
2078 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
2079 or die "Cannot parse cpu description: $cputype\n";
2080 $cpu = $cpuconf->{cputype
};
2083 my $cores = $conf->{cores
} || 1;
2085 my $current_core = ($id - 1) % $cores;
2086 my $current_socket = int(($id - 1 - $current_core)/$cores);
2088 return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2092 'cirrus' => 'cirrus-vga',
2094 'vmware' => 'vmware-svga',
2095 'virtio' => 'virtio-vga',
2098 sub print_vga_device
{
2099 my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2101 my $type = $vga_map->{$vga->{type
}};
2102 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2103 $type = 'virtio-gpu';
2105 my $vgamem_mb = $vga->{memory
};
2107 $type = $id ?
'qxl' : 'qxl-vga';
2109 die "no devicetype for $vga->{type}\n" if !$type;
2113 if ($vga->{type
} eq 'virtio') {
2114 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
2115 $memory = ",max_hostmem=$bytes";
2117 # from https://www.spice-space.org/multiple-monitors.html
2118 $memory = ",vgamem_mb=$vga->{memory}";
2119 my $ram = $vgamem_mb * 4;
2120 my $vram = $vgamem_mb * 2;
2121 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2123 $memory = ",vgamem_mb=$vga->{memory}";
2125 } elsif ($qxlnum && $id) {
2126 $memory = ",ram_size=67108864,vram_size=33554432";
2129 my $q35 = machine_type_is_q35
($conf);
2130 my $vgaid = "vga" . ($id // '');
2133 if ($q35 && $vgaid eq 'vga') {
2134 # the first display uses pcie.0 bus on q35 machines
2135 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
2137 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
2140 return "$type,id=${vgaid}${memory}${pciaddr}";
2143 sub drive_is_cloudinit
{
2145 return $drive->{file
} =~ m
@[:/]vm-\d
+-cloudinit
(?
:\
.$QEMU_FORMAT_RE)?
$@;
2148 sub drive_is_cdrom
{
2149 my ($drive, $exclude_cloudinit) = @_;
2151 return 0 if $exclude_cloudinit && drive_is_cloudinit
($drive);
2153 return $drive && $drive->{media
} && ($drive->{media
} eq 'cdrom');
2157 sub parse_number_sets
{
2160 foreach my $part (split(/;/, $set)) {
2161 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2162 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2163 push @$res, [ $1, $2 ];
2165 die "invalid range: $part\n";
2174 my $res = PVE
::JSONSchema
::parse_property_string
($numa_fmt, $data);
2175 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
2176 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
2183 return undef if !$value;
2185 my $res = PVE
::JSONSchema
::parse_property_string
($hostpci_fmt, $value);
2187 my @idlist = split(/;/, $res->{host
});
2188 delete $res->{host
};
2189 foreach my $id (@idlist) {
2190 if ($id =~ m/\./) { # full id 00:00.1
2191 push @{$res->{pciid
}}, {
2194 } else { # partial id 00:00
2195 $res->{pciid
} = PVE
::SysFSTools
::lspci
($id);
2201 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2205 my $res = eval { PVE
::JSONSchema
::parse_property_string
($net_fmt, $data) };
2210 if (!defined($res->{macaddr
})) {
2211 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
2212 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
2217 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2218 sub parse_ipconfig
{
2221 my $res = eval { PVE
::JSONSchema
::parse_property_string
($ipconfig_fmt, $data) };
2227 if ($res->{gw
} && !$res->{ip
}) {
2228 warn 'gateway specified without specifying an IP address';
2231 if ($res->{gw6
} && !$res->{ip6
}) {
2232 warn 'IPv6 gateway specified without specifying an IPv6 address';
2235 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
2236 warn 'gateway specified together with DHCP';
2239 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
2241 warn "IPv6 gateway specified together with $res->{ip6} address";
2245 if (!$res->{ip
} && !$res->{ip6
}) {
2246 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
2255 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
2258 sub add_random_macs
{
2259 my ($settings) = @_;
2261 foreach my $opt (keys %$settings) {
2262 next if $opt !~ m/^net(\d+)$/;
2263 my $net = parse_net
($settings->{$opt});
2265 $settings->{$opt} = print_net
($net);
2269 sub vm_is_volid_owner
{
2270 my ($storecfg, $vmid, $volid) = @_;
2272 if ($volid !~ m
|^/|) {
2274 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
2275 if ($owner && ($owner == $vmid)) {
2283 sub split_flagged_list
{
2284 my $text = shift || '';
2285 $text =~ s/[,;]/ /g;
2287 return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2290 sub join_flagged_list
{
2291 my ($how, $lst) = @_;
2292 join $how, map { $lst->{$_} . $_ } keys %$lst;
2295 sub vmconfig_delete_pending_option
{
2296 my ($conf, $key, $force) = @_;
2298 delete $conf->{pending
}->{$key};
2299 my $pending_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
2300 $pending_delete_hash->{$key} = $force ?
'!' : '';
2301 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
2304 sub vmconfig_undelete_pending_option
{
2305 my ($conf, $key) = @_;
2307 my $pending_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
2308 delete $pending_delete_hash->{$key};
2310 if (%$pending_delete_hash) {
2311 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
2313 delete $conf->{pending
}->{delete};
2317 sub vmconfig_register_unused_drive
{
2318 my ($storecfg, $vmid, $conf, $drive) = @_;
2320 if (drive_is_cloudinit
($drive)) {
2321 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2323 } elsif (!drive_is_cdrom
($drive)) {
2324 my $volid = $drive->{file
};
2325 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2326 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2331 sub vmconfig_cleanup_pending
{
2334 # remove pending changes when nothing changed
2336 foreach my $opt (keys %{$conf->{pending
}}) {
2337 if (defined($conf->{$opt}) && ($conf->{pending
}->{$opt} eq $conf->{$opt})) {
2339 delete $conf->{pending
}->{$opt};
2343 my $current_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
2344 my $pending_delete_hash = {};
2345 while (my ($opt, $force) = each %$current_delete_hash) {
2346 if (defined($conf->{$opt})) {
2347 $pending_delete_hash->{$opt} = $force;
2353 if (%$pending_delete_hash) {
2354 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
2356 delete $conf->{pending
}->{delete};
2362 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2366 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2367 format_description
=> 'UUID',
2368 description
=> "Set SMBIOS1 UUID.",
2373 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2374 format_description
=> 'Base64 encoded string',
2375 description
=> "Set SMBIOS1 version.",
2380 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2381 format_description
=> 'Base64 encoded string',
2382 description
=> "Set SMBIOS1 serial number.",
2387 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2388 format_description
=> 'Base64 encoded string',
2389 description
=> "Set SMBIOS1 manufacturer.",
2394 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2395 format_description
=> 'Base64 encoded string',
2396 description
=> "Set SMBIOS1 product ID.",
2401 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2402 format_description
=> 'Base64 encoded string',
2403 description
=> "Set SMBIOS1 SKU string.",
2408 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2409 format_description
=> 'Base64 encoded string',
2410 description
=> "Set SMBIOS1 family string.",
2415 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2423 my $res = eval { PVE
::JSONSchema
::parse_property_string
($smbios1_fmt, $data) };
2430 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2433 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2435 PVE
::JSONSchema
::register_format
('pve-qm-bootdisk', \
&verify_bootdisk
);
2436 sub verify_bootdisk
{
2437 my ($value, $noerr) = @_;
2439 return $value if is_valid_drivename
($value);
2441 return undef if $noerr;
2443 die "invalid boot disk '$value'\n";
2446 sub parse_watchdog
{
2449 return undef if !$value;
2451 my $res = eval { PVE
::JSONSchema
::parse_property_string
($watchdog_fmt, $value) };
2456 sub parse_guest_agent
{
2459 return {} if !defined($value->{agent
});
2461 my $res = eval { PVE
::JSONSchema
::parse_property_string
($agent_fmt, $value->{agent
}) };
2464 # if the agent is disabled ignore the other potentially set properties
2465 return {} if !$res->{enabled
};
2472 return {} if !$value;
2473 my $res = eval { PVE
::JSONSchema
::parse_property_string
($vga_fmt, $value) };
2478 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2479 sub verify_usb_device
{
2480 my ($value, $noerr) = @_;
2482 return $value if parse_usb_device
($value);
2484 return undef if $noerr;
2486 die "unable to parse usb device\n";
2489 # add JSON properties for create and set function
2490 sub json_config_properties
{
2493 foreach my $opt (keys %$confdesc) {
2494 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2495 $prop->{$opt} = $confdesc->{$opt};
2501 # return copy of $confdesc_cloudinit to generate documentation
2502 sub cloudinit_config_properties
{
2504 return dclone
($confdesc_cloudinit);
2508 my ($key, $value) = @_;
2510 die "unknown setting '$key'\n" if !$confdesc->{$key};
2512 my $type = $confdesc->{$key}->{type
};
2514 if (!defined($value)) {
2515 die "got undefined value\n";
2518 if ($value =~ m/[\n\r]/) {
2519 die "property contains a line feed\n";
2522 if ($type eq 'boolean') {
2523 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2524 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2525 die "type check ('boolean') failed - got '$value'\n";
2526 } elsif ($type eq 'integer') {
2527 return int($1) if $value =~ m/^(\d+)$/;
2528 die "type check ('integer') failed - got '$value'\n";
2529 } elsif ($type eq 'number') {
2530 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2531 die "type check ('number') failed - got '$value'\n";
2532 } elsif ($type eq 'string') {
2533 if (my $fmt = $confdesc->{$key}->{format
}) {
2534 PVE
::JSONSchema
::check_format
($fmt, $value);
2537 $value =~ s/^\"(.*)\"$/$1/;
2540 die "internal error"
2547 my $conf = PVE
::QemuConfig-
>config_file($vmid);
2548 utime undef, undef, $conf;
2552 my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2554 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
2556 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2558 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2560 if ($conf->{template
}) {
2561 # check if any base image is still used by a linked clone
2562 foreach_drive
($conf, sub {
2563 my ($ds, $drive) = @_;
2565 return if drive_is_cdrom
($drive);
2567 my $volid = $drive->{file
};
2569 return if !$volid || $volid =~ m
|^/|;
2571 die "base volume '$volid' is still in use by linked cloned\n"
2572 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2577 # only remove disks owned by this VM
2578 foreach_drive
($conf, sub {
2579 my ($ds, $drive) = @_;
2581 return if drive_is_cdrom
($drive, 1);
2583 my $volid = $drive->{file
};
2585 return if !$volid || $volid =~ m
|^/|;
2587 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2588 return if !$path || !$owner || ($owner != $vmid);
2591 PVE
::Storage
::vdisk_free
($storecfg, $volid);
2593 warn "Could not remove disk '$volid', check manually: $@" if $@;
2597 if ($keep_empty_config) {
2598 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
2603 # also remove unused disk
2605 my $dl = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2608 PVE
::Storage
::foreach_volid
($dl, sub {
2609 my ($volid, $sid, $volname, $d) = @_;
2610 PVE
::Storage
::vdisk_free
($storecfg, $volid);
2619 sub parse_vm_config
{
2620 my ($filename, $raw) = @_;
2622 return undef if !defined($raw);
2625 digest
=> Digest
::SHA
::sha1_hex
($raw),
2630 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2631 || die "got strange filename '$filename'";
2639 my @lines = split(/\n/, $raw);
2640 foreach my $line (@lines) {
2641 next if $line =~ m/^\s*$/;
2643 if ($line =~ m/^\[PENDING\]\s*$/i) {
2644 $section = 'pending';
2645 if (defined($descr)) {
2647 $conf->{description
} = $descr;
2650 $conf = $res->{$section} = {};
2653 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2655 if (defined($descr)) {
2657 $conf->{description
} = $descr;
2660 $conf = $res->{snapshots
}->{$section} = {};
2664 if ($line =~ m/^\#(.*)\s*$/) {
2665 $descr = '' if !defined($descr);
2666 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2670 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2671 $descr = '' if !defined($descr);
2672 $descr .= PVE
::Tools
::decode_text
($2);
2673 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2674 $conf->{snapstate
} = $1;
2675 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2678 $conf->{$key} = $value;
2679 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2681 if ($section eq 'pending') {
2682 $conf->{delete} = $value; # we parse this later
2684 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2686 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2689 eval { $value = check_type
($key, $value); };
2691 warn "vm $vmid - unable to parse value of '$key' - $@";
2693 $key = 'ide2' if $key eq 'cdrom';
2694 my $fmt = $confdesc->{$key}->{format
};
2695 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2696 my $v = parse_drive
($key, $value);
2697 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2698 $v->{file
} = $volid;
2699 $value = print_drive
($vmid, $v);
2701 warn "vm $vmid - unable to parse value of '$key'\n";
2706 $conf->{$key} = $value;
2711 if (defined($descr)) {
2713 $conf->{description
} = $descr;
2715 delete $res->{snapstate
}; # just to be sure
2720 sub write_vm_config
{
2721 my ($filename, $conf) = @_;
2723 delete $conf->{snapstate
}; # just to be sure
2725 if ($conf->{cdrom
}) {
2726 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2727 $conf->{ide2
} = $conf->{cdrom
};
2728 delete $conf->{cdrom
};
2731 # we do not use 'smp' any longer
2732 if ($conf->{sockets
}) {
2733 delete $conf->{smp
};
2734 } elsif ($conf->{smp
}) {
2735 $conf->{sockets
} = $conf->{smp
};
2736 delete $conf->{cores
};
2737 delete $conf->{smp
};
2740 my $used_volids = {};
2742 my $cleanup_config = sub {
2743 my ($cref, $pending, $snapname) = @_;
2745 foreach my $key (keys %$cref) {
2746 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2747 $key eq 'snapstate' || $key eq 'pending';
2748 my $value = $cref->{$key};
2749 if ($key eq 'delete') {
2750 die "propertry 'delete' is only allowed in [PENDING]\n"
2752 # fixme: check syntax?
2755 eval { $value = check_type
($key, $value); };
2756 die "unable to parse value of '$key' - $@" if $@;
2758 $cref->{$key} = $value;
2760 if (!$snapname && is_valid_drivename
($key)) {
2761 my $drive = parse_drive
($key, $value);
2762 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2767 &$cleanup_config($conf);
2769 &$cleanup_config($conf->{pending
}, 1);
2771 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2772 die "internal error" if $snapname eq 'pending';
2773 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2776 # remove 'unusedX' settings if we re-add a volume
2777 foreach my $key (keys %$conf) {
2778 my $value = $conf->{$key};
2779 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2780 delete $conf->{$key};
2784 my $generate_raw_config = sub {
2785 my ($conf, $pending) = @_;
2789 # add description as comment to top of file
2790 if (defined(my $descr = $conf->{description
})) {
2792 foreach my $cl (split(/\n/, $descr)) {
2793 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2796 $raw .= "#\n" if $pending;
2800 foreach my $key (sort keys %$conf) {
2801 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2802 $raw .= "$key: $conf->{$key}\n";
2807 my $raw = &$generate_raw_config($conf);
2809 if (scalar(keys %{$conf->{pending
}})){
2810 $raw .= "\n[PENDING]\n";
2811 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2814 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2815 $raw .= "\n[$snapname]\n";
2816 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2826 # we use static defaults from our JSON schema configuration
2827 foreach my $key (keys %$confdesc) {
2828 if (defined(my $default = $confdesc->{$key}->{default})) {
2829 $res->{$key} = $default;
2837 my $vmlist = PVE
::Cluster
::get_vmlist
();
2839 return $res if !$vmlist || !$vmlist->{ids
};
2840 my $ids = $vmlist->{ids
};
2842 foreach my $vmid (keys %$ids) {
2843 my $d = $ids->{$vmid};
2844 next if !$d->{node
} || $d->{node
} ne $nodename;
2845 next if !$d->{type
} || $d->{type
} ne 'qemu';
2846 $res->{$vmid}->{exists} = 1;
2851 # test if VM uses local resources (to prevent migration)
2852 sub check_local_resources
{
2853 my ($conf, $noerr) = @_;
2857 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2858 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2860 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2862 foreach my $k (keys %$conf) {
2863 next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2864 # sockets are safe: they will recreated be on the target side post-migrate
2865 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2866 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2869 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2874 # check if used storages are available on all nodes (use by migrate)
2875 sub check_storage_availability
{
2876 my ($storecfg, $conf, $node) = @_;
2878 foreach_drive
($conf, sub {
2879 my ($ds, $drive) = @_;
2881 my $volid = $drive->{file
};
2884 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2887 # check if storage is available on both nodes
2888 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2889 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2893 # list nodes where all VM images are available (used by has_feature API)
2895 my ($conf, $storecfg) = @_;
2897 my $nodelist = PVE
::Cluster
::get_nodelist
();
2898 my $nodehash = { map { $_ => 1 } @$nodelist };
2899 my $nodename = PVE
::INotify
::nodename
();
2901 foreach_drive
($conf, sub {
2902 my ($ds, $drive) = @_;
2904 my $volid = $drive->{file
};
2907 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2909 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2910 if ($scfg->{disable
}) {
2912 } elsif (my $avail = $scfg->{nodes
}) {
2913 foreach my $node (keys %$nodehash) {
2914 delete $nodehash->{$node} if !$avail->{$node};
2916 } elsif (!$scfg->{shared
}) {
2917 foreach my $node (keys %$nodehash) {
2918 delete $nodehash->{$node} if $node ne $nodename
2928 my ($pidfile, $pid) = @_;
2930 my $fh = IO
::File-
>new("/proc/$pid/cmdline", "r");
2934 return undef if !$line;
2935 my @param = split(/\0/, $line);
2937 my $cmd = $param[0];
2938 return if !$cmd || ($cmd !~ m
|kvm
$| && $cmd !~ m
@(?
:^|/)qemu-system-[^/]+$@);
2940 for (my $i = 0; $i < scalar (@param); $i++) {
2943 if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2944 my $p = $param[$i+1];
2945 return 1 if $p && ($p eq $pidfile);
2954 my ($vmid, $nocheck, $node) = @_;
2956 my $filename = PVE
::QemuConfig-
>config_file($vmid, $node);
2958 die "unable to find configuration file for VM $vmid - no such machine\n"
2959 if !$nocheck && ! -f
$filename;
2961 my $pidfile = pidfile_name
($vmid);
2963 if (my $fd = IO
::File-
>new("<$pidfile")) {
2968 my $mtime = $st->mtime;
2969 if ($mtime > time()) {
2970 warn "file '$filename' modified in future\n";
2973 if ($line =~ m/^(\d+)$/) {
2975 if (check_cmdline
($pidfile, $pid)) {
2976 if (my $pinfo = PVE
::ProcFSTools
::check_process_running
($pid)) {
2988 my $vzlist = config_list
();
2990 my $fd = IO
::Dir-
>new($var_run_tmpdir) || return $vzlist;
2992 while (defined(my $de = $fd->read)) {
2993 next if $de !~ m/^(\d+)\.pid$/;
2995 next if !defined($vzlist->{$vmid});
2996 if (my $pid = check_running
($vmid)) {
2997 $vzlist->{$vmid}->{pid
} = $pid;
3005 my ($storecfg, $conf) = @_;
3007 my $bootdisk = $conf->{bootdisk
};
3008 return undef if !$bootdisk;
3009 return undef if !is_valid_drivename
($bootdisk);
3011 return undef if !$conf->{$bootdisk};
3013 my $drive = parse_drive
($bootdisk, $conf->{$bootdisk});
3014 return undef if !defined($drive);
3016 return undef if drive_is_cdrom
($drive);
3018 my $volid = $drive->{file
};
3019 return undef if !$volid;
3021 return $drive->{size
};
3024 our $vmstatus_return_properties = {
3025 vmid
=> get_standard_option
('pve-vmid'),
3027 description
=> "Qemu process status.",
3029 enum
=> ['stopped', 'running'],
3032 description
=> "Maximum memory in bytes.",
3035 renderer
=> 'bytes',
3038 description
=> "Root disk size in bytes.",
3041 renderer
=> 'bytes',
3044 description
=> "VM name.",
3049 description
=> "Qemu QMP agent status.",
3054 description
=> "PID of running qemu process.",
3059 description
=> "Uptime.",
3062 renderer
=> 'duration',
3065 description
=> "Maximum usable CPUs.",
3070 description
=> "The current config lock, if any.",
3076 my $last_proc_pid_stat;
3078 # get VM status information
3079 # This must be fast and should not block ($full == false)
3080 # We only query KVM using QMP if $full == true (this can be slow)
3082 my ($opt_vmid, $full) = @_;
3086 my $storecfg = PVE
::Storage
::config
();
3088 my $list = vzlist
();
3089 my $defaults = load_defaults
();
3091 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
3093 my $cpucount = $cpuinfo->{cpus
} || 1;
3095 foreach my $vmid (keys %$list) {
3096 next if $opt_vmid && ($vmid ne $opt_vmid);
3098 my $cfspath = PVE
::QemuConfig-
>cfs_config_path($vmid);
3099 my $conf = PVE
::Cluster
::cfs_read_file
($cfspath) || {};
3101 my $d = { vmid
=> $vmid };
3102 $d->{pid
} = $list->{$vmid}->{pid
};
3104 # fixme: better status?
3105 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
3107 my $size = disksize
($storecfg, $conf);
3108 if (defined($size)) {
3109 $d->{disk
} = 0; # no info available
3110 $d->{maxdisk
} = $size;
3116 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
3117 * ($conf->{cores
} || $defaults->{cores
});
3118 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
3119 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
3121 $d->{name
} = $conf->{name
} || "VM $vmid";
3122 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
3123 : $defaults->{memory
}*(1024*1024);
3125 if ($conf->{balloon
}) {
3126 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
3127 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
3128 : $defaults->{shares
};
3139 $d->{diskwrite
} = 0;
3141 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
3143 $d->{serial
} = 1 if conf_has_serial
($conf);
3144 $d->{lock} = $conf->{lock} if $conf->{lock};
3149 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
3150 foreach my $dev (keys %$netdev) {
3151 next if $dev !~ m/^tap([1-9]\d*)i/;
3153 my $d = $res->{$vmid};
3156 $d->{netout
} += $netdev->{$dev}->{receive
};
3157 $d->{netin
} += $netdev->{$dev}->{transmit
};
3160 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
3161 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
3166 my $ctime = gettimeofday
;
3168 foreach my $vmid (keys %$list) {
3170 my $d = $res->{$vmid};
3171 my $pid = $d->{pid
};
3174 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
3175 next if !$pstat; # not running
3177 my $used = $pstat->{utime} + $pstat->{stime
};
3179 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
3181 if ($pstat->{vsize
}) {
3182 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
3185 my $old = $last_proc_pid_stat->{$pid};
3187 $last_proc_pid_stat->{$pid} = {
3195 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
3197 if ($dtime > 1000) {
3198 my $dutime = $used - $old->{used
};
3200 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
3201 $last_proc_pid_stat->{$pid} = {
3207 $d->{cpu
} = $old->{cpu
};
3211 return $res if !$full;
3213 my $qmpclient = PVE
::QMPClient-
>new();
3215 my $ballooncb = sub {
3216 my ($vmid, $resp) = @_;
3218 my $info = $resp->{'return'};
3219 return if !$info->{max_mem
};
3221 my $d = $res->{$vmid};
3223 # use memory assigned to VM
3224 $d->{maxmem
} = $info->{max_mem
};
3225 $d->{balloon
} = $info->{actual
};
3227 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
3228 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
3229 $d->{freemem
} = $info->{free_mem
};
3232 $d->{ballooninfo
} = $info;
3235 my $blockstatscb = sub {
3236 my ($vmid, $resp) = @_;
3237 my $data = $resp->{'return'} || [];
3238 my $totalrdbytes = 0;
3239 my $totalwrbytes = 0;
3241 for my $blockstat (@$data) {
3242 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
3243 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
3245 $blockstat->{device
} =~ s/drive-//;
3246 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
3248 $res->{$vmid}->{diskread
} = $totalrdbytes;
3249 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
3252 my $statuscb = sub {
3253 my ($vmid, $resp) = @_;
3255 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3256 # this fails if ballon driver is not loaded, so this must be
3257 # the last commnand (following command are aborted if this fails).
3258 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3260 my $status = 'unknown';
3261 if (!defined($status = $resp->{'return'}->{status
})) {
3262 warn "unable to get VM status\n";
3266 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3269 foreach my $vmid (keys %$list) {
3270 next if $opt_vmid && ($vmid ne $opt_vmid);
3271 next if !$res->{$vmid}->{pid
}; # not running
3272 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3275 $qmpclient->queue_execute(undef, 2);
3277 foreach my $vmid (keys %$list) {
3278 next if $opt_vmid && ($vmid ne $opt_vmid);
3279 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3286 my ($conf, $func, @param) = @_;
3288 foreach my $ds (valid_drive_names
()) {
3289 next if !defined($conf->{$ds});
3291 my $drive = parse_drive
($ds, $conf->{$ds});
3294 &$func($ds, $drive, @param);
3299 my ($conf, $func, @param) = @_;
3303 my $test_volid = sub {
3304 my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3308 $volhash->{$volid}->{cdrom
} //= 1;
3309 $volhash->{$volid}->{cdrom
} = 0 if !$is_cdrom;
3311 $volhash->{$volid}->{replicate
} //= 0;
3312 $volhash->{$volid}->{replicate
} = 1 if $replicate;
3314 $volhash->{$volid}->{shared
} //= 0;
3315 $volhash->{$volid}->{shared
} = 1 if $shared;
3317 $volhash->{$volid}->{referenced_in_config
} //= 0;
3318 $volhash->{$volid}->{referenced_in_config
} = 1 if !defined($snapname);
3320 $volhash->{$volid}->{referenced_in_snapshot
}->{$snapname} = 1
3321 if defined($snapname);
3324 foreach_drive
($conf, sub {
3325 my ($ds, $drive) = @_;
3326 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, undef);
3329 foreach my $snapname (keys %{$conf->{snapshots
}}) {
3330 my $snap = $conf->{snapshots
}->{$snapname};
3331 $test_volid->($snap->{vmstate
}, 0, 1, $snapname);
3332 foreach_drive
($snap, sub {
3333 my ($ds, $drive) = @_;
3334 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, $snapname);
3338 foreach my $volid (keys %$volhash) {
3339 &$func($volid, $volhash->{$volid}, @param);
3343 sub conf_has_serial
{
3346 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3347 if ($conf->{"serial$i"}) {
3355 sub vga_conf_has_spice
{
3358 my $vgaconf = parse_vga
($vga);
3359 my $vgatype = $vgaconf->{type
};
3360 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3365 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3366 sub get_host_arch
() {
3367 $host_arch = (POSIX
::uname
())[4] if !$host_arch;
3373 return get_host_arch
() eq $arch;
3376 my $default_machines = {
3381 sub get_basic_machine_info
{
3382 my ($conf, $forcemachine) = @_;
3384 my $arch = $conf->{arch
} // get_host_arch
();
3385 my $machine = $forcemachine || $conf->{machine
} || $default_machines->{$arch};
3386 return ($arch, $machine);
3389 sub get_ovmf_files
($) {
3392 my $ovmf = $OVMF->{$arch}
3393 or die "no OVMF images known for architecture '$arch'\n";
3399 aarch64
=> '/usr/bin/qemu-system-aarch64',
3400 x86_64
=> '/usr/bin/qemu-system-x86_64',
3402 sub get_command_for_arch
($) {
3404 return '/usr/bin/kvm' if is_native
($arch);
3406 my $cmd = $Arch2Qemu->{$arch}
3407 or die "don't know how to emulate architecture '$arch'\n";
3411 sub get_cpu_options
{
3412 my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3415 my $ostype = $conf->{ostype
};
3417 my $cpu = $kvm ?
"kvm64" : "qemu64";
3418 if ($arch eq 'aarch64') {
3419 $cpu = 'cortex-a57';
3422 if (my $cputype = $conf->{cpu
}) {
3423 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
3424 or die "Cannot parse cpu description: $cputype\n";
3425 $cpu = $cpuconf->{cputype
};
3426 $kvm_off = 1 if $cpuconf->{hidden
};
3427 $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3429 if (defined(my $flags = $cpuconf->{flags
})) {
3430 push @$cpuFlags, split(";", $flags);
3434 push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3436 push @$cpuFlags , '-x2apic'
3437 if $conf->{ostype
} && $conf->{ostype
} eq 'solaris';
3439 push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3441 push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3443 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3445 push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3446 push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3449 add_hyperv_enlightenments
($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios
}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3451 push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3453 push @$cpuFlags, 'kvm=off' if $kvm_off;
3455 if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3456 push @$cpuFlags, "vendor=${cpu_vendor}"
3457 if $cpu_vendor ne 'default';
3458 } elsif ($arch ne 'aarch64') {
3459 die "internal error"; # should not happen
3462 $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3464 return ('-cpu', $cpu);
3467 sub config_to_command
{
3468 my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3471 my $globalFlags = [];
3472 my $machineFlags = [];
3477 my $kvmver = kvm_user_version
();
3478 my $vernum = 0; # unknown
3479 my $ostype = $conf->{ostype
};
3480 my $winversion = windows_version
($ostype);
3481 my $kvm = $conf->{kvm
};
3483 my ($arch, $machine_type) = get_basic_machine_info
($conf, $forcemachine);
3484 $kvm //= 1 if is_native
($arch);
3487 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3488 if !defined kvm_version
();
3491 if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3492 $vernum = $1*1000000+$2*1000;
3493 } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3494 $vernum = $1*1000000+$2*1000+$3;
3497 die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3499 my $have_ovz = -f
'/proc/vz/vestat';
3501 my $q35 = machine_type_is_q35
($conf);
3502 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3503 my $use_old_bios_files = undef;
3504 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3506 my $cpuunits = defined($conf->{cpuunits
}) ?
3507 $conf->{cpuunits
} : $defaults->{cpuunits
};
3509 push @$cmd, get_command_for_arch
($arch);
3511 push @$cmd, '-id', $vmid;
3513 my $vmname = $conf->{name
} || "vm$vmid";
3515 push @$cmd, '-name', $vmname;
3519 my $qmpsocket = qmp_socket
($vmid);
3520 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3521 push @$cmd, '-mon', "chardev=qmp,mode=control";
3523 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 12)) {
3524 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3525 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3528 push @$cmd, '-pidfile' , pidfile_name
($vmid);
3530 push @$cmd, '-daemonize';
3532 if ($conf->{smbios1
}) {
3533 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3534 if ($smbios_conf->{base64
}) {
3535 # Do not pass base64 flag to qemu
3536 delete $smbios_conf->{base64
};
3537 my $smbios_string = "";
3538 foreach my $key (keys %$smbios_conf) {
3540 if ($key eq "uuid") {
3541 $value = $smbios_conf->{uuid
}
3543 $value = decode_base64
($smbios_conf->{$key});
3545 # qemu accepts any binary data, only commas need escaping by double comma
3547 $smbios_string .= "," . $key . "=" . $value if $value;
3549 push @$cmd, '-smbios', "type=1" . $smbios_string;
3551 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3555 if ($conf->{vmgenid
}) {
3556 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3559 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch);
3560 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3561 die "uefi base image not found\n" if ! -f
$ovmf_code;
3565 if (my $efidisk = $conf->{efidisk0
}) {
3566 my $d = PVE
::JSONSchema
::parse_property_string
($efidisk_fmt, $efidisk);
3567 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3568 $format = $d->{format
};
3570 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3571 if (!defined($format)) {
3572 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3573 $format = qemu_img_format
($scfg, $volname);
3577 die "efidisk format must be specified\n"
3578 if !defined($format);
3581 warn "no efidisk configured! Using temporary efivars disk.\n";
3582 $path = "/tmp/$vmid-ovmf.fd";
3583 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3587 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3588 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3592 # add usb controllers
3593 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
($conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3594 push @$devices, @usbcontrollers if @usbcontrollers;
3595 my $vga = parse_vga
($conf->{vga
});
3597 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3598 $vga->{type
} = 'qxl' if $qxlnum;
3600 if (!$vga->{type
}) {
3601 if ($arch eq 'aarch64') {
3602 $vga->{type
} = 'virtio';
3603 } elsif (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 9)) {
3604 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3606 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3610 # enable absolute mouse coordinates (needed by vnc)
3612 if (defined($conf->{tablet
})) {
3613 $tablet = $conf->{tablet
};
3615 $tablet = $defaults->{tablet
};
3616 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3617 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3621 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3622 my $kbd = print_keyboarddevice_full
($conf, $arch);
3623 push @$devices, '-device', $kbd if defined($kbd);
3627 my $gpu_passthrough;
3630 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
3631 my $d = parse_hostpci
($conf->{"hostpci$i"});
3634 my $pcie = $d->{pcie
};
3636 die "q35 machine model is not enabled" if !$q35;
3637 # win7 wants to have the pcie devices directly on the pcie bus
3638 # instead of in the root port
3639 if ($winversion == 7) {
3640 $pciaddr = print_pcie_addr
("hostpci${i}bus0");
3642 $pciaddr = print_pcie_addr
("hostpci$i");
3645 $pciaddr = print_pci_addr
("hostpci$i", $bridges, $arch, $machine_type);
3648 my $rombar = defined($d->{rombar
}) && !$d->{rombar
} ?
',rombar=0' : '';
3649 my $romfile = $d->{romfile
};
3652 if ($d->{'x-vga'}) {
3653 $xvga = ',x-vga=on';
3655 $vga->{type
} = 'none' if !defined($conf->{vga
});
3656 $gpu_passthrough = 1;
3658 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3662 my $pcidevices = $d->{pciid
};
3663 my $multifunction = 1 if @$pcidevices > 1;
3665 if ($d->{mdev
} && scalar(@$pcidevices) == 1) {
3666 my $id = $pcidevices->[0]->{id
};
3667 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $i);
3668 $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3669 } elsif ($d->{mdev
}) {
3670 warn "ignoring mediated device with multifunction device\n";
3674 foreach my $pcidevice (@$pcidevices) {
3676 my $id = "hostpci$i";
3677 $id .= ".$j" if $multifunction;
3678 my $addr = $pciaddr;
3679 $addr .= ".$j" if $multifunction;
3680 my $devicestr = "vfio-pci";
3682 $devicestr .= ",sysfsdev=$sysfspath";
3684 $devicestr .= ",host=$pcidevice->{id}";
3686 $devicestr .= ",id=$id$addr";
3689 $devicestr .= "$rombar$xvga";
3690 $devicestr .= ",multifunction=on" if $multifunction;
3691 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3694 push @$devices, '-device', $devicestr;
3700 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
($conf, $usbdesc->{format
}, $MAX_USB_DEVICES);
3701 push @$devices, @usbdevices if @usbdevices;
3703 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3704 if (my $path = $conf->{"serial$i"}) {
3705 if ($path eq 'socket') {
3706 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3707 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3708 # On aarch64, serial0 is the UART device. Qemu only allows
3709 # connecting UART devices via the '-serial' command line, as
3710 # the device has a fixed slot on the hardware...
3711 if ($arch eq 'aarch64' && $i == 0) {
3712 push @$devices, '-serial', "chardev:serial$i";
3714 push @$devices, '-device', "isa-serial,chardev=serial$i";
3717 die "no such serial device\n" if ! -c
$path;
3718 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3719 push @$devices, '-device', "isa-serial,chardev=serial$i";
3725 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3726 if (my $path = $conf->{"parallel$i"}) {
3727 die "no such parallel device\n" if ! -c
$path;
3728 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3729 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3730 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3736 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3737 $sockets = $conf->{sockets
} if $conf->{sockets
};
3739 my $cores = $conf->{cores
} || 1;
3741 my $maxcpus = $sockets * $cores;
3743 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3745 my $allowed_vcpus = $cpuinfo->{cpus
};
3747 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3748 if ($allowed_vcpus < $maxcpus);
3750 if($hotplug_features->{cpu
} && qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 7)) {
3752 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3753 for (my $i = 2; $i <= $vcpus; $i++) {
3754 my $cpustr = print_cpu_device
($conf,$i);
3755 push @$cmd, '-device', $cpustr;
3760 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3762 push @$cmd, '-nodefaults';
3764 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
3766 my $bootindex_hash = {};
3768 foreach my $o (split(//, $bootorder)) {
3769 $bootindex_hash->{$o} = $i*100;
3773 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3775 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3777 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3779 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3780 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3781 my $socket = vnc_socket
($vmid);
3782 push @$cmd, '-vnc', "unix:$socket,password";
3784 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3785 push @$cmd, '-nographic';
3789 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3791 my $useLocaltime = $conf->{localtime};
3793 if ($winversion >= 5) { # windows
3794 $useLocaltime = 1 if !defined($conf->{localtime});
3796 # use time drift fix when acpi is enabled
3797 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3798 $tdf = 1 if !defined($conf->{tdf
});
3802 if ($winversion >= 6) {
3803 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3804 push @$cmd, '-no-hpet';
3807 push @$rtcFlags, 'driftfix=slew' if $tdf;
3810 push @$machineFlags, 'accel=tcg';
3813 if ($machine_type) {
3814 push @$machineFlags, "type=${machine_type}";
3817 if ($conf->{startdate
}) {
3818 push @$rtcFlags, "base=$conf->{startdate}";
3819 } elsif ($useLocaltime) {
3820 push @$rtcFlags, 'base=localtime';
3823 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3825 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3827 push @$cmd, '-S' if $conf->{freeze
};
3829 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3832 #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3833 #push @$cmd, '-soundhw', 'es1370';
3834 #push @$cmd, '-soundhw', $soundhw if $soundhw;
3836 if (parse_guest_agent
($conf)->{enabled
}) {
3837 my $qgasocket = qmp_socket
($vmid, 1);
3838 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3839 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3840 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3841 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3849 for(my $i = 1; $i < $qxlnum; $i++){
3850 push @$devices, '-device', print_vga_device
($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3853 # assume other OS works like Linux
3854 my ($ram, $vram) = ("134217728", "67108864");
3855 if ($vga->{memory
}) {
3856 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3857 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3859 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3860 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3864 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3866 my $nodename = PVE
::INotify
::nodename
();
3867 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3868 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3869 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3870 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3871 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3873 push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3875 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3876 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3877 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3880 # enable balloon by default, unless explicitly disabled
3881 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3882 $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3883 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3886 if ($conf->{watchdog
}) {
3887 my $wdopts = parse_watchdog
($conf->{watchdog
});
3888 $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3889 my $watchdog = $wdopts->{model
} || 'i6300esb';
3890 push @$devices, '-device', "$watchdog$pciaddr";
3891 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3895 my $scsicontroller = {};
3896 my $ahcicontroller = {};
3897 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3899 # Add iscsi initiator name if available
3900 if (my $initiator = get_initiator_name
()) {
3901 push @$devices, '-iscsi', "initiator-name=$initiator";
3904 foreach_drive
($conf, sub {
3905 my ($ds, $drive) = @_;
3907 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3908 push @$vollist, $drive->{file
};
3911 # ignore efidisk here, already added in bios/fw handling code above
3912 return if $drive->{interface
} eq 'efidisk';
3914 $use_virtio = 1 if $ds =~ m/^virtio/;
3916 if (drive_is_cdrom
($drive)) {
3917 if ($bootindex_hash->{d
}) {
3918 $drive->{bootindex
} = $bootindex_hash->{d
};
3919 $bootindex_hash->{d
} += 1;
3922 if ($bootindex_hash->{c
}) {
3923 $drive->{bootindex
} = $bootindex_hash->{c
} if $conf->{bootdisk
} && ($conf->{bootdisk
} eq $ds);
3924 $bootindex_hash->{c
} += 1;
3928 if($drive->{interface
} eq 'virtio'){
3929 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3932 if ($drive->{interface
} eq 'scsi') {
3934 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3936 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3937 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3940 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3941 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3942 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3943 } elsif ($drive->{iothread
}) {
3944 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3948 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3949 $queues = ",num_queues=$drive->{queues}";
3952 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3953 $scsicontroller->{$controller}=1;
3956 if ($drive->{interface
} eq 'sata') {
3957 my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3958 $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3959 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3960 $ahcicontroller->{$controller}=1;
3963 my $drive_cmd = print_drive_full
($storecfg, $vmid, $drive);
3964 push @$devices, '-drive',$drive_cmd;
3965 push @$devices, '-device', print_drivedevice_full
($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3968 for (my $i = 0; $i < $MAX_NETS; $i++) {
3969 next if !$conf->{"net$i"};
3970 my $d = parse_net
($conf->{"net$i"});
3973 $use_virtio = 1 if $d->{model
} eq 'virtio';
3975 if ($bootindex_hash->{n
}) {
3976 $d->{bootindex
} = $bootindex_hash->{n
};
3977 $bootindex_hash->{n
} += 1;
3980 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, "net$i");
3981 push @$devices, '-netdev', $netdevfull;
3983 my $netdevicefull = print_netdevice_full
($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3984 push @$devices, '-device', $netdevicefull;
3987 if ($conf->{ivshmem
}) {
3988 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
3992 $bus = print_pcie_addr
("ivshmem");
3994 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
3997 my $ivshmem_name = $ivshmem->{name
} // $vmid;
3998 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4000 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4001 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4006 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3)) {
4011 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4013 while (my ($k, $v) = each %$bridges) {
4014 $pciaddr = print_pci_addr
("pci.$k", undef, $arch, $machine_type);
4015 unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4019 push @$cmd, @$devices;
4020 push @$cmd, '-rtc', join(',', @$rtcFlags)
4021 if scalar(@$rtcFlags);
4022 push @$cmd, '-machine', join(',', @$machineFlags)
4023 if scalar(@$machineFlags);
4024 push @$cmd, '-global', join(',', @$globalFlags)
4025 if scalar(@$globalFlags);
4027 if (my $vmstate = $conf->{vmstate
}) {
4028 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4029 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
4030 push @$cmd, '-loadstate', $statepath;
4034 if ($conf->{args
}) {
4035 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4039 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
4044 return "${var_run_tmpdir}/$vmid.vnc";
4050 my $res = vm_mon_cmd
($vmid, 'query-spice');
4052 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4056 my ($vmid, $qga, $name) = @_;
4057 my $sockettype = $qga ?
'qga' : 'qmp';
4058 my $ext = $name ?
'-'.$name : '';
4059 return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4064 return "${var_run_tmpdir}/$vmid.pid";
4067 sub vm_devices_list
{
4070 my $res = vm_mon_cmd
($vmid, 'query-pci');
4071 my $devices_to_check = [];
4073 foreach my $pcibus (@$res) {
4074 push @$devices_to_check, @{$pcibus->{devices
}},
4077 while (@$devices_to_check) {
4079 for my $d (@$devices_to_check) {
4080 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4081 next if !$d->{'pci_bridge'};
4083 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4084 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4086 $devices_to_check = $to_check;
4089 my $resblock = vm_mon_cmd
($vmid, 'query-block');
4090 foreach my $block (@$resblock) {
4091 if($block->{device
} =~ m/^drive-(\S+)/){
4096 my $resmice = vm_mon_cmd
($vmid, 'query-mice');
4097 foreach my $mice (@$resmice) {
4098 if ($mice->{name
} eq 'QEMU HID Tablet') {
4099 $devices->{tablet
} = 1;
4104 # for usb devices there is no query-usb
4105 # but we can iterate over the entries in
4106 # qom-list path=/machine/peripheral
4107 my $resperipheral = vm_mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4108 foreach my $per (@$resperipheral) {
4109 if ($per->{name
} =~ m/^usb\d+$/) {
4110 $devices->{$per->{name
}} = 1;
4118 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4120 my $q35 = machine_type_is_q35
($conf);
4122 my $devices_list = vm_devices_list
($vmid);
4123 return 1 if defined($devices_list->{$deviceid});
4125 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4127 if ($deviceid eq 'tablet') {
4129 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4131 } elsif ($deviceid eq 'keyboard') {
4133 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4135 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4137 die "usb hotplug currently not reliable\n";
4138 # since we can't reliably hot unplug all added usb devices
4139 # and usb passthrough disables live migration
4140 # we disable usb hotplugging for now
4141 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device));
4143 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4145 qemu_iothread_add
($vmid, $deviceid, $device);
4147 qemu_driveadd
($storecfg, $vmid, $device);
4148 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4150 qemu_deviceadd
($vmid, $devicefull);
4151 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4153 eval { qemu_drivedel
($vmid, $deviceid); };
4158 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4161 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4162 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4163 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4165 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4167 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4168 qemu_iothread_add
($vmid, $deviceid, $device);
4169 $devicefull .= ",iothread=iothread-$deviceid";
4172 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4173 $devicefull .= ",num_queues=$device->{queues}";
4176 qemu_deviceadd
($vmid, $devicefull);
4177 qemu_deviceaddverify
($vmid, $deviceid);
4179 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4181 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4182 qemu_driveadd
($storecfg, $vmid, $device);
4184 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4185 eval { qemu_deviceadd
($vmid, $devicefull); };
4187 eval { qemu_drivedel
($vmid, $deviceid); };
4192 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4194 return undef if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4196 my $machine_type = PVE
::QemuServer
::qemu_machine_pxe
($vmid, $conf);
4197 my $use_old_bios_files = undef;
4198 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4200 my $netdevicefull = print_netdevice_full
($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4201 qemu_deviceadd
($vmid, $netdevicefull);
4203 qemu_deviceaddverify
($vmid, $deviceid);
4204 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4207 eval { qemu_netdevdel
($vmid, $deviceid); };
4212 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4215 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4216 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4218 qemu_deviceadd
($vmid, $devicefull);
4219 qemu_deviceaddverify
($vmid, $deviceid);
4222 die "can't hotplug device '$deviceid'\n";
4228 # fixme: this should raise exceptions on error!
4229 sub vm_deviceunplug
{
4230 my ($vmid, $conf, $deviceid) = @_;
4232 my $devices_list = vm_devices_list
($vmid);
4233 return 1 if !defined($devices_list->{$deviceid});
4235 die "can't unplug bootdisk" if $conf->{bootdisk
} && $conf->{bootdisk
} eq $deviceid;
4237 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4239 qemu_devicedel
($vmid, $deviceid);
4241 } elsif ($deviceid =~ m/^usb\d+$/) {
4243 die "usb hotplug currently not reliable\n";
4244 # when unplugging usb devices this way,
4245 # there may be remaining usb controllers/hubs
4246 # so we disable it for now
4247 qemu_devicedel
($vmid, $deviceid);
4248 qemu_devicedelverify
($vmid, $deviceid);
4250 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4252 qemu_devicedel
($vmid, $deviceid);
4253 qemu_devicedelverify
($vmid, $deviceid);
4254 qemu_drivedel
($vmid, $deviceid);
4255 qemu_iothread_del
($conf, $vmid, $deviceid);
4257 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4259 qemu_devicedel
($vmid, $deviceid);
4260 qemu_devicedelverify
($vmid, $deviceid);
4261 qemu_iothread_del
($conf, $vmid, $deviceid);
4263 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4265 qemu_devicedel
($vmid, $deviceid);
4266 qemu_drivedel
($vmid, $deviceid);
4267 qemu_deletescsihw
($conf, $vmid, $deviceid);
4269 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4271 qemu_devicedel
($vmid, $deviceid);
4272 qemu_devicedelverify
($vmid, $deviceid);
4273 qemu_netdevdel
($vmid, $deviceid);
4276 die "can't unplug device '$deviceid'\n";
4282 sub qemu_deviceadd
{
4283 my ($vmid, $devicefull) = @_;
4285 $devicefull = "driver=".$devicefull;
4286 my %options = split(/[=,]/, $devicefull);
4288 vm_mon_cmd
($vmid, "device_add" , %options);
4291 sub qemu_devicedel
{
4292 my ($vmid, $deviceid) = @_;
4294 my $ret = vm_mon_cmd
($vmid, "device_del", id
=> $deviceid);
4297 sub qemu_iothread_add
{
4298 my($vmid, $deviceid, $device) = @_;
4300 if ($device->{iothread
}) {
4301 my $iothreads = vm_iothreads_list
($vmid);
4302 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4306 sub qemu_iothread_del
{
4307 my($conf, $vmid, $deviceid) = @_;
4309 my $confid = $deviceid;
4310 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4311 $confid = 'scsi' . $1;
4313 my $device = parse_drive
($confid, $conf->{$confid});
4314 if ($device->{iothread
}) {
4315 my $iothreads = vm_iothreads_list
($vmid);
4316 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4320 sub qemu_objectadd
{
4321 my($vmid, $objectid, $qomtype) = @_;
4323 vm_mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4328 sub qemu_objectdel
{
4329 my($vmid, $objectid) = @_;
4331 vm_mon_cmd
($vmid, "object-del", id
=> $objectid);
4337 my ($storecfg, $vmid, $device) = @_;
4339 my $drive = print_drive_full
($storecfg, $vmid, $device);
4340 $drive =~ s/\\/\\\\/g;
4341 my $ret = vm_human_monitor_command
($vmid, "drive_add auto \"$drive\"");
4343 # If the command succeeds qemu prints: "OK
"
4344 return 1 if $ret =~ m/OK/s;
4346 die "adding drive failed
: $ret\n";
4350 my($vmid, $deviceid) = @_;
4352 my $ret = vm_human_monitor_command($vmid, "drive_del drive-
$deviceid");
4355 return 1 if $ret eq "";
4357 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4358 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4360 die "deleting drive
$deviceid failed
: $ret\n";
4363 sub qemu_deviceaddverify {
4364 my ($vmid, $deviceid) = @_;
4366 for (my $i = 0; $i <= 5; $i++) {
4367 my $devices_list = vm_devices_list($vmid);
4368 return 1 if defined($devices_list->{$deviceid});
4372 die "error on hotplug device
'$deviceid'\n";
4376 sub qemu_devicedelverify {
4377 my ($vmid, $deviceid) = @_;
4379 # need to verify that the device is correctly removed as device_del
4380 # is async and empty return is not reliable
4382 for (my $i = 0; $i <= 5; $i++) {
4383 my $devices_list = vm_devices_list($vmid);
4384 return 1 if !defined($devices_list->{$deviceid});
4388 die "error on hot-unplugging device
'$deviceid'\n";
4391 sub qemu_findorcreatescsihw {
4392 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4394 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4396 my $scsihwid="$controller_prefix$controller";
4397 my $devices_list = vm_devices_list($vmid);
4399 if(!defined($devices_list->{$scsihwid})) {
4400 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4406 sub qemu_deletescsihw {
4407 my ($conf, $vmid, $opt) = @_;
4409 my $device = parse_drive($opt, $conf->{$opt});
4411 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4412 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4416 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4418 my $devices_list = vm_devices_list($vmid);
4419 foreach my $opt (keys %{$devices_list}) {
4420 if (PVE::QemuServer::is_valid_drivename($opt)) {
4421 my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4422 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4428 my $scsihwid="scsihw
$controller";
4430 vm_deviceunplug($vmid, $conf, $scsihwid);
4435 sub qemu_add_pci_bridge {
4436 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4442 print_pci_addr($device, $bridges, $arch, $machine_type);
4444 while (my ($k, $v) = each %$bridges) {
4447 return 1 if !defined($bridgeid) || $bridgeid < 1;
4449 my $bridge = "pci
.$bridgeid";
4450 my $devices_list = vm_devices_list($vmid);
4452 if (!defined($devices_list->{$bridge})) {
4453 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4459 sub qemu_set_link_status {
4460 my ($vmid, $device, $up) = @_;
4462 vm_mon_cmd($vmid, "set_link
", name => $device,
4463 up => $up ? JSON::true : JSON::false);
4466 sub qemu_netdevadd {
4467 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4469 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4470 my %options = split(/[=,]/, $netdev);
4472 vm_mon_cmd($vmid, "netdev_add
", %options);
4476 sub qemu_netdevdel {
4477 my ($vmid, $deviceid) = @_;
4479 vm_mon_cmd($vmid, "netdev_del
", id => $deviceid);
4482 sub qemu_usb_hotplug {
4483 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4487 # remove the old one first
4488 vm_deviceunplug($vmid, $conf, $deviceid);
4490 # check if xhci controller is necessary and available
4491 if ($device->{usb3}) {
4493 my $devicelist = vm_devices_list($vmid);
4495 if (!$devicelist->{xhci}) {
4496 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4497 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4500 my $d = parse_usb_device($device->{host});
4501 $d->{usb3} = $device->{usb3};
4504 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4507 sub qemu_cpu_hotplug {
4508 my ($vmid, $conf, $vcpus) = @_;
4510 my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4513 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4514 $sockets = $conf->{sockets} if $conf->{sockets};
4515 my $cores = $conf->{cores} || 1;
4516 my $maxcpus = $sockets * $cores;
4518 $vcpus = $maxcpus if !$vcpus;
4520 die "you can
't add more vcpus than maxcpus\n"
4521 if $vcpus > $maxcpus;
4523 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4525 if ($vcpus < $currentvcpus) {
4527 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4529 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4530 qemu_devicedel($vmid, "cpu$i");
4532 my $currentrunningvcpus = undef;
4534 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4535 last if scalar(@{$currentrunningvcpus}) == $i-1;
4536 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4540 #update conf after each succesfull cpu unplug
4541 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4542 PVE::QemuConfig->write_config($vmid, $conf);
4545 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4551 my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4552 die "vcpus in running vm does not match its configuration\n"
4553 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4555 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4557 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4558 my $cpustr = print_cpu_device($conf, $i);
4559 qemu_deviceadd($vmid, $cpustr);
4562 my $currentrunningvcpus = undef;
4564 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4565 last if scalar(@{$currentrunningvcpus}) == $i;
4566 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4570 #update conf after each succesfull cpu hotplug
4571 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4572 PVE::QemuConfig->write_config($vmid, $conf);
4576 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4577 vm_mon_cmd($vmid, "cpu-add", id => int($i));
4582 sub qemu_block_set_io_throttle {
4583 my ($vmid, $deviceid,
4584 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4585 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4586 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4587 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4589 return if !check_running($vmid) ;
4591 vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4593 bps_rd => int($bps_rd),
4594 bps_wr => int($bps_wr),
4596 iops_rd => int($iops_rd),
4597 iops_wr => int($iops_wr),
4598 bps_max => int($bps_max),
4599 bps_rd_max => int($bps_rd_max),
4600 bps_wr_max => int($bps_wr_max),
4601 iops_max => int($iops_max),
4602 iops_rd_max => int($iops_rd_max),
4603 iops_wr_max => int($iops_wr_max),
4604 bps_max_length => int($bps_max_length),
4605 bps_rd_max_length => int($bps_rd_max_length),
4606 bps_wr_max_length => int($bps_wr_max_length),
4607 iops_max_length => int($iops_max_length),
4608 iops_rd_max_length => int($iops_rd_max_length),
4609 iops_wr_max_length => int($iops_wr_max_length),
4614 # old code, only used to shutdown old VM after update
4616 my ($fh, $timeout) = @_;
4618 my $sel = new IO::Select;
4625 while (scalar (@ready = $sel->can_read($timeout))) {
4627 if ($count = $fh->sysread($buf, 8192)) {
4628 if ($buf =~ /^(.*)\(qemu\) $/s) {
4635 if (!defined($count)) {
4642 die "monitor read timeout\n" if !scalar(@ready);
4647 sub qemu_block_resize {
4648 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4650 my $running = check_running($vmid);
4652 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4654 return if !$running;
4656 vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4660 sub qemu_volume_snapshot {
4661 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4663 my $running = check_running($vmid);
4665 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4666 vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4668 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4672 sub qemu_volume_snapshot_delete {
4673 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4675 my $running = check_running($vmid);
4680 my $conf = PVE::QemuConfig->load_config($vmid);
4681 foreach_drive($conf, sub {
4682 my ($ds, $drive) = @_;
4683 $running = 1 if $drive->{file} eq $volid;
4687 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4688 vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4690 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4694 sub set_migration_caps {
4700 "auto-converge" => 1,
4702 "x-rdma-pin-all" => 0,
4707 my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4709 for my $supported_capability (@$supported_capabilities) {
4711 capability => $supported_capability->{capability},
4712 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4716 vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4719 my $fast_plug_option = {
4727 'vmstatestorage
' => 1,
4731 # hotplug changes in [PENDING]
4732 # $selection hash can be used to only apply specified options, for
4733 # example: { cores => 1 } (only apply changed 'cores
')
4734 # $errors ref is used to return error messages
4735 sub vmconfig_hotplug_pending {
4736 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4738 my $defaults = load_defaults();
4739 my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4741 # commit values which do not have any impact on running VM first
4742 # Note: those option cannot raise errors, we we do not care about
4743 # $selection and always apply them.
4745 my $add_error = sub {
4746 my ($opt, $msg) = @_;
4747 $errors->{$opt} = "hotplug problem - $msg";
4751 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4752 if ($fast_plug_option->{$opt}) {
4753 $conf->{$opt} = $conf->{pending}->{$opt};
4754 delete $conf->{pending}->{$opt};
4760 PVE::QemuConfig->write_config($vmid, $conf);
4761 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4764 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4766 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4767 while (my ($opt, $force) = each %$pending_delete_hash) {
4768 next if $selection && !$selection->{$opt};
4770 if ($opt eq 'hotplug
') {
4771 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4772 } elsif ($opt eq 'tablet
') {
4773 die "skip\n" if !$hotplug_features->{usb};
4774 if ($defaults->{tablet}) {
4775 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4776 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4777 if $arch eq 'aarch64
';
4779 vm_deviceunplug($vmid, $conf, 'tablet
');
4780 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4782 } elsif ($opt =~ m/^usb\d+/) {
4784 # since we cannot reliably hot unplug usb devices
4785 # we are disabling it
4786 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4787 vm_deviceunplug($vmid, $conf, $opt);
4788 } elsif ($opt eq 'vcpus
') {
4789 die "skip\n" if !$hotplug_features->{cpu};
4790 qemu_cpu_hotplug($vmid, $conf, undef);
4791 } elsif ($opt eq 'balloon
') {
4792 # enable balloon device is not hotpluggable
4793 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4794 # here we reset the ballooning value to memory
4795 my $balloon = $conf->{memory} || $defaults->{memory};
4796 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4797 } elsif ($fast_plug_option->{$opt}) {
4799 } elsif ($opt =~ m/^net(\d+)$/) {
4800 die "skip\n" if !$hotplug_features->{network};
4801 vm_deviceunplug($vmid, $conf, $opt);
4802 } elsif (is_valid_drivename($opt)) {
4803 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4804 vm_deviceunplug($vmid, $conf, $opt);
4805 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4806 } elsif ($opt =~ m/^memory$/) {
4807 die "skip\n" if !$hotplug_features->{memory};
4808 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4809 } elsif ($opt eq 'cpuunits
') {
4810 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4811 } elsif ($opt eq 'cpulimit
') {
4812 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4818 &$add_error($opt, $err) if $err ne "skip\n";
4820 # save new config if hotplug was successful
4821 delete $conf->{$opt};
4822 vmconfig_undelete_pending_option($conf, $opt);
4823 PVE::QemuConfig->write_config($vmid, $conf);
4824 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4828 my $apply_pending_cloudinit;
4829 $apply_pending_cloudinit = sub {
4830 my ($key, $value) = @_;
4831 $apply_pending_cloudinit = sub {}; # once is enough
4833 my @cloudinit_opts = keys %$confdesc_cloudinit;
4834 foreach my $opt (keys %{$conf->{pending}}) {
4835 next if !grep { $_ eq $opt } @cloudinit_opts;
4836 $conf->{$opt} = delete $conf->{pending}->{$opt};
4839 my $new_conf = { %$conf };
4840 $new_conf->{$key} = $value;
4841 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4844 foreach my $opt (keys %{$conf->{pending}}) {
4845 next if $selection && !$selection->{$opt};
4846 my $value = $conf->{pending}->{$opt};
4848 if ($opt eq 'hotplug
') {
4849 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4850 } elsif ($opt eq 'tablet
') {
4851 die "skip\n" if !$hotplug_features->{usb};
4853 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4854 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4855 if $arch eq 'aarch64
';
4856 } elsif ($value == 0) {
4857 vm_deviceunplug($vmid, $conf, 'tablet
');
4858 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4860 } elsif ($opt =~ m/^usb\d+$/) {
4862 # since we cannot reliably hot unplug usb devices
4863 # we are disabling it
4864 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4865 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4866 die "skip\n" if !$d;
4867 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4868 } elsif ($opt eq 'vcpus
') {
4869 die "skip\n" if !$hotplug_features->{cpu};
4870 qemu_cpu_hotplug($vmid, $conf, $value);
4871 } elsif ($opt eq 'balloon
') {
4872 # enable/disable balloning device is not hotpluggable
4873 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4874 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4875 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4877 # allow manual ballooning if shares is set to zero
4878 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4879 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4880 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4882 } elsif ($opt =~ m/^net(\d+)$/) {
4883 # some changes can be done without hotplug
4884 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4885 $vmid, $opt, $value, $arch, $machine_type);
4886 } elsif (is_valid_drivename($opt)) {
4887 # some changes can be done without hotplug
4888 my $drive = parse_drive($opt, $value);
4889 if (drive_is_cloudinit($drive)) {
4890 &$apply_pending_cloudinit($opt, $value);
4892 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4893 $vmid, $opt, $value, 1, $arch, $machine_type);
4894 } elsif ($opt =~ m/^memory$/) { #dimms
4895 die "skip\n" if !$hotplug_features->{memory};
4896 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4897 } elsif ($opt eq 'cpuunits
') {
4898 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4899 } elsif ($opt eq 'cpulimit
') {
4900 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4901 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4903 die "skip\n"; # skip non-hot-pluggable options
4907 &$add_error($opt, $err) if $err ne "skip\n";
4909 # save new config if hotplug was successful
4910 $conf->{$opt} = $value;
4911 delete $conf->{pending}->{$opt};
4912 PVE::QemuConfig->write_config($vmid, $conf);
4913 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4918 sub try_deallocate_drive {
4919 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4921 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4922 my $volid = $drive->{file};
4923 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4924 my $sid = PVE::Storage::parse_volume_id($volid);
4925 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4927 # check if the disk is really unused
4928 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4929 if is_volume_in_use($storecfg, $conf, $key, $volid);
4930 PVE::Storage::vdisk_free($storecfg, $volid);
4933 # If vm is not owner of this disk remove from config
4941 sub vmconfig_delete_or_detach_drive {
4942 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4944 my $drive = parse_drive($opt, $conf->{$opt});
4946 my $rpcenv = PVE::RPCEnvironment::get();
4947 my $authuser = $rpcenv->get_user();
4950 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4951 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4953 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4957 sub vmconfig_apply_pending {
4958 my ($vmid, $conf, $storecfg) = @_;
4962 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4963 while (my ($opt, $force) = each %$pending_delete_hash) {
4964 die "internal error" if $opt =~ m/^unused/;
4965 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4966 if (!defined($conf->{$opt})) {
4967 vmconfig_undelete_pending_option($conf, $opt);
4968 PVE::QemuConfig->write_config($vmid, $conf);
4969 } elsif (is_valid_drivename($opt)) {
4970 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4971 vmconfig_undelete_pending_option($conf, $opt);
4972 delete $conf->{$opt};
4973 PVE::QemuConfig->write_config($vmid, $conf);
4975 vmconfig_undelete_pending_option($conf, $opt);
4976 delete $conf->{$opt};
4977 PVE::QemuConfig->write_config($vmid, $conf);
4981 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4983 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4984 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4986 if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4987 # skip if nothing changed
4988 } elsif (is_valid_drivename($opt)) {
4989 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4990 if defined($conf->{$opt});
4991 $conf->{$opt} = $conf->{pending}->{$opt};
4993 $conf->{$opt} = $conf->{pending}->{$opt};
4996 delete $conf->{pending}->{$opt};
4997 PVE::QemuConfig->write_config($vmid, $conf);
5001 my $safe_num_ne = sub {
5004 return 0 if !defined($a) && !defined($b);
5005 return 1 if !defined($a);
5006 return 1 if !defined($b);
5011 my $safe_string_ne = sub {
5014 return 0 if !defined($a) && !defined($b);
5015 return 1 if !defined($a);
5016 return 1 if !defined($b);
5021 sub vmconfig_update_net {
5022 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5024 my $newnet = parse_net($value);
5026 if ($conf->{$opt}) {
5027 my $oldnet = parse_net($conf->{$opt});
5029 if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5030 &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5031 &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5032 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5034 # for non online change, we try to hot-unplug
5035 die "skip\n" if !$hotplug;
5036 vm_deviceunplug($vmid, $conf, $opt);
5039 die "internal error" if $opt !~ m/net(\d+)/;
5040 my $iface = "tap${vmid}i$1";
5042 if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5043 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5044 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5045 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5046 PVE::Network::tap_unplug($iface);
5047 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5048 } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5049 # Rate can be applied on its own but any change above needs to
5050 # include the rate in tap_plug since OVS resets everything.
5051 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5054 if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5055 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5063 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5069 sub vmconfig_update_disk {
5070 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5072 # fixme: do we need force?
5074 my $drive = parse_drive($opt, $value);
5076 if ($conf->{$opt}) {
5078 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
5080 my $media = $drive->{media} || 'disk
';
5081 my $oldmedia = $old_drive->{media} || 'disk
';
5082 die "unable to change media type\n" if $media ne $oldmedia;
5084 if (!drive_is_cdrom($old_drive)) {
5086 if ($drive->{file} ne $old_drive->{file}) {
5088 die "skip\n" if !$hotplug;
5090 # unplug and register as unused
5091 vm_deviceunplug($vmid, $conf, $opt);
5092 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5095 # update existing disk
5097 # skip non hotpluggable value
5098 if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5099 &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5100 &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5101 &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5106 if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5107 &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5108 &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5109 &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5110 &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5111 &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5112 &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5113 &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5114 &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5115 &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5116 &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5117 &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5118 &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5119 &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5120 &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5121 &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5122 &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5123 &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5125 qemu_block_set_io_throttle($vmid,"drive-$opt",
5126 ($drive->{mbps} || 0)*1024*1024,
5127 ($drive->{mbps_rd} || 0)*1024*1024,
5128 ($drive->{mbps_wr} || 0)*1024*1024,
5129 $drive->{iops} || 0,
5130 $drive->{iops_rd} || 0,
5131 $drive->{iops_wr} || 0,
5132 ($drive->{mbps_max} || 0)*1024*1024,
5133 ($drive->{mbps_rd_max} || 0)*1024*1024,
5134 ($drive->{mbps_wr_max} || 0)*1024*1024,
5135 $drive->{iops_max} || 0,
5136 $drive->{iops_rd_max} || 0,
5137 $drive->{iops_wr_max} || 0,
5138 $drive->{bps_max_length} || 1,
5139 $drive->{bps_rd_max_length} || 1,
5140 $drive->{bps_wr_max_length} || 1,
5141 $drive->{iops_max_length} || 1,
5142 $drive->{iops_rd_max_length} || 1,
5143 $drive->{iops_wr_max_length} || 1);
5152 if ($drive->{file} eq 'none
') {
5153 vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5154 if (drive_is_cloudinit($old_drive)) {
5155 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5158 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5159 vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5160 vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5168 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5170 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5171 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5175 my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5176 $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5178 PVE::QemuConfig->lock_config($vmid, sub {
5179 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5181 die "you can't start a vm
if it
's a template\n" if PVE::QemuConfig->is_template($conf);
5183 my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended
');
5185 PVE::QemuConfig->check_lock($conf)
5186 if !($skiplock || $is_suspended);
5188 die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5190 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5191 vmconfig_apply_pending($vmid, $conf, $storecfg);
5192 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5195 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5197 my $defaults = load_defaults();
5199 # set environment variable useful inside network script
5200 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5202 my $local_volumes = {};
5204 if ($targetstorage) {
5205 foreach_drive($conf, sub {
5206 my ($ds, $drive) = @_;
5208 return if drive_is_cdrom($drive);
5210 my $volid = $drive->{file};
5214 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5216 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5217 return if $scfg->{shared};
5218 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5223 foreach my $opt (sort keys %$local_volumes) {
5225 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5226 my $drive = parse_drive($opt, $conf->{$opt});
5228 #if remote storage is specified, use default format
5229 if ($targetstorage && $targetstorage ne "1") {
5230 $storeid = $targetstorage;
5231 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5232 $format = $defFormat;
5234 #else we use same format than original
5235 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5236 $format = qemu_img_format($scfg, $volid);
5239 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5240 my $newdrive = $drive;
5241 $newdrive->{format} = $format;
5242 $newdrive->{file} = $newvolid;
5243 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5244 $local_volumes->{$opt} = $drivestr;
5245 #pass drive to conf for command line
5246 $conf->{$opt} = $drivestr;
5250 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start
', 1);
5252 if ($is_suspended) {
5253 # enforce machine type on suspended vm to ensure HW compatibility
5254 $forcemachine = $conf->{runningmachine};
5255 print "Resuming suspended VM\n";
5258 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5260 my $migrate_port = 0;
5263 if ($statefile eq 'tcp
') {
5264 my $localip = "localhost";
5265 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
5266 my $nodename = PVE::INotify::nodename();
5268 if (!defined($migration_type)) {
5269 if (defined($datacenterconf->{migration}->{type})) {
5270 $migration_type = $datacenterconf->{migration}->{type};
5272 $migration_type = 'secure
';
5276 if ($migration_type eq 'insecure
') {
5277 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5278 if ($migrate_network_addr) {
5279 $localip = $migrate_network_addr;
5281 $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5284 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5287 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5288 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5289 $migrate_uri = "tcp:${localip}:${migrate_port}";
5290 push @$cmd, '-incoming
', $migrate_uri;
5293 } elsif ($statefile eq 'unix
') {
5294 # should be default for secure migrations as a ssh TCP forward
5295 # tunnel is not deterministic reliable ready and fails regurarly
5296 # to set up in time, so use UNIX socket forwards
5297 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5298 unlink $socket_addr;
5300 $migrate_uri = "unix:$socket_addr";
5302 push @$cmd, '-incoming
', $migrate_uri;
5306 push @$cmd, '-loadstate
', $statefile;
5313 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
5314 my $d = parse_hostpci($conf->{"hostpci$i"});
5316 my $pcidevices = $d->{pciid};
5317 foreach my $pcidevice (@$pcidevices) {
5318 my $pciid = $pcidevice->{id};
5320 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5321 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5322 die "no pci device info for device '$pciid'\n" if !$info;
5325 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5326 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5328 die "can't unbind
/bind pci group to vfio
'$pciid'\n"
5329 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5330 die "can
't reset pci device '$pciid'\n"
5331 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5336 PVE::Storage::activate_volumes($storecfg, $vollist);
5339 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
5340 outfunc => sub {}, errfunc => sub {});
5342 # Issues with the above 'stop
' not being fully completed are extremely rare, a very low
5343 # timeout should be more than enough here...
5344 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5346 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5347 : $defaults->{cpuunits};
5349 my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5350 my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5353 Slice => 'qemu
.slice
',
5355 CPUShares => $cpuunits
5358 if (my $cpulimit = $conf->{cpulimit}) {
5359 $properties{CPUQuota} = int($cpulimit * 100);
5361 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5363 my $run_qemu = sub {
5364 PVE::Tools::run_fork sub {
5365 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5366 run_command($cmd, %run_params);
5370 if ($conf->{hugepages}) {
5373 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5374 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5376 PVE::QemuServer::Memory::hugepages_mount();
5377 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5379 eval { $run_qemu->() };
5381 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5385 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5387 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5390 eval { $run_qemu->() };
5394 # deactivate volumes if start fails
5395 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5396 die "start failed: $err";
5399 print "migration listens on $migrate_uri\n" if $migrate_uri;
5401 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
5402 eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5406 #start nbd server for storage migration
5407 if ($targetstorage) {
5408 my $nodename = PVE::INotify::nodename();
5409 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5410 my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5411 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5412 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5414 vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet
', data => { host => "${localip}", port => "${migrate_port}" } } );
5416 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5418 foreach my $opt (sort keys %$local_volumes) {
5419 my $volid = $local_volumes->{$opt};
5420 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5421 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5422 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5426 if ($migratedfrom) {
5428 set_migration_caps($vmid);
5433 print "spice listens on port $spice_port\n";
5434 if ($spice_ticket) {
5435 vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice
', password => $spice_ticket);
5436 vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice
', time => "+30");
5441 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5442 if !$statefile && $conf->{balloon};
5444 foreach my $opt (keys %$conf) {
5445 next if $opt !~ m/^net\d+$/;
5446 my $nicconf = parse_net($conf->{$opt});
5447 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5451 vm_mon_cmd_nocheck($vmid, 'qom-set
',
5452 path => "machine/peripheral/balloon0",
5453 property => "guest-stats-polling-interval",
5454 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5456 if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5457 print "Resumed VM, removing state\n";
5458 delete $conf->@{qw(lock vmstate runningmachine)};
5459 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5460 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5461 PVE
::QemuConfig-
>write_config($vmid, $conf);
5464 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5469 my ($vmid, $execute, %params) = @_;
5471 my $cmd = { execute
=> $execute, arguments
=> \
%params };
5472 vm_qmp_command
($vmid, $cmd);
5475 sub vm_mon_cmd_nocheck
{
5476 my ($vmid, $execute, %params) = @_;
5478 my $cmd = { execute
=> $execute, arguments
=> \
%params };
5479 vm_qmp_command
($vmid, $cmd, 1);
5482 sub vm_qmp_command
{
5483 my ($vmid, $cmd, $nocheck) = @_;
5488 if ($cmd->{arguments
} && $cmd->{arguments
}->{timeout
}) {
5489 $timeout = $cmd->{arguments
}->{timeout
};
5490 delete $cmd->{arguments
}->{timeout
};
5494 die "VM $vmid not running\n" if !check_running
($vmid, $nocheck);
5495 my $sname = qmp_socket
($vmid);
5496 if (-e
$sname) { # test if VM is reasonambe new and supports qmp/qga
5497 my $qmpclient = PVE
::QMPClient-
>new();
5499 $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5501 die "unable to open monitor socket\n";
5505 syslog
("err", "VM $vmid qmp command failed - $err");
5512 sub vm_human_monitor_command
{
5513 my ($vmid, $cmdline) = @_;
5518 execute
=> 'human-monitor-command',
5519 arguments
=> { 'command-line' => $cmdline},
5522 return vm_qmp_command
($vmid, $cmd);
5525 sub vm_commandline
{
5526 my ($storecfg, $vmid, $snapname) = @_;
5528 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5531 my $snapshot = $conf->{snapshots
}->{$snapname};
5532 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5534 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5539 my $defaults = load_defaults
();
5541 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults);
5543 return PVE
::Tools
::cmd2string
($cmd);
5547 my ($vmid, $skiplock) = @_;
5549 PVE
::QemuConfig-
>lock_config($vmid, sub {
5551 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5553 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5555 vm_mon_cmd
($vmid, "system_reset");
5559 sub get_vm_volumes
{
5563 foreach_volid
($conf, sub {
5564 my ($volid, $attr) = @_;
5566 return if $volid =~ m
|^/|;
5568 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5571 push @$vollist, $volid;
5577 sub vm_stop_cleanup
{
5578 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5583 my $vollist = get_vm_volumes
($conf);
5584 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5587 foreach my $ext (qw(mon qmp pid vnc qga)) {
5588 unlink "/var/run/qemu-server/${vmid}.$ext";
5591 if ($conf->{ivshmem
}) {
5592 my $ivshmem = PVE
::JSONSchema
::parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5593 # just delete it for now, VMs which have this already open do not
5594 # are affected, but new VMs will get a separated one. If this
5595 # becomes an issue we either add some sort of ref-counting or just
5596 # add a "don't delete on stop" flag to the ivshmem format.
5597 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5600 foreach my $key (keys %$conf) {
5601 next if $key !~ m/^hostpci(\d+)$/;
5602 my $hostpciindex = $1;
5603 my $d = parse_hostpci
($conf->{$key});
5604 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5606 foreach my $pci (@{$d->{pciid
}}) {
5607 my $pciid = $pci->{id
};
5608 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5612 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5614 warn $@ if $@; # avoid errors - just warn
5617 # Note: use $nockeck to skip tests if VM configuration file exists.
5618 # We need that when migration VMs to other nodes (files already moved)
5619 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5621 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5623 $force = 1 if !defined($force) && !$shutdown;
5626 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5627 kill 15, $pid if $pid;
5628 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5629 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5633 PVE
::QemuConfig-
>lock_config($vmid, sub {
5635 my $pid = check_running
($vmid, $nocheck);
5640 $conf = PVE
::QemuConfig-
>load_config($vmid);
5641 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5642 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5643 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5644 $timeout = $opts->{down
} if $opts->{down
};
5646 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5651 if (defined($conf) && parse_guest_agent
($conf)->{enabled
}) {
5652 vm_qmp_command
($vmid, {
5653 execute
=> "guest-shutdown",
5654 arguments
=> { timeout
=> $timeout }
5657 vm_qmp_command
($vmid, { execute
=> "system_powerdown" }, $nocheck);
5660 vm_qmp_command
($vmid, { execute
=> "quit" }, $nocheck);
5666 $timeout = 60 if !defined($timeout);
5669 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5674 if ($count >= $timeout) {
5676 warn "VM still running - terminating now with SIGTERM\n";
5679 die "VM quit/powerdown failed - got timeout\n";
5682 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5687 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5690 die "VM quit/powerdown failed\n";
5698 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5703 if ($count >= $timeout) {
5704 warn "VM still running - terminating now with SIGKILL\n";
5709 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5714 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5721 PVE
::QemuConfig-
>lock_config($vmid, sub {
5723 $conf = PVE
::QemuConfig-
>load_config($vmid);
5725 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5726 PVE
::QemuConfig-
>check_lock($conf)
5727 if !($skiplock || $is_backing_up);
5729 die "cannot suspend to disk during backup\n"
5730 if $is_backing_up && $includestate;
5732 if ($includestate) {
5733 $conf->{lock} = 'suspending';
5734 my $date = strftime
("%Y-%m-%d", localtime(time()));
5735 $storecfg = PVE
::Storage
::config
();
5736 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5737 $path = PVE
::Storage
::path
($storecfg, $vmstate);
5738 PVE
::QemuConfig-
>write_config($vmid, $conf);
5740 vm_mon_cmd
($vmid, "stop");
5744 if ($includestate) {
5746 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
5749 vm_mon_cmd
($vmid, "savevm-start", statefile
=> $path);
5751 my $state = vm_mon_cmd_nocheck
($vmid, "query-savevm");
5752 if (!$state->{status
}) {
5753 die "savevm not active\n";
5754 } elsif ($state->{status
} eq 'active') {
5757 } elsif ($state->{status
} eq 'completed') {
5758 print "State saved, quitting\n";
5760 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
5761 die "query-savevm failed with error '$state->{error}'\n"
5763 die "query-savevm returned status '$state->{status}'\n";
5769 PVE
::QemuConfig-
>lock_config($vmid, sub {
5770 $conf = PVE
::QemuConfig-
>load_config($vmid);
5772 # cleanup, but leave suspending lock, to indicate something went wrong
5774 vm_mon_cmd
($vmid, "savevm-end");
5775 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5776 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5777 delete $conf->@{qw(vmstate runningmachine)};
5778 PVE
::QemuConfig-
>write_config($vmid, $conf);
5784 die "lock changed unexpectedly\n"
5785 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
5787 vm_qmp_command
($vmid, { execute
=> "quit" });
5788 $conf->{lock} = 'suspended';
5789 PVE
::QemuConfig-
>write_config($vmid, $conf);
5795 my ($vmid, $skiplock, $nocheck) = @_;
5797 PVE
::QemuConfig-
>lock_config($vmid, sub {
5798 my $vm_mon_cmd = $nocheck ? \
&vm_mon_cmd_nocheck
: \
&vm_mon_cmd
;
5799 my $res = $vm_mon_cmd->($vmid, 'query-status');
5800 my $resume_cmd = 'cont';
5802 if ($res->{status
} && $res->{status
} eq 'suspended') {
5803 $resume_cmd = 'system_wakeup';
5808 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5810 PVE
::QemuConfig-
>check_lock($conf)
5811 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5814 $vm_mon_cmd->($vmid, $resume_cmd);
5819 my ($vmid, $skiplock, $key) = @_;
5821 PVE
::QemuConfig-
>lock_config($vmid, sub {
5823 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5825 # there is no qmp command, so we use the human monitor command
5826 vm_human_monitor_command
($vmid, "sendkey $key");
5831 my ($storecfg, $vmid, $skiplock) = @_;
5833 PVE
::QemuConfig-
>lock_config($vmid, sub {
5835 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5837 if (!check_running
($vmid)) {
5838 destroy_vm
($storecfg, $vmid, undef, $skiplock);
5840 die "VM $vmid is running - destroy failed\n";
5845 # vzdump restore implementaion
5847 sub tar_archive_read_firstfile
{
5848 my $archive = shift;
5850 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5852 # try to detect archive type first
5853 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5854 die "unable to open file '$archive'\n";
5855 my $firstfile = <$fh>;
5859 die "ERROR: archive contaions no data\n" if !$firstfile;
5865 sub tar_restore_cleanup
{
5866 my ($storecfg, $statfile) = @_;
5868 print STDERR
"starting cleanup\n";
5870 if (my $fd = IO
::File-
>new($statfile, "r")) {
5871 while (defined(my $line = <$fd>)) {
5872 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5875 if ($volid =~ m
|^/|) {
5876 unlink $volid || die 'unlink failed\n';
5878 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5880 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5882 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5884 print STDERR
"unable to parse line in statfile - $line";
5891 sub restore_archive
{
5892 my ($archive, $vmid, $user, $opts) = @_;
5894 my $format = $opts->{format
};
5897 if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5898 $format = 'tar' if !$format;
5900 } elsif ($archive =~ m/\.tar$/) {
5901 $format = 'tar' if !$format;
5902 } elsif ($archive =~ m/.tar.lzo$/) {
5903 $format = 'tar' if !$format;
5905 } elsif ($archive =~ m/\.vma$/) {
5906 $format = 'vma' if !$format;
5907 } elsif ($archive =~ m/\.vma\.gz$/) {
5908 $format = 'vma' if !$format;
5910 } elsif ($archive =~ m/\.vma\.lzo$/) {
5911 $format = 'vma' if !$format;
5914 $format = 'vma' if !$format; # default
5917 # try to detect archive format
5918 if ($format eq 'tar') {
5919 return restore_tar_archive
($archive, $vmid, $user, $opts);
5921 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5925 sub restore_update_config_line
{
5926 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5928 return if $line =~ m/^\#qmdump\#/;
5929 return if $line =~ m/^\#vzdump\#/;
5930 return if $line =~ m/^lock:/;
5931 return if $line =~ m/^unused\d+:/;
5932 return if $line =~ m/^parent:/;
5934 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5935 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5936 # try to convert old 1.X settings
5937 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5938 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5939 my ($model, $macaddr) = split(/\=/, $devconfig);
5940 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5943 bridge
=> "vmbr$ind",
5944 macaddr
=> $macaddr,
5946 my $netstr = print_net
($net);
5948 print $outfd "net$cookie->{netcount}: $netstr\n";
5949 $cookie->{netcount
}++;
5951 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5952 my ($id, $netstr) = ($1, $2);
5953 my $net = parse_net
($netstr);
5954 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5955 $netstr = print_net
($net);
5956 print $outfd "$id: $netstr\n";
5957 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5960 my $di = parse_drive
($virtdev, $value);
5961 if (defined($di->{backup
}) && !$di->{backup
}) {
5962 print $outfd "#$line";
5963 } elsif ($map->{$virtdev}) {
5964 delete $di->{format
}; # format can change on restore
5965 $di->{file
} = $map->{$virtdev};
5966 $value = print_drive
($vmid, $di);
5967 print $outfd "$virtdev: $value\n";
5971 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5973 if ($vmgenid ne '0') {
5974 # always generate a new vmgenid if there was a valid one setup
5975 $vmgenid = generate_uuid
();
5977 print $outfd "vmgenid: $vmgenid\n";
5978 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5979 my ($uuid, $uuid_str);
5980 UUID
::generate
($uuid);
5981 UUID
::unparse
($uuid, $uuid_str);
5982 my $smbios1 = parse_smbios1
($2);
5983 $smbios1->{uuid
} = $uuid_str;
5984 print $outfd $1.print_smbios1
($smbios1)."\n";
5991 my ($cfg, $vmid) = @_;
5993 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5995 my $volid_hash = {};
5996 foreach my $storeid (keys %$info) {
5997 foreach my $item (@{$info->{$storeid}}) {
5998 next if !($item->{volid
} && $item->{size
});
5999 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6000 $volid_hash->{$item->{volid
}} = $item;
6007 sub is_volume_in_use
{
6008 my ($storecfg, $conf, $skip_drive, $volid) = @_;
6010 my $path = PVE
::Storage
::path
($storecfg, $volid);
6012 my $scan_config = sub {
6013 my ($cref, $snapname) = @_;
6015 foreach my $key (keys %$cref) {
6016 my $value = $cref->{$key};
6017 if (is_valid_drivename
($key)) {
6018 next if $skip_drive && $key eq $skip_drive;
6019 my $drive = parse_drive
($key, $value);
6020 next if !$drive || !$drive->{file
} || drive_is_cdrom
($drive);
6021 return 1 if $volid eq $drive->{file
};
6022 if ($drive->{file
} =~ m!^/!) {
6023 return 1 if $drive->{file
} eq $path;
6025 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
}, 1);
6027 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid, 1);
6029 return 1 if $path eq PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
6037 return 1 if &$scan_config($conf);
6041 foreach my $snapname (keys %{$conf->{snapshots
}}) {
6042 return 1 if &$scan_config($conf->{snapshots
}->{$snapname}, $snapname);
6048 sub update_disksize
{
6049 my ($vmid, $conf, $volid_hash) = @_;
6052 my $prefix = "VM $vmid:";
6054 # used and unused disks
6055 my $referenced = {};
6057 # Note: it is allowed to define multiple storages with same path (alias), so
6058 # we need to check both 'volid' and real 'path' (two different volid can point
6059 # to the same path).
6061 my $referencedpath = {};
6064 foreach my $opt (keys %$conf) {
6065 if (is_valid_drivename
($opt)) {
6066 my $drive = parse_drive
($opt, $conf->{$opt});
6067 my $volid = $drive->{file
};
6070 $referenced->{$volid} = 1;
6071 if ($volid_hash->{$volid} &&
6072 (my $path = $volid_hash->{$volid}->{path
})) {
6073 $referencedpath->{$path} = 1;
6076 next if drive_is_cdrom
($drive);
6077 next if !$volid_hash->{$volid};
6079 $drive->{size
} = $volid_hash->{$volid}->{size
};
6080 my $new = print_drive
($vmid, $drive);
6081 if ($new ne $conf->{$opt}) {
6083 $conf->{$opt} = $new;
6084 print "$prefix update disk '$opt' information.\n";
6089 # remove 'unusedX' entry if volume is used
6090 foreach my $opt (keys %$conf) {
6091 next if $opt !~ m/^unused\d+$/;
6092 my $volid = $conf->{$opt};
6093 my $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6094 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6095 print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6097 delete $conf->{$opt};
6100 $referenced->{$volid} = 1;
6101 $referencedpath->{$path} = 1 if $path;
6104 foreach my $volid (sort keys %$volid_hash) {
6105 next if $volid =~ m/vm-$vmid-state-/;
6106 next if $referenced->{$volid};
6107 my $path = $volid_hash->{$volid}->{path
};
6108 next if !$path; # just to be sure
6109 next if $referencedpath->{$path};
6111 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6112 print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6113 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6120 my ($vmid, $nolock, $dryrun) = @_;
6122 my $cfg = PVE
::Storage
::config
();
6124 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6125 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6126 foreach my $stor (keys %{$cfg->{ids
}}) {
6127 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
6130 print "rescan volumes...\n";
6131 my $volid_hash = scan_volids
($cfg, $vmid);
6133 my $updatefn = sub {
6136 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6138 PVE
::QemuConfig-
>check_lock($conf);
6141 foreach my $volid (keys %$volid_hash) {
6142 my $info = $volid_hash->{$volid};
6143 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6146 my $changes = update_disksize
($vmid, $conf, $vm_volids);
6148 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6151 if (defined($vmid)) {
6155 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6158 my $vmlist = config_list
();
6159 foreach my $vmid (keys %$vmlist) {
6163 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6169 sub restore_vma_archive
{
6170 my ($archive, $vmid, $user, $opts, $comp) = @_;
6172 my $readfrom = $archive;
6174 my $cfg = PVE
::Storage
::config
();
6176 my $bwlimit = $opts->{bwlimit
};
6178 my $dbg_cmdstring = '';
6179 my $add_pipe = sub {
6181 push @$commands, $cmd;
6182 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6183 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6188 if ($archive eq '-') {
6191 # If we use a backup from a PVE defined storage we also consider that
6192 # storage's rate limit:
6193 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6194 if (defined($volid)) {
6195 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6196 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6198 print STDERR
"applying read rate limit: $readlimit\n";
6199 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6200 $add_pipe->($cstream);
6207 if ($comp eq 'gzip') {
6208 $cmd = ['zcat', $readfrom];
6209 } elsif ($comp eq 'lzop') {
6210 $cmd = ['lzop', '-d', '-c', $readfrom];
6212 die "unknown compression method '$comp'\n";
6217 my $tmpdir = "/var/tmp/vzdumptmp$$";
6220 # disable interrupts (always do cleanups)
6224 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6226 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6227 POSIX
::mkfifo
($mapfifo, 0600);
6230 my $openfifo = sub {
6231 open($fifofh, '>', $mapfifo) || die $!;
6234 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6241 my $rpcenv = PVE
::RPCEnvironment
::get
();
6243 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6244 my $tmpfn = "$conffile.$$.tmp";
6246 # Note: $oldconf is undef if VM does not exists
6247 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6248 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6252 my $print_devmap = sub {
6253 my $virtdev_hash = {};
6255 my $cfgfn = "$tmpdir/qemu-server.conf";
6257 # we can read the config - that is already extracted
6258 my $fh = IO
::File-
>new($cfgfn, "r") ||
6259 "unable to read qemu-server.conf - $!\n";
6261 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6263 my $pve_firewall_dir = '/etc/pve/firewall';
6264 mkdir $pve_firewall_dir; # make sure the dir exists
6265 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6268 while (defined(my $line = <$fh>)) {
6269 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6270 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6271 die "archive does not contain data for drive '$virtdev'\n"
6272 if !$devinfo->{$devname};
6273 if (defined($opts->{storage
})) {
6274 $storeid = $opts->{storage
} || 'local';
6275 } elsif (!$storeid) {
6278 $format = 'raw' if !$format;
6279 $devinfo->{$devname}->{devname
} = $devname;
6280 $devinfo->{$devname}->{virtdev
} = $virtdev;
6281 $devinfo->{$devname}->{format
} = $format;
6282 $devinfo->{$devname}->{storeid
} = $storeid;
6284 # check permission on storage
6285 my $pool = $opts->{pool
}; # todo: do we need that?
6286 if ($user ne 'root@pam') {
6287 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6290 $storage_limits{$storeid} = $bwlimit;
6292 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6293 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6295 my $drive = parse_drive
($virtdev, $2);
6296 if (drive_is_cloudinit
($drive)) {
6297 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6298 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
6299 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6303 storeid
=> $opts->{storage
} // $storeid,
6304 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6305 file
=> $drive->{file
}, # to make drive_is_cloudinit check possible
6306 name
=> "vm-$vmid-cloudinit",
6309 $virtdev_hash->{$virtdev} = $d;
6314 foreach my $key (keys %storage_limits) {
6315 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
6317 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
6318 $storage_limits{$key} = $limit * 1024;
6321 foreach my $devname (keys %$devinfo) {
6322 die "found no device mapping information for device '$devname'\n"
6323 if !$devinfo->{$devname}->{virtdev
};
6326 # create empty/temp config
6328 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6329 foreach_drive
($oldconf, sub {
6330 my ($ds, $drive) = @_;
6332 return if !$drive->{is_cloudinit
} && drive_is_cdrom
($drive);
6334 my $volid = $drive->{file
};
6335 return if !$volid || $volid =~ m
|^/|;
6337 my ($path, $owner) = PVE
::Storage
::path
($cfg, $volid);
6338 return if !$path || !$owner || ($owner != $vmid);
6340 # Note: only delete disk we want to restore
6341 # other volumes will become unused
6342 if ($virtdev_hash->{$ds}) {
6343 eval { PVE
::Storage
::vdisk_free
($cfg, $volid); };
6350 # delete vmstate files, after the restore we have no snapshots anymore
6351 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
6352 my $snap = $oldconf->{snapshots
}->{$snapname};
6353 if ($snap->{vmstate
}) {
6354 eval { PVE
::Storage
::vdisk_free
($cfg, $snap->{vmstate
}); };
6363 foreach my $virtdev (sort keys %$virtdev_hash) {
6364 my $d = $virtdev_hash->{$virtdev};
6365 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6366 my $storeid = $d->{storeid
};
6367 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
6370 if (my $limit = $storage_limits{$storeid}) {
6371 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6374 # test if requested format is supported
6375 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($cfg, $storeid);
6376 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6377 $d->{format
} = $defFormat if !$supported;
6380 if ($d->{is_cloudinit
}) {
6382 $name .= ".$d->{format}" if $d->{format
} ne 'raw';
6385 my $volid = PVE
::Storage
::vdisk_alloc
($cfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6386 print STDERR
"new volume ID is '$volid'\n";
6387 $d->{volid
} = $volid;
6389 PVE
::Storage
::activate_volumes
($cfg, [$volid]);
6391 my $write_zeros = 1;
6392 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
6396 if (!$d->{is_cloudinit
}) {
6397 my $path = PVE
::Storage
::path
($cfg, $volid);
6399 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6401 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6403 $map->{$virtdev} = $volid;
6406 $fh->seek(0, 0) || die "seek failed - $!\n";
6408 my $outfd = new IO
::File
($tmpfn, "w") ||
6409 die "unable to write config for VM $vmid\n";
6411 my $cookie = { netcount
=> 0 };
6412 while (defined(my $line = <$fh>)) {
6413 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6426 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6427 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6429 $oldtimeout = alarm($timeout);
6436 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6437 my ($dev_id, $size, $devname) = ($1, $2, $3);
6438 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6439 } elsif ($line =~ m/^CTIME: /) {
6440 # we correctly received the vma config, so we can disable
6441 # the timeout now for disk allocation (set to 10 minutes, so
6442 # that we always timeout if something goes wrong)
6445 print $fifofh "done\n";
6446 my $tmp = $oldtimeout || 0;
6447 $oldtimeout = undef;
6453 print "restore vma archive: $dbg_cmdstring\n";
6454 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6458 alarm($oldtimeout) if $oldtimeout;
6461 foreach my $devname (keys %$devinfo) {
6462 my $volid = $devinfo->{$devname}->{volid
};
6463 push @$vollist, $volid if $volid;
6466 PVE
::Storage
::deactivate_volumes
($cfg, $vollist);
6474 foreach my $devname (keys %$devinfo) {
6475 my $volid = $devinfo->{$devname}->{volid
};
6478 if ($volid =~ m
|^/|) {
6479 unlink $volid || die 'unlink failed\n';
6481 PVE
::Storage
::vdisk_free
($cfg, $volid);
6483 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6485 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6492 rename($tmpfn, $conffile) ||
6493 die "unable to commit configuration file '$conffile'\n";
6495 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6497 eval { rescan
($vmid, 1); };
6501 sub restore_tar_archive
{
6502 my ($archive, $vmid, $user, $opts) = @_;
6504 if ($archive ne '-') {
6505 my $firstfile = tar_archive_read_firstfile
($archive);
6506 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6507 if $firstfile ne 'qemu-server.conf';
6510 my $storecfg = PVE
::Storage
::config
();
6512 # destroy existing data - keep empty config
6513 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6514 destroy_vm
($storecfg, $vmid, 1) if -f
$vmcfgfn;
6516 my $tocmd = "/usr/lib/qemu-server/qmextract";
6518 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6519 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6520 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6521 $tocmd .= ' --info' if $opts->{info
};
6523 # tar option "xf" does not autodetect compression when read from STDIN,
6524 # so we pipe to zcat
6525 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6526 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6528 my $tmpdir = "/var/tmp/vzdumptmp$$";
6531 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6532 local $ENV{VZDUMP_VMID
} = $vmid;
6533 local $ENV{VZDUMP_USER
} = $user;
6535 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6536 my $tmpfn = "$conffile.$$.tmp";
6538 # disable interrupts (always do cleanups)
6542 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6550 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6552 if ($archive eq '-') {
6553 print "extracting archive from STDIN\n";
6554 run_command
($cmd, input
=> "<&STDIN");
6556 print "extracting archive '$archive'\n";
6560 return if $opts->{info
};
6564 my $statfile = "$tmpdir/qmrestore.stat";
6565 if (my $fd = IO
::File-
>new($statfile, "r")) {
6566 while (defined (my $line = <$fd>)) {
6567 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6568 $map->{$1} = $2 if $1;
6570 print STDERR
"unable to parse line in statfile - $line\n";
6576 my $confsrc = "$tmpdir/qemu-server.conf";
6578 my $srcfd = new IO
::File
($confsrc, "r") ||
6579 die "unable to open file '$confsrc'\n";
6581 my $outfd = new IO
::File
($tmpfn, "w") ||
6582 die "unable to write config for VM $vmid\n";
6584 my $cookie = { netcount
=> 0 };
6585 while (defined (my $line = <$srcfd>)) {
6586 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6598 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6605 rename $tmpfn, $conffile ||
6606 die "unable to commit configuration file '$conffile'\n";
6608 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6610 eval { rescan
($vmid, 1); };
6614 sub foreach_storage_used_by_vm
{
6615 my ($conf, $func) = @_;
6619 foreach_drive
($conf, sub {
6620 my ($ds, $drive) = @_;
6621 return if drive_is_cdrom
($drive);
6623 my $volid = $drive->{file
};
6625 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6626 $sidhash->{$sid} = $sid if $sid;
6629 foreach my $sid (sort keys %$sidhash) {
6634 sub do_snapshots_with_qemu
{
6635 my ($storecfg, $volid) = @_;
6637 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6638 my $scfg = $storecfg->{ids
}->{$storage_name};
6640 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
6644 if ($volid =~ m/\.(qcow2|qed)$/){
6651 sub qga_check_running
{
6652 my ($vmid, $nowarn) = @_;
6654 eval { vm_mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6656 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6662 sub template_create
{
6663 my ($vmid, $conf, $disk) = @_;
6665 my $storecfg = PVE
::Storage
::config
();
6667 foreach_drive
($conf, sub {
6668 my ($ds, $drive) = @_;
6670 return if drive_is_cdrom
($drive);
6671 return if $disk && $ds ne $disk;
6673 my $volid = $drive->{file
};
6674 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6676 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6677 $drive->{file
} = $voliddst;
6678 $conf->{$ds} = print_drive
($vmid, $drive);
6679 PVE
::QemuConfig-
>write_config($vmid, $conf);
6683 sub convert_iscsi_path
{
6686 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
6691 my $initiator_name = get_initiator_name
();
6693 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6694 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6697 die "cannot convert iscsi path '$path', unkown format\n";
6700 sub qemu_img_convert
{
6701 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6703 my $storecfg = PVE
::Storage
::config
();
6704 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6705 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6707 if ($src_storeid && $dst_storeid) {
6709 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6711 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6712 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6714 my $src_format = qemu_img_format
($src_scfg, $src_volname);
6715 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6717 my $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6718 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6720 my $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
6721 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
6724 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6725 push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6726 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6727 push @$cmd, '-T', 'none' if $src_scfg->{type
} eq 'zfspool';
6729 if ($src_is_iscsi) {
6730 push @$cmd, '--image-opts';
6731 $src_path = convert_iscsi_path
($src_path);
6733 push @$cmd, '-f', $src_format;
6736 if ($dst_is_iscsi) {
6737 push @$cmd, '--target-image-opts';
6738 $dst_path = convert_iscsi_path
($dst_path);
6740 push @$cmd, '-O', $dst_format;
6743 push @$cmd, $src_path;
6745 if (!$dst_is_iscsi && $is_zero_initialized) {
6746 push @$cmd, "zeroinit:$dst_path";
6748 push @$cmd, $dst_path;
6753 if($line =~ m/\((\S+)\/100\
%\)/){
6755 my $transferred = int($size * $percent / 100);
6756 my $remaining = $size - $transferred;
6758 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6763 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6765 die "copy failed: $err" if $err;
6769 sub qemu_img_format
{
6770 my ($scfg, $volname) = @_;
6772 if ($scfg->{path
} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6779 sub qemu_drive_mirror
{
6780 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6782 $jobs = {} if !$jobs;
6786 $jobs->{"drive-$drive"} = {};
6788 if ($dst_volid =~ /^nbd:/) {
6789 $qemu_target = $dst_volid;
6792 my $storecfg = PVE
::Storage
::config
();
6793 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6795 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6797 $format = qemu_img_format
($dst_scfg, $dst_volname);
6799 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6801 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6804 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6805 $opts->{format
} = $format if $format;
6807 if (defined($bwlimit)) {
6808 $opts->{speed
} = $bwlimit * 1024;
6809 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6811 print "drive mirror is starting for drive-$drive\n";
6814 # if a job already runs for this device we get an error, catch it for cleanup
6815 eval { vm_mon_cmd
($vmid, "drive-mirror", %$opts); };
6817 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6819 die "mirroring error: $err\n";
6822 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6825 sub qemu_drive_mirror_monitor
{
6826 my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6829 my $err_complete = 0;
6832 die "storage migration timed out\n" if $err_complete > 300;
6834 my $stats = vm_mon_cmd
($vmid, "query-block-jobs");
6836 my $running_mirror_jobs = {};
6837 foreach my $stat (@$stats) {
6838 next if $stat->{type
} ne 'mirror';
6839 $running_mirror_jobs->{$stat->{device
}} = $stat;
6842 my $readycounter = 0;
6844 foreach my $job (keys %$jobs) {
6846 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6847 print "$job : finished\n";
6848 delete $jobs->{$job};
6852 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6854 my $busy = $running_mirror_jobs->{$job}->{busy
};
6855 my $ready = $running_mirror_jobs->{$job}->{ready
};
6856 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6857 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6858 my $remaining = $total - $transferred;
6859 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6861 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6864 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6867 last if scalar(keys %$jobs) == 0;
6869 if ($readycounter == scalar(keys %$jobs)) {
6870 print "all mirroring jobs are ready \n";
6871 last if $skipcomplete; #do the complete later
6873 if ($vmiddst && $vmiddst != $vmid) {
6874 my $agent_running = $qga && qga_check_running
($vmid);
6875 if ($agent_running) {
6876 print "freeze filesystem\n";
6877 eval { PVE
::QemuServer
::vm_mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6879 print "suspend vm\n";
6880 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6883 # if we clone a disk for a new target vm, we don't switch the disk
6884 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6886 if ($agent_running) {
6887 print "unfreeze filesystem\n";
6888 eval { PVE
::QemuServer
::vm_mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6890 print "resume vm\n";
6891 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6897 foreach my $job (keys %$jobs) {
6898 # try to switch the disk if source and destination are on the same guest
6899 print "$job: Completing block job...\n";
6901 eval { vm_mon_cmd
($vmid, "block-job-complete", device
=> $job) };
6902 if ($@ =~ m/cannot be completed/) {
6903 print "$job: Block job cannot be completed, try again.\n";
6906 print "$job: Completed successfully.\n";
6907 $jobs->{$job}->{complete
} = 1;
6918 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6919 die "mirroring error: $err";
6924 sub qemu_blockjobs_cancel
{
6925 my ($vmid, $jobs) = @_;
6927 foreach my $job (keys %$jobs) {
6928 print "$job: Cancelling block job\n";
6929 eval { vm_mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6930 $jobs->{$job}->{cancel
} = 1;
6934 my $stats = vm_mon_cmd
($vmid, "query-block-jobs");
6936 my $running_jobs = {};
6937 foreach my $stat (@$stats) {
6938 $running_jobs->{$stat->{device
}} = $stat;
6941 foreach my $job (keys %$jobs) {
6943 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6944 print "$job: Done.\n";
6945 delete $jobs->{$job};
6949 last if scalar(keys %$jobs) == 0;
6956 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6957 $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6962 print "create linked clone of drive $drivename ($drive->{file})\n";
6963 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6964 push @$newvollist, $newvolid;
6967 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6968 $storeid = $storage if $storage;
6970 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6971 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
6973 print "create full clone of drive $drivename ($drive->{file})\n";
6975 if (drive_is_cloudinit
($drive)) {
6976 $name = "vm-$newvmid-cloudinit";
6978 # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
6979 if ($dst_format ne 'raw') {
6980 $name .= ".$dst_format";
6983 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6984 push @$newvollist, $newvolid;
6986 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
6988 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6989 if (!$running || $snapname) {
6990 # TODO: handle bwlimits
6991 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
6994 my $kvmver = get_running_qemu_version
($vmid);
6995 if (!qemu_machine_feature_enabled
(undef, $kvmver, 2, 7)) {
6996 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6997 if $drive->{iothread
};
7000 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7004 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
7007 $disk->{format
} = undef;
7008 $disk->{file
} = $newvolid;
7009 $disk->{size
} = $size;
7014 # this only works if VM is running
7015 sub get_current_qemu_machine
{
7018 my $cmd = { execute
=> 'query-machines', arguments
=> {} };
7019 my $res = vm_qmp_command
($vmid, $cmd);
7021 my ($current, $default);
7022 foreach my $e (@$res) {
7023 $default = $e->{name
} if $e->{'is-default'};
7024 $current = $e->{name
} if $e->{'is-current'};
7027 # fallback to the default machine if current is not supported by qemu
7028 return $current || $default || 'pc';
7031 sub get_running_qemu_version
{
7033 my $cmd = { execute
=> 'query-version', arguments
=> {} };
7034 my $res = vm_qmp_command
($vmid, $cmd);
7035 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7038 sub qemu_machine_feature_enabled
{
7039 my ($machine, $kvmver, $version_major, $version_minor) = @_;
7044 if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7046 $current_major = $3;
7047 $current_minor = $4;
7049 } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7051 $current_major = $1;
7052 $current_minor = $2;
7055 return 1 if $current_major > $version_major ||
7056 ($current_major == $version_major &&
7057 $current_minor >= $version_minor);
7060 sub qemu_machine_pxe
{
7061 my ($vmid, $conf, $machine) = @_;
7063 $machine = PVE
::QemuServer
::get_current_qemu_machine
($vmid) if !$machine;
7065 if ($conf->{machine
} && $conf->{machine
} =~ m/\.pxe$/) {
7072 sub qemu_use_old_bios_files
{
7073 my ($machine_type) = @_;
7075 return if !$machine_type;
7077 my $use_old_bios_files = undef;
7079 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7081 $use_old_bios_files = 1;
7083 my $kvmver = kvm_user_version
();
7084 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7085 # load new efi bios files on migration. So this hack is required to allow
7086 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7087 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7088 $use_old_bios_files = !qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 4);
7091 return ($use_old_bios_files, $machine_type);
7094 sub create_efidisk
($$$$$) {
7095 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7097 my (undef, $ovmf_vars) = get_ovmf_files
($arch);
7098 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7100 my $vars_size = PVE
::Tools
::convert_size
(-s
$ovmf_vars, 'b' => 'kb');
7101 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7102 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7104 my $path = PVE
::Storage
::path
($storecfg, $volid);
7106 run_command
(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7108 die "Copying EFI vars image failed: $@" if $@;
7110 return ($volid, $vars_size);
7113 sub vm_iothreads_list
{
7116 my $res = vm_mon_cmd
($vmid, 'query-iothreads');
7119 foreach my $iothread (@$res) {
7120 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7127 my ($conf, $drive) = @_;
7131 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7133 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7139 my $controller = int($drive->{index} / $maxdev);
7140 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single') ?
"virtioscsi" : "scsihw";
7142 return ($maxdev, $controller, $controller_prefix);
7145 sub add_hyperv_enlightenments
{
7146 my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7148 return if $winversion < 6;
7149 return if $bios && $bios eq 'ovmf' && $winversion < 8;
7151 if ($gpu_passthrough || defined($hv_vendor_id)) {
7152 $hv_vendor_id //= 'proxmox';
7153 push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7156 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3)) {
7157 push @$cpuFlags , 'hv_spinlocks=0x1fff';
7158 push @$cpuFlags , 'hv_vapic';
7159 push @$cpuFlags , 'hv_time';
7161 push @$cpuFlags , 'hv_spinlocks=0xffff';
7164 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 6)) {
7165 push @$cpuFlags , 'hv_reset';
7166 push @$cpuFlags , 'hv_vpindex';
7167 push @$cpuFlags , 'hv_runtime';
7170 if ($winversion >= 7) {
7171 push @$cpuFlags , 'hv_relaxed';
7173 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 12)) {
7174 push @$cpuFlags , 'hv_synic';
7175 push @$cpuFlags , 'hv_stimer';
7178 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 3, 0)) {
7179 push @$cpuFlags , 'hv_tlbflush';
7182 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 3, 1)) {
7183 push @$cpuFlags , 'hv_ipi';
7184 push @$cpuFlags , 'hv_evmcs';
7189 sub windows_version
{
7192 return 0 if !$ostype;
7196 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7198 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7200 } elsif ($ostype =~ m/^win(\d+)$/) {
7207 sub resolve_dst_disk_format
{
7208 my ($storecfg, $storeid, $src_volname, $format) = @_;
7209 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7212 # if no target format is specified, use the source disk format as hint
7214 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7215 $format = qemu_img_format
($scfg, $src_volname);
7221 # test if requested format is supported - else use default
7222 my $supported = grep { $_ eq $format } @$validFormats;
7223 $format = $defFormat if !$supported;
7227 sub resolve_first_disk
{
7229 my @disks = PVE
::QemuServer
::valid_drive_names
();
7231 foreach my $ds (reverse @disks) {
7232 next if !$conf->{$ds};
7233 my $disk = PVE
::QemuServer
::parse_drive
($ds, $conf->{$ds});
7234 next if PVE
::QemuServer
::drive_is_cdrom
($disk);
7241 my ($uuid, $uuid_str);
7242 UUID
::generate
($uuid);
7243 UUID
::unparse
($uuid, $uuid_str);
7247 sub generate_smbios1_uuid
{
7248 return "uuid=".generate_uuid
();
7254 vm_mon_cmd
($vmid, 'nbd-server-stop');
7257 # bash completion helper
7259 sub complete_backup_archives
{
7260 my ($cmdname, $pname, $cvalue) = @_;
7262 my $cfg = PVE
::Storage
::config
();
7266 if ($cvalue =~ m/^([^:]+):/) {
7270 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
7273 foreach my $id (keys %$data) {
7274 foreach my $item (@{$data->{$id}}) {
7275 next if $item->{format
} !~ m/^vma\.(gz|lzo)$/;
7276 push @$res, $item->{volid
} if defined($item->{volid
});
7283 my $complete_vmid_full = sub {
7286 my $idlist = vmstatus
();
7290 foreach my $id (keys %$idlist) {
7291 my $d = $idlist->{$id};
7292 if (defined($running)) {
7293 next if $d->{template
};
7294 next if $running && $d->{status
} ne 'running';
7295 next if !$running && $d->{status
} eq 'running';
7304 return &$complete_vmid_full();
7307 sub complete_vmid_stopped
{
7308 return &$complete_vmid_full(0);
7311 sub complete_vmid_running
{
7312 return &$complete_vmid_full(1);
7315 sub complete_storage
{
7317 my $cfg = PVE
::Storage
::config
();
7318 my $ids = $cfg->{ids
};
7321 foreach my $sid (keys %$ids) {
7322 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
7323 next if !$ids->{$sid}->{content
}->{images
};