1 package PVE
::QemuServer
;
22 use Storable
qw(dclone);
23 use PVE
::Exception
qw(raise raise_param_exc);
25 use PVE
::Tools
qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
26 use PVE
::JSONSchema
qw(get_standard_option);
27 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
32 use PVE
::RPCEnvironment
;
33 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr);
34 use PVE
::QemuServer
::Memory
;
35 use PVE
::QemuServer
::USB
qw(parse_usb_device);
36 use PVE
::QemuServer
::Cloudinit
;
38 use Time
::HiRes
qw(gettimeofday);
39 use File
::Copy
qw(copy);
42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
46 my $qemu_snap_storage = {rbd
=> 1, sheepdog
=> 1};
48 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
52 # Note about locking: we use flock on the config file protect
53 # against concurent actions.
54 # Aditionaly, we have a 'lock' setting in the config file. This
55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
56 # allowed when such lock is set. But you can ignore this kind of
57 # lock with the --skiplock flag.
59 cfs_register_file
('/qemu-server/',
63 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
64 description
=> "Some command save/restore state from this location.",
70 PVE
::JSONSchema
::register_standard_option
('pve-snapshot-name', {
71 description
=> "The name of the snapshot.",
72 type
=> 'string', format
=> 'pve-configid',
76 PVE
::JSONSchema
::register_standard_option
('pve-qm-image-format', {
78 enum
=> [qw(raw cow qcow qed qcow2 vmdk cloop)],
79 description
=> "The drive's backing file's data format.",
83 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
84 description
=> "Specifies the Qemu machine type.",
86 pattern
=> '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
91 #no warnings 'redefine';
94 my ($controller, $vmid, $option, $value) = @_;
96 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
97 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
101 my $nodename = PVE
::INotify
::nodename
();
103 mkdir "/etc/pve/nodes/$nodename";
104 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
107 my $var_run_tmpdir = "/var/run/qemu-server";
108 mkdir $var_run_tmpdir;
110 my $lock_dir = "/var/lock/qemu-server";
113 my $pcisysfs = "/sys/bus/pci";
115 my $cpu_vendor_list = {
117 486 => 'GenuineIntel',
118 pentium
=> 'GenuineIntel',
119 pentium2
=> 'GenuineIntel',
120 pentium3
=> 'GenuineIntel',
121 coreduo
=> 'GenuineIntel',
122 core2duo
=> 'GenuineIntel',
123 Conroe
=> 'GenuineIntel',
124 Penryn
=> 'GenuineIntel',
125 Nehalem
=> 'GenuineIntel',
126 'Nehalem-IBRS' => 'GenuineIntel',
127 Westmere
=> 'GenuineIntel',
128 'Westmere-IBRS' => 'GenuineIntel',
129 SandyBridge
=> 'GenuineIntel',
130 'SandyBridge-IBRS' => 'GenuineIntel',
131 IvyBridge
=> 'GenuineIntel',
132 'IvyBridge-IBRS' => 'GenuineIntel',
133 Haswell
=> 'GenuineIntel',
134 'Haswell-IBRS' => 'GenuineIntel',
135 'Haswell-noTSX' => 'GenuineIntel',
136 'Haswell-noTSX-IBRS' => 'GenuineIntel',
137 Broadwell
=> 'GenuineIntel',
138 'Broadwell-IBRS' => 'GenuineIntel',
139 'Broadwell-noTSX' => 'GenuineIntel',
140 'Broadwell-noTSX-IBRS' => 'GenuineIntel',
141 'Skylake-Client' => 'GenuineIntel',
142 'Skylake-Client-IBRS' => 'GenuineIntel',
143 'Skylake-Server' => 'GenuineIntel',
144 'Skylake-Server-IBRS' => 'GenuineIntel',
147 athlon
=> 'AuthenticAMD',
148 phenom
=> 'AuthenticAMD',
149 Opteron_G1
=> 'AuthenticAMD',
150 Opteron_G2
=> 'AuthenticAMD',
151 Opteron_G3
=> 'AuthenticAMD',
152 Opteron_G4
=> 'AuthenticAMD',
153 Opteron_G5
=> 'AuthenticAMD',
154 EPYC
=> 'AuthenticAMD',
155 'EPYC-IBPB' => 'AuthenticAMD',
157 # generic types, use vendor from host node
166 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
170 description
=> "Emulated CPU type.",
172 enum
=> [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
177 description
=> "Do not identify as a KVM virtual machine.",
183 description
=> "List of additional CPU flags separated by ';'."
184 . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
185 . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
186 format_description
=> '+FLAG[;-FLAG...]',
188 pattern
=> qr/$cpu_flag(;$cpu_flag)*/,
197 enum
=> [qw(i6300esb ib700)],
198 description
=> "Watchdog type to emulate.",
199 default => 'i6300esb',
204 enum
=> [qw(reset shutdown poweroff pause debug none)],
205 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
209 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
213 description
=> "Enable/disable Qemu GuestAgent.",
218 fstrim_cloned_disks
=> {
219 description
=> "Run fstrim after cloning/moving a disk.",
230 description
=> "Specifies whether a VM will be started during system bootup.",
236 description
=> "Automatic restart after crash (currently ignored).",
241 type
=> 'string', format
=> 'pve-hotplug-features',
242 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
243 default => 'network,disk,usb',
248 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
254 description
=> "Lock/unlock the VM.",
255 enum
=> [qw(migrate backup snapshot rollback)],
260 description
=> "Limit of CPU usage.",
261 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
269 description
=> "CPU weight for a VM.",
270 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
278 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
285 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
291 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
299 description
=> "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
300 "It should not be necessary to set it.",
301 enum
=> PVE
::Tools
::kvmkeymaplist
(),
306 type
=> 'string', format
=> 'dns-name',
307 description
=> "Set a name for the VM. Only used on the configuration web interface.",
312 description
=> "SCSI controller model",
313 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
319 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
324 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
325 description
=> "Specify guest operating system.",
326 verbose_description
=> <<EODESC,
327 Specify guest operating system. This is used to enable special
328 optimization/features for specific operating systems:
331 other;; unspecified OS
332 wxp;; Microsoft Windows XP
333 w2k;; Microsoft Windows 2000
334 w2k3;; Microsoft Windows 2003
335 w2k8;; Microsoft Windows 2008
336 wvista;; Microsoft Windows Vista
337 win7;; Microsoft Windows 7
338 win8;; Microsoft Windows 8/2012/2012r2
339 win10;; Microsoft Windows 10/2016
340 l24;; Linux 2.4 Kernel
341 l26;; Linux 2.6/3.X Kernel
342 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
348 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
349 pattern
=> '[acdn]{1,4}',
354 type
=> 'string', format
=> 'pve-qm-bootdisk',
355 description
=> "Enable booting from specified disk.",
356 pattern
=> '(ide|sata|scsi|virtio)\d+',
361 description
=> "The number of CPUs. Please use option -sockets instead.",
368 description
=> "The number of CPU sockets.",
375 description
=> "The number of cores per socket.",
382 description
=> "Enable/disable NUMA.",
388 description
=> "Enable/disable hugepages memory.",
389 enum
=> [qw(any 2 1024)],
394 description
=> "Number of hotplugged vcpus.",
401 description
=> "Enable/disable ACPI.",
406 description
=> "Enable/disable Qemu GuestAgent and its properties.",
408 format
=> $agent_fmt,
413 description
=> "Enable/disable KVM hardware virtualization.",
419 description
=> "Enable/disable time drift fix.",
425 description
=> "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
430 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
435 description
=> "Select the VGA type.",
436 verbose_description
=> "Select the VGA type. If you want to use high resolution" .
437 " modes (>= 1280x1024x16) then you should use the options " .
438 "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
439 "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
440 "display sever. For win* OS you can select how many independent " .
441 "displays you want, Linux guests can add displays them self. " .
442 "You can also run without any graphic card, using a serial device" .
444 enum
=> [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
448 type
=> 'string', format
=> 'pve-qm-watchdog',
449 description
=> "Create a virtual hardware watchdog device.",
450 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled" .
451 " (by a guest action), the watchdog must be periodically polled " .
452 "by an agent inside the guest or else the watchdog will reset " .
453 "the guest (or execute the respective action specified)",
458 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
459 description
=> "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
460 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
463 startup
=> get_standard_option
('pve-startup-order'),
467 description
=> "Enable/disable Template.",
473 description
=> "Arbitrary arguments passed to kvm.",
474 verbose_description
=> <<EODESCR,
475 Arbitrary arguments passed to kvm, for example:
477 args: -no-reboot -no-hpet
479 NOTE: this option is for experts only.
486 description
=> "Enable/disable the USB tablet device.",
487 verbose_description
=> "Enable/disable the USB tablet device. This device is " .
488 "usually needed to allow absolute mouse positioning with VNC. " .
489 "Else the mouse runs out of sync with normal VNC clients. " .
490 "If you're running lots of console-only guests on one host, " .
491 "you may consider disabling this to save some context switches. " .
492 "This is turned off by default if you use spice (-vga=qxl).",
497 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
501 migrate_downtime
=> {
504 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
510 type
=> 'string', format
=> 'pve-qm-ide',
511 typetext
=> '<volume>',
512 description
=> "This is an alias for option -ide2",
516 description
=> "Emulated CPU type.",
520 parent
=> get_standard_option
('pve-snapshot-name', {
522 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
526 description
=> "Timestamp for snapshots.",
532 type
=> 'string', format
=> 'pve-volume-id',
533 description
=> "Reference to a volume which stores the VM state. This is used internally for snapshots.",
535 vmstatestorage
=> get_standard_option
('pve-storage-id', {
536 description
=> "Default storage for VM state volumes/files.",
539 runningmachine
=> get_standard_option
('pve-qemu-machine', {
540 description
=> "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
542 machine
=> get_standard_option
('pve-qemu-machine'),
544 description
=> "Specify SMBIOS type 1 fields.",
545 type
=> 'string', format
=> 'pve-qm-smbios1',
552 description
=> "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
558 enum
=> [ qw(seabios ovmf) ],
559 description
=> "Select BIOS implementation.",
560 default => 'seabios',
564 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
565 format_description
=> 'UUID',
566 description
=> "Set VM Generation ID UUID. Use special value 1 to autogenerate one (API only). Use special value 0 to disable explicitly.",
571 my $confdesc_cloudinit = {
575 description
=> 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
576 enum
=> ['configdrive2', 'nocloud'],
581 description
=> "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
586 description
=> 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
591 description
=> "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
595 type
=> 'string', format
=> 'address-list',
596 description
=> "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
601 format
=> 'urlencoded',
602 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
606 # what about other qemu settings ?
608 #machine => 'string',
621 ##soundhw => 'string',
623 while (my ($k, $v) = each %$confdesc) {
624 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
627 my $MAX_IDE_DISKS = 4;
628 my $MAX_SCSI_DISKS = 14;
629 my $MAX_VIRTIO_DISKS = 16;
630 my $MAX_SATA_DISKS = 6;
631 my $MAX_USB_DEVICES = 5;
633 my $MAX_UNUSED_DISKS = 8;
634 my $MAX_HOSTPCI_DEVICES = 4;
635 my $MAX_SERIAL_PORTS = 4;
636 my $MAX_PARALLEL_PORTS = 3;
642 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
643 description
=> "CPUs accessing this NUMA node.",
644 format_description
=> "id[-id];...",
648 description
=> "Amount of memory this NUMA node provides.",
653 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
654 description
=> "Host NUMA nodes to use.",
655 format_description
=> "id[-id];...",
660 enum
=> [qw(preferred bind interleave)],
661 description
=> "NUMA allocation policy.",
665 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
668 type
=> 'string', format
=> $numa_fmt,
669 description
=> "NUMA topology.",
671 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
673 for (my $i = 0; $i < $MAX_NUMA; $i++) {
674 $confdesc->{"numa$i"} = $numadesc;
677 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
678 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
679 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
680 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
682 my $net_fmt_bridge_descr = <<__EOD__;
683 Bridge to attach the network device to. The Proxmox VE standard bridge
686 If you do not specify a bridge, we create a kvm user (NATed) network
687 device, which provides DHCP and DNS services. The following addresses
694 The DHCP server assign addresses to the guest starting from 10.0.2.15.
700 pattern
=> qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
701 description
=> "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
702 format_description
=> "XX:XX:XX:XX:XX:XX",
707 description
=> "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
708 enum
=> $nic_model_list,
711 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
714 description
=> $net_fmt_bridge_descr,
715 format_description
=> 'bridge',
720 minimum
=> 0, maximum
=> 16,
721 description
=> 'Number of packet queues to be used on the device.',
727 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
732 minimum
=> 1, maximum
=> 4094,
733 description
=> 'VLAN tag to apply to packets on this interface.',
738 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
739 description
=> 'VLAN trunks to pass through this interface.',
740 format_description
=> 'vlanid[;vlanid...]',
745 description
=> 'Whether this interface should be protected by the firewall.',
750 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
757 type
=> 'string', format
=> $net_fmt,
758 description
=> "Specify network devices.",
761 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
766 format
=> 'pve-ipv4-config',
767 format_description
=> 'IPv4Format/CIDR',
768 description
=> 'IPv4 address in CIDR format.',
775 format_description
=> 'GatewayIPv4',
776 description
=> 'Default gateway for IPv4 traffic.',
782 format
=> 'pve-ipv6-config',
783 format_description
=> 'IPv6Format/CIDR',
784 description
=> 'IPv6 address in CIDR format.',
791 format_description
=> 'GatewayIPv6',
792 description
=> 'Default gateway for IPv6 traffic.',
797 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
800 type
=> 'string', format
=> 'pve-qm-ipconfig',
801 description
=> <<'EODESCR',
802 cloud-init: Specify IP addresses and gateways for the corresponding interface.
804 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
806 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
807 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
809 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
812 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
814 for (my $i = 0; $i < $MAX_NETS; $i++) {
815 $confdesc->{"net$i"} = $netdesc;
816 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
819 foreach my $key (keys %$confdesc_cloudinit) {
820 $confdesc->{$key} = $confdesc_cloudinit->{$key};
823 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
824 sub verify_volume_id_or_qm_path
{
825 my ($volid, $noerr) = @_;
827 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m
|^/|) {
831 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
832 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
834 return undef if $noerr;
842 my %drivedesc_base = (
843 volume
=> { alias
=> 'file' },
846 format
=> 'pve-volume-id-or-qm-path',
848 format_description
=> 'volume',
849 description
=> "The drive's backing volume.",
853 enum
=> [qw(cdrom disk)],
854 description
=> "The drive's media type.",
860 description
=> "Force the drive's physical geometry to have a specific cylinder count.",
865 description
=> "Force the drive's physical geometry to have a specific head count.",
870 description
=> "Force the drive's physical geometry to have a specific sector count.",
875 enum
=> [qw(none lba auto)],
876 description
=> "Force disk geometry bios translation mode.",
881 description
=> "Controls qemu's snapshot mode feature."
882 . " If activated, changes made to the disk are temporary and will"
883 . " be discarded when the VM is shutdown.",
888 enum
=> [qw(none writethrough writeback unsafe directsync)],
889 description
=> "The drive's cache mode",
892 format
=> get_standard_option
('pve-qm-image-format'),
895 format
=> 'disk-size',
896 format_description
=> 'DiskSize',
897 description
=> "Disk size. This is purely informational and has no effect.",
902 description
=> "Whether the drive should be included when making backups.",
907 description
=> 'Whether the drive should considered for replication jobs.',
913 enum
=> [qw(ignore report stop)],
914 description
=> 'Read error action.',
919 enum
=> [qw(enospc ignore report stop)],
920 description
=> 'Write error action.',
925 enum
=> [qw(native threads)],
926 description
=> 'AIO type to use.',
931 enum
=> [qw(ignore on)],
932 description
=> 'Controls whether to pass discard/trim requests to the underlying storage.',
937 description
=> 'Controls whether to detect and try to optimize writes of zeroes.',
942 format
=> 'urlencoded',
943 format_description
=> 'serial',
944 maxLength
=> 20*3, # *3 since it's %xx url enoded
945 description
=> "The drive's reported serial number, url-encoded, up to 20 bytes long.",
950 description
=> 'Mark this locally-managed volume as available on all nodes',
951 verbose_description
=> "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
957 my %iothread_fmt = ( iothread
=> {
959 description
=> "Whether to use iothreads for this drive",
966 format
=> 'urlencoded',
967 format_description
=> 'model',
968 maxLength
=> 40*3, # *3 since it's %xx url enoded
969 description
=> "The drive's reported model name, url-encoded, up to 40 bytes long.",
977 description
=> "Number of queues.",
983 my %scsiblock_fmt = (
986 description
=> "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
992 my $add_throttle_desc = sub {
993 my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
996 format_description
=> $unit,
997 description
=> "Maximum $what in $longunit.",
1000 $d->{minimum
} = $minimum if defined($minimum);
1001 $drivedesc_base{$key} = $d;
1003 # throughput: (leaky bucket)
1004 $add_throttle_desc->('bps', 'integer', 'r/w speed', 'bps', 'bytes per second');
1005 $add_throttle_desc->('bps_rd', 'integer', 'read speed', 'bps', 'bytes per second');
1006 $add_throttle_desc->('bps_wr', 'integer', 'write speed', 'bps', 'bytes per second');
1007 $add_throttle_desc->('mbps', 'number', 'r/w speed', 'mbps', 'megabytes per second');
1008 $add_throttle_desc->('mbps_rd', 'number', 'read speed', 'mbps', 'megabytes per second');
1009 $add_throttle_desc->('mbps_wr', 'number', 'write speed', 'mbps', 'megabytes per second');
1010 $add_throttle_desc->('iops', 'integer', 'r/w I/O', 'iops', 'operations per second');
1011 $add_throttle_desc->('iops_rd', 'integer', 'read I/O', 'iops', 'operations per second');
1012 $add_throttle_desc->('iops_wr', 'integer', 'write I/O', 'iops', 'operations per second');
1014 # pools: (pool of IO before throttling starts taking effect)
1015 $add_throttle_desc->('mbps_max', 'number', 'unthrottled r/w pool', 'mbps', 'megabytes per second');
1016 $add_throttle_desc->('mbps_rd_max', 'number', 'unthrottled read pool', 'mbps', 'megabytes per second');
1017 $add_throttle_desc->('mbps_wr_max', 'number', 'unthrottled write pool', 'mbps', 'megabytes per second');
1018 $add_throttle_desc->('iops_max', 'integer', 'unthrottled r/w I/O pool', 'iops', 'operations per second');
1019 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool', 'iops', 'operations per second');
1020 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1023 $add_throttle_desc->('bps_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1024 $add_throttle_desc->('bps_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1025 $add_throttle_desc->('bps_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1026 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts', 'seconds', 'seconds', 1);
1027 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts', 'seconds', 'seconds', 1);
1028 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1031 $drivedesc_base{'bps_rd_length'} = { alias
=> 'bps_rd_max_length' };
1032 $drivedesc_base{'bps_wr_length'} = { alias
=> 'bps_wr_max_length' };
1033 $drivedesc_base{'iops_rd_length'} = { alias
=> 'iops_rd_max_length' };
1034 $drivedesc_base{'iops_wr_length'} = { alias
=> 'iops_wr_max_length' };
1040 PVE
::JSONSchema
::register_format
("pve-qm-ide", $ide_fmt);
1044 type
=> 'string', format
=> $ide_fmt,
1045 description
=> "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1047 PVE
::JSONSchema
::register_standard_option
("pve-qm-ide", $idedesc);
1057 type
=> 'string', format
=> $scsi_fmt,
1058 description
=> "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1060 PVE
::JSONSchema
::register_standard_option
("pve-qm-scsi", $scsidesc);
1067 type
=> 'string', format
=> $sata_fmt,
1068 description
=> "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1070 PVE
::JSONSchema
::register_standard_option
("pve-qm-sata", $satadesc);
1078 type
=> 'string', format
=> $virtio_fmt,
1079 description
=> "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1081 PVE
::JSONSchema
::register_standard_option
("pve-qm-virtio", $virtiodesc);
1083 my $alldrive_fmt = {
1092 volume
=> { alias
=> 'file' },
1095 format
=> 'pve-volume-id-or-qm-path',
1097 format_description
=> 'volume',
1098 description
=> "The drive's backing volume.",
1100 format
=> get_standard_option
('pve-qm-image-format'),
1103 format
=> 'disk-size',
1104 format_description
=> 'DiskSize',
1105 description
=> "Disk size. This is purely informational and has no effect.",
1110 my $efidisk_desc = {
1112 type
=> 'string', format
=> $efidisk_fmt,
1113 description
=> "Configure a Disk for storing EFI vars",
1116 PVE
::JSONSchema
::register_standard_option
("pve-qm-efidisk", $efidisk_desc);
1121 type
=> 'string', format
=> 'pve-qm-usb-device',
1122 format_description
=> 'HOSTUSBDEVICE|spice',
1123 description
=> <<EODESCR,
1124 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1126 'bus-port(.port)*' (decimal numbers) or
1127 'vendor_id:product_id' (hexadeciaml numbers) or
1130 You can use the 'lsusb -t' command to list existing usb devices.
1132 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1134 The value 'spice' can be used to add a usb redirection devices for spice.
1140 description
=> "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1147 type
=> 'string', format
=> $usb_fmt,
1148 description
=> "Configure an USB device (n is 0 to 4).",
1150 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1152 # NOTE: the match-groups of this regex are used in parse_hostpci
1153 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1158 pattern
=> qr/$PCIRE(;$PCIRE)*/,
1159 format_description
=> 'HOSTPCIID[;HOSTPCIID2...]',
1160 description
=> <<EODESCR,
1161 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1162 of PCI virtual functions of the host. HOSTPCIID syntax is:
1164 'bus:dev.func' (hexadecimal numbers)
1166 You can us the 'lspci' command to list existing PCI devices.
1171 description
=> "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1177 pattern
=> '[^,;]+',
1178 format_description
=> 'string',
1179 description
=> "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1184 description
=> "Choose the PCI-express bus (needs the 'q35' machine model).",
1190 description
=> "Enable vfio-vga device support.",
1195 PVE
::JSONSchema
::register_format
('pve-qm-hostpci', $hostpci_fmt);
1199 type
=> 'string', format
=> 'pve-qm-hostpci',
1200 description
=> "Map host PCI devices into guest.",
1201 verbose_description
=> <<EODESCR,
1202 Map host PCI devices into guest.
1204 NOTE: This option allows direct access to host hardware. So it is no longer
1205 possible to migrate such machines - use with special care.
1207 CAUTION: Experimental! User reported problems with this option.
1210 PVE
::JSONSchema
::register_standard_option
("pve-qm-hostpci", $hostpcidesc);
1215 pattern
=> '(/dev/.+|socket)',
1216 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1217 verbose_description
=> <<EODESCR,
1218 Create a serial device inside the VM (n is 0 to 3), and pass through a
1219 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1220 host side (use 'qm terminal' to open a terminal connection).
1222 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1224 CAUTION: Experimental! User reported problems with this option.
1231 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1232 description
=> "Map host parallel devices (n is 0 to 2).",
1233 verbose_description
=> <<EODESCR,
1234 Map host parallel devices (n is 0 to 2).
1236 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1238 CAUTION: Experimental! User reported problems with this option.
1242 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1243 $confdesc->{"parallel$i"} = $paralleldesc;
1246 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1247 $confdesc->{"serial$i"} = $serialdesc;
1250 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
1251 $confdesc->{"hostpci$i"} = $hostpcidesc;
1254 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) {
1255 $drivename_hash->{"ide$i"} = 1;
1256 $confdesc->{"ide$i"} = $idedesc;
1259 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) {
1260 $drivename_hash->{"sata$i"} = 1;
1261 $confdesc->{"sata$i"} = $satadesc;
1264 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) {
1265 $drivename_hash->{"scsi$i"} = 1;
1266 $confdesc->{"scsi$i"} = $scsidesc ;
1269 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) {
1270 $drivename_hash->{"virtio$i"} = 1;
1271 $confdesc->{"virtio$i"} = $virtiodesc;
1274 $drivename_hash->{efidisk0
} = 1;
1275 $confdesc->{efidisk0
} = $efidisk_desc;
1277 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1278 $confdesc->{"usb$i"} = $usbdesc;
1283 type
=> 'string', format
=> 'pve-volume-id',
1284 description
=> "Reference to unused volumes. This is used internally, and should not be modified manually.",
1287 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1288 $confdesc->{"unused$i"} = $unuseddesc;
1291 my $kvm_api_version = 0;
1295 return $kvm_api_version if $kvm_api_version;
1297 my $fh = IO
::File-
>new("</dev/kvm") ||
1300 if (my $v = $fh->ioctl(KVM_GET_API_VERSION
(), 0)) {
1301 $kvm_api_version = $v;
1306 return $kvm_api_version;
1309 my $kvm_user_version;
1311 sub kvm_user_version
{
1313 return $kvm_user_version if $kvm_user_version;
1315 $kvm_user_version = 'unknown';
1319 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1320 $kvm_user_version = $2;
1324 eval { run_command
("kvm -version", outfunc
=> $code); };
1327 return $kvm_user_version;
1331 my $kernel_has_vhost_net = -c
'/dev/vhost-net';
1333 sub valid_drive_names
{
1334 # order is important - used to autoselect boot disk
1335 return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1336 (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1337 (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1338 (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1342 sub is_valid_drivename
{
1345 return defined($drivename_hash->{$dev});
1350 return defined($confdesc->{$key});
1354 return $nic_model_list;
1357 sub os_list_description
{
1361 wxp
=> 'Windows XP',
1362 w2k
=> 'Windows 2000',
1363 w2k3
=>, 'Windows 2003',
1364 w2k8
=> 'Windows 2008',
1365 wvista
=> 'Windows Vista',
1366 win7
=> 'Windows 7',
1367 win8
=> 'Windows 8/2012',
1368 win10
=> 'Windows 10/2016',
1376 sub get_cdrom_path
{
1378 return $cdrom_path if $cdrom_path;
1380 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1381 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1382 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1386 my ($storecfg, $vmid, $cdrom) = @_;
1388 if ($cdrom eq 'cdrom') {
1389 return get_cdrom_path
();
1390 } elsif ($cdrom eq 'none') {
1392 } elsif ($cdrom =~ m
|^/|) {
1395 return PVE
::Storage
::path
($storecfg, $cdrom);
1399 # try to convert old style file names to volume IDs
1400 sub filename_to_volume_id
{
1401 my ($vmid, $file, $media) = @_;
1403 if (!($file eq 'none' || $file eq 'cdrom' ||
1404 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1406 return undef if $file =~ m
|/|;
1408 if ($media && $media eq 'cdrom') {
1409 $file = "local:iso/$file";
1411 $file = "local:$vmid/$file";
1418 sub verify_media_type
{
1419 my ($opt, $vtype, $media) = @_;
1424 if ($media eq 'disk') {
1426 } elsif ($media eq 'cdrom') {
1429 die "internal error";
1432 return if ($vtype eq $etype);
1434 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1437 sub cleanup_drive_path
{
1438 my ($opt, $storecfg, $drive) = @_;
1440 # try to convert filesystem paths to volume IDs
1442 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1443 ($drive->{file
} !~ m
|^/dev/.+|) &&
1444 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1445 ($drive->{file
} !~ m/^\d+$/)) {
1446 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1447 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1448 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1449 verify_media_type
($opt, $vtype, $drive->{media
});
1450 $drive->{file
} = $volid;
1453 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1456 sub parse_hotplug_features
{
1461 return $res if $data eq '0';
1463 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1465 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1466 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1469 die "invalid hotplug feature '$feature'\n";
1475 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1476 sub pve_verify_hotplug_features
{
1477 my ($value, $noerr) = @_;
1479 return $value if parse_hotplug_features
($value);
1481 return undef if $noerr;
1483 die "unable to parse hotplug option\n";
1486 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1487 # [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1488 # [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1489 # [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1490 # [,iothread=on][,serial=serial][,model=model]
1493 my ($key, $data) = @_;
1495 my ($interface, $index);
1497 if ($key =~ m/^([^\d]+)(\d+)$/) {
1504 my $desc = $key =~ /^unused\d+$/ ?
$alldrive_fmt
1505 : $confdesc->{$key}->{format
};
1507 warn "invalid drive key: $key\n";
1510 my $res = eval { PVE
::JSONSchema
::parse_property_string
($desc, $data) };
1511 return undef if !$res;
1512 $res->{interface
} = $interface;
1513 $res->{index} = $index;
1516 foreach my $opt (qw(bps bps_rd bps_wr)) {
1517 if (my $bps = defined(delete $res->{$opt})) {
1518 if (defined($res->{"m$opt"})) {
1519 warn "both $opt and m$opt specified\n";
1523 $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1527 # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1528 for my $requirement (
1529 [mbps_max
=> 'mbps'],
1530 [mbps_rd_max
=> 'mbps_rd'],
1531 [mbps_wr_max
=> 'mbps_wr'],
1532 [miops_max
=> 'miops'],
1533 [miops_rd_max
=> 'miops_rd'],
1534 [miops_wr_max
=> 'miops_wr'],
1535 [bps_max_length
=> 'mbps_max'],
1536 [bps_rd_max_length
=> 'mbps_rd_max'],
1537 [bps_wr_max_length
=> 'mbps_wr_max'],
1538 [iops_max_length
=> 'iops_max'],
1539 [iops_rd_max_length
=> 'iops_rd_max'],
1540 [iops_wr_max_length
=> 'iops_wr_max']) {
1541 my ($option, $requires) = @$requirement;
1542 if ($res->{$option} && !$res->{$requires}) {
1543 warn "$option requires $requires\n";
1548 return undef if $error;
1550 return undef if $res->{mbps_rd
} && $res->{mbps
};
1551 return undef if $res->{mbps_wr
} && $res->{mbps
};
1552 return undef if $res->{iops_rd
} && $res->{iops
};
1553 return undef if $res->{iops_wr
} && $res->{iops
};
1555 if ($res->{media
} && ($res->{media
} eq 'cdrom')) {
1556 return undef if $res->{snapshot
} || $res->{trans
} || $res->{format
};
1557 return undef if $res->{heads
} || $res->{secs
} || $res->{cyls
};
1558 return undef if $res->{interface
} eq 'virtio';
1561 if (my $size = $res->{size
}) {
1562 return undef if !defined($res->{size
} = PVE
::JSONSchema
::parse_size
($size));
1569 my ($vmid, $drive) = @_;
1570 my $data = { %$drive };
1571 delete $data->{$_} for qw(index interface);
1572 return PVE
::JSONSchema
::print_property_string
($data, $alldrive_fmt);
1576 my($fh, $noerr) = @_;
1579 my $SG_GET_VERSION_NUM = 0x2282;
1581 my $versionbuf = "\x00" x
8;
1582 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1584 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1587 my $version = unpack("I", $versionbuf);
1588 if ($version < 30000) {
1589 die "scsi generic interface too old\n" if !$noerr;
1593 my $buf = "\x00" x
36;
1594 my $sensebuf = "\x00" x
8;
1595 my $cmd = pack("C x3 C x1", 0x12, 36);
1597 # see /usr/include/scsi/sg.h
1598 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1600 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1601 length($sensebuf), 0, length($buf), $buf,
1602 $cmd, $sensebuf, 6000);
1604 $ret = ioctl($fh, $SG_IO, $packet);
1606 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1610 my @res = unpack($sg_io_hdr_t, $packet);
1611 if ($res[17] || $res[18]) {
1612 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1617 (my $byte0, my $byte1, $res->{vendor
},
1618 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1620 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1621 $res->{type
} = $byte0 & 31;
1629 my $fh = IO
::File-
>new("+<$path") || return undef;
1630 my $res = scsi_inquiry
($fh, 1);
1636 sub machine_type_is_q35
{
1639 return $conf->{machine
} && ($conf->{machine
} =~ m/q35/) ?
1 : 0;
1642 sub print_tabletdevice_full
{
1645 my $q35 = machine_type_is_q35
($conf);
1647 # we use uhci for old VMs because tablet driver was buggy in older qemu
1648 my $usbbus = $q35 ?
"ehci" : "uhci";
1650 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1653 sub print_drivedevice_full
{
1654 my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1659 if ($drive->{interface
} eq 'virtio') {
1660 my $pciaddr = print_pci_addr
("$drive->{interface}$drive->{index}", $bridges);
1661 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1662 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread
};
1663 } elsif ($drive->{interface
} eq 'scsi') {
1665 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1666 my $unit = $drive->{index} % $maxdev;
1667 my $devicetype = 'hd';
1669 if (drive_is_cdrom
($drive)) {
1672 if ($drive->{file
} =~ m
|^/|) {
1673 $path = $drive->{file
};
1674 if (my $info = path_is_scsi
($path)) {
1675 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1676 $devicetype = 'block';
1677 } elsif ($info->{type
} == 1) { # tape
1678 $devicetype = 'generic';
1682 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1685 if($path =~ m/^iscsi\:\/\
//){
1686 $devicetype = 'generic';
1690 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1691 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1693 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1696 } elsif ($drive->{interface
} eq 'ide'){
1698 my $controller = int($drive->{index} / $maxdev);
1699 my $unit = $drive->{index} % $maxdev;
1700 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1702 $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1703 if ($devicetype eq 'hd' && (my $model = $drive->{model
})) {
1704 $model = URI
::Escape
::uri_unescape
($model);
1705 $device .= ",model=$model";
1707 } elsif ($drive->{interface
} eq 'sata'){
1708 my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1709 my $unit = $drive->{index} % $MAX_SATA_DISKS;
1710 $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1711 } elsif ($drive->{interface
} eq 'usb') {
1713 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1715 die "unsupported interface type";
1718 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1720 if (my $serial = $drive->{serial
}) {
1721 $serial = URI
::Escape
::uri_unescape
($serial);
1722 $device .= ",serial=$serial";
1729 sub get_initiator_name
{
1732 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return undef;
1733 while (defined(my $line = <$fh>)) {
1734 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1743 sub print_drive_full
{
1744 my ($storecfg, $vmid, $drive) = @_;
1747 my $volid = $drive->{file
};
1750 if (drive_is_cdrom
($drive)) {
1751 $path = get_iso_path
($storecfg, $vmid, $volid);
1753 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1755 $path = PVE
::Storage
::path
($storecfg, $volid);
1756 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1757 $format = qemu_img_format
($scfg, $volname);
1765 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1766 foreach my $o (@qemu_drive_options) {
1767 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1770 # snapshot only accepts on|off
1771 if (defined($drive->{snapshot
})) {
1772 my $v = $drive->{snapshot
} ?
'on' : 'off';
1773 $opts .= ",snapshot=$v";
1776 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1777 my ($dir, $qmpname) = @$type;
1778 if (my $v = $drive->{"mbps$dir"}) {
1779 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1781 if (my $v = $drive->{"mbps${dir}_max"}) {
1782 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1784 if (my $v = $drive->{"bps${dir}_max_length"}) {
1785 $opts .= ",throttling.bps$qmpname-max-length=$v";
1787 if (my $v = $drive->{"iops${dir}"}) {
1788 $opts .= ",throttling.iops$qmpname=$v";
1790 if (my $v = $drive->{"iops${dir}_max"}) {
1791 $opts .= ",throttling.iops$qmpname-max=$v";
1793 if (my $v = $drive->{"iops${dir}_max_length"}) {
1794 $opts .= ",throttling.iops$qmpname-max-length=$v";
1798 $opts .= ",format=$format" if $format && !$drive->{format
};
1800 my $cache_direct = 0;
1802 if (my $cache = $drive->{cache
}) {
1803 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1804 } elsif (!drive_is_cdrom
($drive)) {
1805 $opts .= ",cache=none";
1809 # aio native works only with O_DIRECT
1810 if (!$drive->{aio
}) {
1812 $opts .= ",aio=native";
1814 $opts .= ",aio=threads";
1818 if (!drive_is_cdrom
($drive)) {
1820 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1821 $detectzeroes = 'off';
1822 } elsif ($drive->{discard
}) {
1823 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1825 # This used to be our default with discard not being specified:
1826 $detectzeroes = 'on';
1828 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1831 my $pathinfo = $path ?
"file=$path," : '';
1833 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1836 sub print_netdevice_full
{
1837 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1839 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
1841 my $device = $net->{model
};
1842 if ($net->{model
} eq 'virtio') {
1843 $device = 'virtio-net-pci';
1846 my $pciaddr = print_pci_addr
("$netid", $bridges);
1847 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1848 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1849 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1850 my $vectors = $net->{queues
} * 2 + 2;
1851 $tmpstr .= ",vectors=$vectors,mq=on";
1853 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1855 if ($use_old_bios_files) {
1857 if ($device eq 'virtio-net-pci') {
1858 $romfile = 'pxe-virtio.rom';
1859 } elsif ($device eq 'e1000') {
1860 $romfile = 'pxe-e1000.rom';
1861 } elsif ($device eq 'ne2k') {
1862 $romfile = 'pxe-ne2k_pci.rom';
1863 } elsif ($device eq 'pcnet') {
1864 $romfile = 'pxe-pcnet.rom';
1865 } elsif ($device eq 'rtl8139') {
1866 $romfile = 'pxe-rtl8139.rom';
1868 $tmpstr .= ",romfile=$romfile" if $romfile;
1874 sub print_netdev_full
{
1875 my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1878 if ($netid =~ m/^net(\d+)$/) {
1882 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1884 my $ifname = "tap${vmid}i$i";
1886 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1887 die "interface name '$ifname' is too long (max 15 character)\n"
1888 if length($ifname) >= 16;
1890 my $vhostparam = '';
1891 $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model
} eq 'virtio';
1893 my $vmname = $conf->{name
} || "vm$vmid";
1896 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1898 if ($net->{bridge
}) {
1899 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1901 $netdev = "type=user,id=$netid,hostname=$vmname";
1904 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1910 sub print_cpu_device
{
1911 my ($conf, $id) = @_;
1913 my $kvm = $conf->{kvm
} // 1;
1914 my $cpu = $kvm ?
"kvm64" : "qemu64";
1915 if (my $cputype = $conf->{cpu
}) {
1916 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
1917 or die "Cannot parse cpu description: $cputype\n";
1918 $cpu = $cpuconf->{cputype
};
1921 my $cores = $conf->{cores
} || 1;
1923 my $current_core = ($id - 1) % $cores;
1924 my $current_socket = int(($id - 1 - $current_core)/$cores);
1926 return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1929 sub drive_is_cloudinit
{
1931 return $drive->{file
} =~ m
@[:/]vm-\d
+-cloudinit
(?
:\
.$QEMU_FORMAT_RE)?
$@;
1934 sub drive_is_cdrom
{
1935 my ($drive, $exclude_cloudinit) = @_;
1937 return 0 if $exclude_cloudinit && drive_is_cloudinit
($drive);
1939 return $drive && $drive->{media
} && ($drive->{media
} eq 'cdrom');
1943 sub parse_number_sets
{
1946 foreach my $part (split(/;/, $set)) {
1947 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1948 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1949 push @$res, [ $1, $2 ];
1951 die "invalid range: $part\n";
1960 my $res = PVE
::JSONSchema
::parse_property_string
($numa_fmt, $data);
1961 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1962 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1969 return undef if !$value;
1971 my $res = PVE
::JSONSchema
::parse_property_string
($hostpci_fmt, $value);
1973 my @idlist = split(/;/, $res->{host
});
1974 delete $res->{host
};
1975 foreach my $id (@idlist) {
1976 if ($id =~ /^$PCIRE$/) {
1978 push @{$res->{pciid
}}, { id
=> $1, function
=> $2 };
1980 my $pcidevices = lspci
($1);
1981 $res->{pciid
} = $pcidevices->{$1};
1984 # should have been caught by parse_property_string already
1985 die "failed to parse PCI id: $id\n";
1991 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1995 my $res = eval { PVE
::JSONSchema
::parse_property_string
($net_fmt, $data) };
2000 if (!defined($res->{macaddr
})) {
2001 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
2002 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
2007 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2008 sub parse_ipconfig
{
2011 my $res = eval { PVE
::JSONSchema
::parse_property_string
($ipconfig_fmt, $data) };
2017 if ($res->{gw
} && !$res->{ip
}) {
2018 warn 'gateway specified without specifying an IP address';
2021 if ($res->{gw6
} && !$res->{ip6
}) {
2022 warn 'IPv6 gateway specified without specifying an IPv6 address';
2025 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
2026 warn 'gateway specified together with DHCP';
2029 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
2031 warn "IPv6 gateway specified together with $res->{ip6} address";
2035 if (!$res->{ip
} && !$res->{ip6
}) {
2036 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
2045 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
2048 sub add_random_macs
{
2049 my ($settings) = @_;
2051 foreach my $opt (keys %$settings) {
2052 next if $opt !~ m/^net(\d+)$/;
2053 my $net = parse_net
($settings->{$opt});
2055 $settings->{$opt} = print_net
($net);
2059 sub vm_is_volid_owner
{
2060 my ($storecfg, $vmid, $volid) = @_;
2062 if ($volid !~ m
|^/|) {
2064 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
2065 if ($owner && ($owner == $vmid)) {
2073 sub split_flagged_list
{
2074 my $text = shift || '';
2075 $text =~ s/[,;]/ /g;
2077 return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2080 sub join_flagged_list
{
2081 my ($how, $lst) = @_;
2082 join $how, map { $lst->{$_} . $_ } keys %$lst;
2085 sub vmconfig_delete_pending_option
{
2086 my ($conf, $key, $force) = @_;
2088 delete $conf->{pending
}->{$key};
2089 my $pending_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
2090 $pending_delete_hash->{$key} = $force ?
'!' : '';
2091 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
2094 sub vmconfig_undelete_pending_option
{
2095 my ($conf, $key) = @_;
2097 my $pending_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
2098 delete $pending_delete_hash->{$key};
2100 if (%$pending_delete_hash) {
2101 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
2103 delete $conf->{pending
}->{delete};
2107 sub vmconfig_register_unused_drive
{
2108 my ($storecfg, $vmid, $conf, $drive) = @_;
2110 if (drive_is_cloudinit
($drive)) {
2111 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
2113 } elsif (!drive_is_cdrom
($drive)) {
2114 my $volid = $drive->{file
};
2115 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
2116 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2121 sub vmconfig_cleanup_pending
{
2124 # remove pending changes when nothing changed
2126 foreach my $opt (keys %{$conf->{pending
}}) {
2127 if (defined($conf->{$opt}) && ($conf->{pending
}->{$opt} eq $conf->{$opt})) {
2129 delete $conf->{pending
}->{$opt};
2133 my $current_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
2134 my $pending_delete_hash = {};
2135 while (my ($opt, $force) = each %$current_delete_hash) {
2136 if (defined($conf->{$opt})) {
2137 $pending_delete_hash->{$opt} = $force;
2143 if (%$pending_delete_hash) {
2144 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
2146 delete $conf->{pending
}->{delete};
2152 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2156 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2157 format_description
=> 'UUID',
2158 description
=> "Set SMBIOS1 UUID.",
2164 format_description
=> 'string',
2165 description
=> "Set SMBIOS1 version.",
2171 format_description
=> 'string',
2172 description
=> "Set SMBIOS1 serial number.",
2178 format_description
=> 'string',
2179 description
=> "Set SMBIOS1 manufacturer.",
2185 format_description
=> 'string',
2186 description
=> "Set SMBIOS1 product ID.",
2192 format_description
=> 'string',
2193 description
=> "Set SMBIOS1 SKU string.",
2199 format_description
=> 'string',
2200 description
=> "Set SMBIOS1 family string.",
2208 my $res = eval { PVE
::JSONSchema
::parse_property_string
($smbios1_fmt, $data) };
2215 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2218 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2220 PVE
::JSONSchema
::register_format
('pve-qm-bootdisk', \
&verify_bootdisk
);
2221 sub verify_bootdisk
{
2222 my ($value, $noerr) = @_;
2224 return $value if is_valid_drivename
($value);
2226 return undef if $noerr;
2228 die "invalid boot disk '$value'\n";
2231 sub parse_watchdog
{
2234 return undef if !$value;
2236 my $res = eval { PVE
::JSONSchema
::parse_property_string
($watchdog_fmt, $value) };
2241 sub parse_guest_agent
{
2244 return {} if !defined($value->{agent
});
2246 my $res = eval { PVE
::JSONSchema
::parse_property_string
($agent_fmt, $value->{agent
}) };
2249 # if the agent is disabled ignore the other potentially set properties
2250 return {} if !$res->{enabled
};
2254 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2255 sub verify_usb_device
{
2256 my ($value, $noerr) = @_;
2258 return $value if parse_usb_device
($value);
2260 return undef if $noerr;
2262 die "unable to parse usb device\n";
2265 # add JSON properties for create and set function
2266 sub json_config_properties
{
2269 foreach my $opt (keys %$confdesc) {
2270 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2271 $prop->{$opt} = $confdesc->{$opt};
2277 # return copy of $confdesc_cloudinit to generate documentation
2278 sub cloudinit_config_properties
{
2280 return dclone
($confdesc_cloudinit);
2284 my ($key, $value) = @_;
2286 die "unknown setting '$key'\n" if !$confdesc->{$key};
2288 my $type = $confdesc->{$key}->{type
};
2290 if (!defined($value)) {
2291 die "got undefined value\n";
2294 if ($value =~ m/[\n\r]/) {
2295 die "property contains a line feed\n";
2298 if ($type eq 'boolean') {
2299 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2300 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2301 die "type check ('boolean') failed - got '$value'\n";
2302 } elsif ($type eq 'integer') {
2303 return int($1) if $value =~ m/^(\d+)$/;
2304 die "type check ('integer') failed - got '$value'\n";
2305 } elsif ($type eq 'number') {
2306 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2307 die "type check ('number') failed - got '$value'\n";
2308 } elsif ($type eq 'string') {
2309 if (my $fmt = $confdesc->{$key}->{format
}) {
2310 PVE
::JSONSchema
::check_format
($fmt, $value);
2313 $value =~ s/^\"(.*)\"$/$1/;
2316 die "internal error"
2320 sub check_iommu_support
{
2321 #fixme : need to check IOMMU support
2322 #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2332 my $conf = PVE
::QemuConfig-
>config_file($vmid);
2333 utime undef, undef, $conf;
2337 my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2339 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
2341 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2343 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2345 if ($conf->{template
}) {
2346 # check if any base image is still used by a linked clone
2347 foreach_drive
($conf, sub {
2348 my ($ds, $drive) = @_;
2350 return if drive_is_cdrom
($drive);
2352 my $volid = $drive->{file
};
2354 return if !$volid || $volid =~ m
|^/|;
2356 die "base volume '$volid' is still in use by linked cloned\n"
2357 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2362 # only remove disks owned by this VM
2363 foreach_drive
($conf, sub {
2364 my ($ds, $drive) = @_;
2366 return if drive_is_cdrom
($drive, 1);
2368 my $volid = $drive->{file
};
2370 return if !$volid || $volid =~ m
|^/|;
2372 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2373 return if !$path || !$owner || ($owner != $vmid);
2376 PVE
::Storage
::vdisk_free
($storecfg, $volid);
2378 warn "Could not remove disk '$volid', check manually: $@" if $@;
2382 if ($keep_empty_config) {
2383 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
2388 # also remove unused disk
2390 my $dl = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
2393 PVE
::Storage
::foreach_volid
($dl, sub {
2394 my ($volid, $sid, $volname, $d) = @_;
2395 PVE
::Storage
::vdisk_free
($storecfg, $volid);
2404 sub parse_vm_config
{
2405 my ($filename, $raw) = @_;
2407 return undef if !defined($raw);
2410 digest
=> Digest
::SHA
::sha1_hex
($raw),
2415 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2416 || die "got strange filename '$filename'";
2424 my @lines = split(/\n/, $raw);
2425 foreach my $line (@lines) {
2426 next if $line =~ m/^\s*$/;
2428 if ($line =~ m/^\[PENDING\]\s*$/i) {
2429 $section = 'pending';
2430 if (defined($descr)) {
2432 $conf->{description
} = $descr;
2435 $conf = $res->{$section} = {};
2438 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2440 if (defined($descr)) {
2442 $conf->{description
} = $descr;
2445 $conf = $res->{snapshots
}->{$section} = {};
2449 if ($line =~ m/^\#(.*)\s*$/) {
2450 $descr = '' if !defined($descr);
2451 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2455 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2456 $descr = '' if !defined($descr);
2457 $descr .= PVE
::Tools
::decode_text
($2);
2458 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2459 $conf->{snapstate
} = $1;
2460 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2463 $conf->{$key} = $value;
2464 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2466 if ($section eq 'pending') {
2467 $conf->{delete} = $value; # we parse this later
2469 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2471 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2474 eval { $value = check_type
($key, $value); };
2476 warn "vm $vmid - unable to parse value of '$key' - $@";
2478 $key = 'ide2' if $key eq 'cdrom';
2479 my $fmt = $confdesc->{$key}->{format
};
2480 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2481 my $v = parse_drive
($key, $value);
2482 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2483 $v->{file
} = $volid;
2484 $value = print_drive
($vmid, $v);
2486 warn "vm $vmid - unable to parse value of '$key'\n";
2491 $conf->{$key} = $value;
2496 if (defined($descr)) {
2498 $conf->{description
} = $descr;
2500 delete $res->{snapstate
}; # just to be sure
2505 sub write_vm_config
{
2506 my ($filename, $conf) = @_;
2508 delete $conf->{snapstate
}; # just to be sure
2510 if ($conf->{cdrom
}) {
2511 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2512 $conf->{ide2
} = $conf->{cdrom
};
2513 delete $conf->{cdrom
};
2516 # we do not use 'smp' any longer
2517 if ($conf->{sockets
}) {
2518 delete $conf->{smp
};
2519 } elsif ($conf->{smp
}) {
2520 $conf->{sockets
} = $conf->{smp
};
2521 delete $conf->{cores
};
2522 delete $conf->{smp
};
2525 my $used_volids = {};
2527 my $cleanup_config = sub {
2528 my ($cref, $pending, $snapname) = @_;
2530 foreach my $key (keys %$cref) {
2531 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2532 $key eq 'snapstate' || $key eq 'pending';
2533 my $value = $cref->{$key};
2534 if ($key eq 'delete') {
2535 die "propertry 'delete' is only allowed in [PENDING]\n"
2537 # fixme: check syntax?
2540 eval { $value = check_type
($key, $value); };
2541 die "unable to parse value of '$key' - $@" if $@;
2543 $cref->{$key} = $value;
2545 if (!$snapname && is_valid_drivename
($key)) {
2546 my $drive = parse_drive
($key, $value);
2547 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2552 &$cleanup_config($conf);
2554 &$cleanup_config($conf->{pending
}, 1);
2556 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2557 die "internal error" if $snapname eq 'pending';
2558 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2561 # remove 'unusedX' settings if we re-add a volume
2562 foreach my $key (keys %$conf) {
2563 my $value = $conf->{$key};
2564 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2565 delete $conf->{$key};
2569 my $generate_raw_config = sub {
2570 my ($conf, $pending) = @_;
2574 # add description as comment to top of file
2575 if (defined(my $descr = $conf->{description
})) {
2577 foreach my $cl (split(/\n/, $descr)) {
2578 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2581 $raw .= "#\n" if $pending;
2585 foreach my $key (sort keys %$conf) {
2586 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2587 $raw .= "$key: $conf->{$key}\n";
2592 my $raw = &$generate_raw_config($conf);
2594 if (scalar(keys %{$conf->{pending
}})){
2595 $raw .= "\n[PENDING]\n";
2596 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2599 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2600 $raw .= "\n[$snapname]\n";
2601 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2611 # we use static defaults from our JSON schema configuration
2612 foreach my $key (keys %$confdesc) {
2613 if (defined(my $default = $confdesc->{$key}->{default})) {
2614 $res->{$key} = $default;
2622 my $vmlist = PVE
::Cluster
::get_vmlist
();
2624 return $res if !$vmlist || !$vmlist->{ids
};
2625 my $ids = $vmlist->{ids
};
2627 foreach my $vmid (keys %$ids) {
2628 my $d = $ids->{$vmid};
2629 next if !$d->{node
} || $d->{node
} ne $nodename;
2630 next if !$d->{type
} || $d->{type
} ne 'qemu';
2631 $res->{$vmid}->{exists} = 1;
2636 # test if VM uses local resources (to prevent migration)
2637 sub check_local_resources
{
2638 my ($conf, $noerr) = @_;
2642 $loc_res = 1 if $conf->{hostusb
}; # old syntax
2643 $loc_res = 1 if $conf->{hostpci
}; # old syntax
2645 foreach my $k (keys %$conf) {
2646 next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2647 # sockets are safe: they will recreated be on the target side post-migrate
2648 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2649 $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2652 die "VM uses local resources\n" if $loc_res && !$noerr;
2657 # check if used storages are available on all nodes (use by migrate)
2658 sub check_storage_availability
{
2659 my ($storecfg, $conf, $node) = @_;
2661 foreach_drive
($conf, sub {
2662 my ($ds, $drive) = @_;
2664 my $volid = $drive->{file
};
2667 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2670 # check if storage is available on both nodes
2671 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2672 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2676 # list nodes where all VM images are available (used by has_feature API)
2678 my ($conf, $storecfg) = @_;
2680 my $nodelist = PVE
::Cluster
::get_nodelist
();
2681 my $nodehash = { map { $_ => 1 } @$nodelist };
2682 my $nodename = PVE
::INotify
::nodename
();
2684 foreach_drive
($conf, sub {
2685 my ($ds, $drive) = @_;
2687 my $volid = $drive->{file
};
2690 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2692 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2693 if ($scfg->{disable
}) {
2695 } elsif (my $avail = $scfg->{nodes
}) {
2696 foreach my $node (keys %$nodehash) {
2697 delete $nodehash->{$node} if !$avail->{$node};
2699 } elsif (!$scfg->{shared
}) {
2700 foreach my $node (keys %$nodehash) {
2701 delete $nodehash->{$node} if $node ne $nodename
2711 my ($pidfile, $pid) = @_;
2713 my $fh = IO
::File-
>new("/proc/$pid/cmdline", "r");
2717 return undef if !$line;
2718 my @param = split(/\0/, $line);
2720 my $cmd = $param[0];
2721 return if !$cmd || ($cmd !~ m
|kvm
$| && $cmd !~ m
|qemu-system-x86_64
$|);
2723 for (my $i = 0; $i < scalar (@param); $i++) {
2726 if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2727 my $p = $param[$i+1];
2728 return 1 if $p && ($p eq $pidfile);
2737 my ($vmid, $nocheck, $node) = @_;
2739 my $filename = PVE
::QemuConfig-
>config_file($vmid, $node);
2741 die "unable to find configuration file for VM $vmid - no such machine\n"
2742 if !$nocheck && ! -f
$filename;
2744 my $pidfile = pidfile_name
($vmid);
2746 if (my $fd = IO
::File-
>new("<$pidfile")) {
2751 my $mtime = $st->mtime;
2752 if ($mtime > time()) {
2753 warn "file '$filename' modified in future\n";
2756 if ($line =~ m/^(\d+)$/) {
2758 if (check_cmdline
($pidfile, $pid)) {
2759 if (my $pinfo = PVE
::ProcFSTools
::check_process_running
($pid)) {
2771 my $vzlist = config_list
();
2773 my $fd = IO
::Dir-
>new($var_run_tmpdir) || return $vzlist;
2775 while (defined(my $de = $fd->read)) {
2776 next if $de !~ m/^(\d+)\.pid$/;
2778 next if !defined($vzlist->{$vmid});
2779 if (my $pid = check_running
($vmid)) {
2780 $vzlist->{$vmid}->{pid
} = $pid;
2788 my ($storecfg, $conf) = @_;
2790 my $bootdisk = $conf->{bootdisk
};
2791 return undef if !$bootdisk;
2792 return undef if !is_valid_drivename
($bootdisk);
2794 return undef if !$conf->{$bootdisk};
2796 my $drive = parse_drive
($bootdisk, $conf->{$bootdisk});
2797 return undef if !defined($drive);
2799 return undef if drive_is_cdrom
($drive);
2801 my $volid = $drive->{file
};
2802 return undef if !$volid;
2804 return $drive->{size
};
2807 our $vmstatus_return_properties = {
2808 vmid
=> get_standard_option
('pve-vmid'),
2810 description
=> "Qemu process status.",
2812 enum
=> ['stopped', 'running'],
2815 description
=> "Maximum memory in bytes.",
2818 renderer
=> 'bytes',
2821 description
=> "Root disk size in bytes.",
2824 renderer
=> 'bytes',
2827 description
=> "VM name.",
2832 description
=> "Qemu QMP agent status.",
2837 description
=> "PID of running qemu process.",
2842 description
=> "Uptime.",
2845 renderer
=> 'duration',
2848 description
=> "Maximum usable CPUs.",
2854 my $last_proc_pid_stat;
2856 # get VM status information
2857 # This must be fast and should not block ($full == false)
2858 # We only query KVM using QMP if $full == true (this can be slow)
2860 my ($opt_vmid, $full) = @_;
2864 my $storecfg = PVE
::Storage
::config
();
2866 my $list = vzlist
();
2867 my $defaults = load_defaults
();
2869 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2871 my $cpucount = $cpuinfo->{cpus
} || 1;
2873 foreach my $vmid (keys %$list) {
2874 next if $opt_vmid && ($vmid ne $opt_vmid);
2876 my $cfspath = PVE
::QemuConfig-
>cfs_config_path($vmid);
2877 my $conf = PVE
::Cluster
::cfs_read_file
($cfspath) || {};
2879 my $d = { vmid
=> $vmid };
2880 $d->{pid
} = $list->{$vmid}->{pid
};
2882 # fixme: better status?
2883 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2885 my $size = disksize
($storecfg, $conf);
2886 if (defined($size)) {
2887 $d->{disk
} = 0; # no info available
2888 $d->{maxdisk
} = $size;
2894 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2895 * ($conf->{cores
} || $defaults->{cores
});
2896 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2897 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2899 $d->{name
} = $conf->{name
} || "VM $vmid";
2900 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2901 : $defaults->{memory
}*(1024*1024);
2903 if ($conf->{balloon
}) {
2904 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2905 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2906 : $defaults->{shares
};
2917 $d->{diskwrite
} = 0;
2919 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
2921 $d->{serial
} = 1 if conf_has_serial
($conf);
2926 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2927 foreach my $dev (keys %$netdev) {
2928 next if $dev !~ m/^tap([1-9]\d*)i/;
2930 my $d = $res->{$vmid};
2933 $d->{netout
} += $netdev->{$dev}->{receive
};
2934 $d->{netin
} += $netdev->{$dev}->{transmit
};
2937 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
2938 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
2943 my $ctime = gettimeofday
;
2945 foreach my $vmid (keys %$list) {
2947 my $d = $res->{$vmid};
2948 my $pid = $d->{pid
};
2951 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2952 next if !$pstat; # not running
2954 my $used = $pstat->{utime} + $pstat->{stime
};
2956 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2958 if ($pstat->{vsize
}) {
2959 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2962 my $old = $last_proc_pid_stat->{$pid};
2964 $last_proc_pid_stat->{$pid} = {
2972 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2974 if ($dtime > 1000) {
2975 my $dutime = $used - $old->{used
};
2977 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2978 $last_proc_pid_stat->{$pid} = {
2984 $d->{cpu
} = $old->{cpu
};
2988 return $res if !$full;
2990 my $qmpclient = PVE
::QMPClient-
>new();
2992 my $ballooncb = sub {
2993 my ($vmid, $resp) = @_;
2995 my $info = $resp->{'return'};
2996 return if !$info->{max_mem
};
2998 my $d = $res->{$vmid};
3000 # use memory assigned to VM
3001 $d->{maxmem
} = $info->{max_mem
};
3002 $d->{balloon
} = $info->{actual
};
3004 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
3005 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
3006 $d->{freemem
} = $info->{free_mem
};
3009 $d->{ballooninfo
} = $info;
3012 my $blockstatscb = sub {
3013 my ($vmid, $resp) = @_;
3014 my $data = $resp->{'return'} || [];
3015 my $totalrdbytes = 0;
3016 my $totalwrbytes = 0;
3018 for my $blockstat (@$data) {
3019 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
3020 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
3022 $blockstat->{device
} =~ s/drive-//;
3023 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
3025 $res->{$vmid}->{diskread
} = $totalrdbytes;
3026 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
3029 my $statuscb = sub {
3030 my ($vmid, $resp) = @_;
3032 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3033 # this fails if ballon driver is not loaded, so this must be
3034 # the last commnand (following command are aborted if this fails).
3035 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3037 my $status = 'unknown';
3038 if (!defined($status = $resp->{'return'}->{status
})) {
3039 warn "unable to get VM status\n";
3043 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3046 foreach my $vmid (keys %$list) {
3047 next if $opt_vmid && ($vmid ne $opt_vmid);
3048 next if !$res->{$vmid}->{pid
}; # not running
3049 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3052 $qmpclient->queue_execute(undef, 2);
3054 foreach my $vmid (keys %$list) {
3055 next if $opt_vmid && ($vmid ne $opt_vmid);
3056 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3063 my ($conf, $func, @param) = @_;
3065 foreach my $ds (valid_drive_names
()) {
3066 next if !defined($conf->{$ds});
3068 my $drive = parse_drive
($ds, $conf->{$ds});
3071 &$func($ds, $drive, @param);
3076 my ($conf, $func, @param) = @_;
3080 my $test_volid = sub {
3081 my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3085 $volhash->{$volid}->{cdrom
} //= 1;
3086 $volhash->{$volid}->{cdrom
} = 0 if !$is_cdrom;
3088 $volhash->{$volid}->{replicate
} //= 0;
3089 $volhash->{$volid}->{replicate
} = 1 if $replicate;
3091 $volhash->{$volid}->{shared
} //= 0;
3092 $volhash->{$volid}->{shared
} = 1 if $shared;
3094 $volhash->{$volid}->{referenced_in_config
} //= 0;
3095 $volhash->{$volid}->{referenced_in_config
} = 1 if !defined($snapname);
3097 $volhash->{$volid}->{referenced_in_snapshot
}->{$snapname} = 1
3098 if defined($snapname);
3101 foreach_drive
($conf, sub {
3102 my ($ds, $drive) = @_;
3103 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, undef);
3106 foreach my $snapname (keys %{$conf->{snapshots
}}) {
3107 my $snap = $conf->{snapshots
}->{$snapname};
3108 $test_volid->($snap->{vmstate
}, 0, 1, $snapname);
3109 foreach_drive
($snap, sub {
3110 my ($ds, $drive) = @_;
3111 $test_volid->($drive->{file
}, drive_is_cdrom
($drive), $drive->{replicate
} // 1, $drive->{shared
}, $snapname);
3115 foreach my $volid (keys %$volhash) {
3116 &$func($volid, $volhash->{$volid}, @param);
3120 sub conf_has_serial
{
3123 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3124 if ($conf->{"serial$i"}) {
3132 sub vga_conf_has_spice
{
3135 return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3140 sub config_to_command
{
3141 my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3144 my $globalFlags = [];
3145 my $machineFlags = [];
3151 my $kvmver = kvm_user_version
();
3152 my $vernum = 0; # unknown
3153 my $ostype = $conf->{ostype
};
3154 my $winversion = windows_version
($ostype);
3155 my $kvm = $conf->{kvm
} // 1;
3157 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm
} && $kvm);
3159 if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3160 $vernum = $1*1000000+$2*1000;
3161 } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3162 $vernum = $1*1000000+$2*1000+$3;
3165 die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3167 my $have_ovz = -f
'/proc/vz/vestat';
3169 my $q35 = machine_type_is_q35
($conf);
3170 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3171 my $machine_type = $forcemachine || $conf->{machine
};
3172 my $use_old_bios_files = undef;
3173 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3175 my $cpuunits = defined($conf->{cpuunits
}) ?
3176 $conf->{cpuunits
} : $defaults->{cpuunits
};
3178 push @$cmd, '/usr/bin/kvm';
3180 push @$cmd, '-id', $vmid;
3182 my $vmname = $conf->{name
} || "vm$vmid";
3184 push @$cmd, '-name', $vmname;
3188 my $qmpsocket = qmp_socket
($vmid);
3189 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3190 push @$cmd, '-mon', "chardev=qmp,mode=control";
3193 push @$cmd, '-pidfile' , pidfile_name
($vmid);
3195 push @$cmd, '-daemonize';
3197 if ($conf->{smbios1
}) {
3198 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3201 if ($conf->{vmgenid
}) {
3202 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3205 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3206 die "uefi base image not found\n" if ! -f
$OVMF_CODE;
3210 if (my $efidisk = $conf->{efidisk0
}) {
3211 my $d = PVE
::JSONSchema
::parse_property_string
($efidisk_fmt, $efidisk);
3212 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3213 $format = $d->{format
};
3215 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3216 if (!defined($format)) {
3217 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3218 $format = qemu_img_format
($scfg, $volname);
3222 die "efidisk format must be specified\n"
3223 if !defined($format);
3226 warn "no efidisk configured! Using temporary efivars disk.\n";
3227 $path = "/tmp/$vmid-ovmf.fd";
3228 PVE
::Tools
::file_copy
($OVMF_VARS, $path, -s
$OVMF_VARS);
3232 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3233 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3237 # add usb controllers
3238 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
($conf, $bridges, $q35, $usbdesc->{format
}, $MAX_USB_DEVICES);
3239 push @$devices, @usbcontrollers if @usbcontrollers;
3240 my $vga = $conf->{vga
};
3242 my $qxlnum = vga_conf_has_spice
($vga);
3243 $vga = 'qxl' if $qxlnum;
3246 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 9)) {
3247 $vga = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3249 $vga = ($winversion >= 6) ?
'std' : 'cirrus';
3253 # enable absolute mouse coordinates (needed by vnc)
3255 if (defined($conf->{tablet
})) {
3256 $tablet = $conf->{tablet
};
3258 $tablet = $defaults->{tablet
};
3259 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3260 $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3263 push @$devices, '-device', print_tabletdevice_full
($conf) if $tablet;
3266 my $gpu_passthrough;
3269 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
3270 my $d = parse_hostpci
($conf->{"hostpci$i"});
3273 my $pcie = $d->{pcie
};
3275 die "q35 machine model is not enabled" if !$q35;
3276 $pciaddr = print_pcie_addr
("hostpci$i");
3278 $pciaddr = print_pci_addr
("hostpci$i", $bridges);
3281 my $rombar = defined($d->{rombar
}) && !$d->{rombar
} ?
',rombar=0' : '';
3282 my $romfile = $d->{romfile
};
3285 if ($d->{'x-vga'}) {
3286 $xvga = ',x-vga=on';
3289 $gpu_passthrough = 1;
3291 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3295 my $pcidevices = $d->{pciid
};
3296 my $multifunction = 1 if @$pcidevices > 1;
3299 foreach my $pcidevice (@$pcidevices) {
3301 my $id = "hostpci$i";
3302 $id .= ".$j" if $multifunction;
3303 my $addr = $pciaddr;
3304 $addr .= ".$j" if $multifunction;
3305 my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3308 $devicestr .= "$rombar$xvga";
3309 $devicestr .= ",multifunction=on" if $multifunction;
3310 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3313 push @$devices, '-device', $devicestr;
3319 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
($conf, $usbdesc->{format
}, $MAX_USB_DEVICES);
3320 push @$devices, @usbdevices if @usbdevices;
3322 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3323 if (my $path = $conf->{"serial$i"}) {
3324 if ($path eq 'socket') {
3325 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3326 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3327 push @$devices, '-device', "isa-serial,chardev=serial$i";
3329 die "no such serial device\n" if ! -c
$path;
3330 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3331 push @$devices, '-device', "isa-serial,chardev=serial$i";
3337 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3338 if (my $path = $conf->{"parallel$i"}) {
3339 die "no such parallel device\n" if ! -c
$path;
3340 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3341 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3342 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3348 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3349 $sockets = $conf->{sockets
} if $conf->{sockets
};
3351 my $cores = $conf->{cores
} || 1;
3353 my $maxcpus = $sockets * $cores;
3355 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3357 my $allowed_vcpus = $cpuinfo->{cpus
};
3359 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3360 if ($allowed_vcpus < $maxcpus);
3362 if($hotplug_features->{cpu
} && qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 7)) {
3364 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3365 for (my $i = 2; $i <= $vcpus; $i++) {
3366 my $cpustr = print_cpu_device
($conf,$i);
3367 push @$cmd, '-device', $cpustr;
3372 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3374 push @$cmd, '-nodefaults';
3376 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
3378 my $bootindex_hash = {};
3380 foreach my $o (split(//, $bootorder)) {
3381 $bootindex_hash->{$o} = $i*100;
3385 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3387 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3389 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3391 push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3393 if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3394 my $socket = vnc_socket
($vmid);
3395 push @$cmd, '-vnc', "unix:$socket,x509,password";
3397 push @$cmd, '-nographic';
3401 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3403 my $useLocaltime = $conf->{localtime};
3405 if ($winversion >= 5) { # windows
3406 $useLocaltime = 1 if !defined($conf->{localtime});
3408 # use time drift fix when acpi is enabled
3409 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3410 $tdf = 1 if !defined($conf->{tdf
});
3414 if ($winversion >= 6) {
3415 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3416 push @$cmd, '-no-hpet';
3419 push @$rtcFlags, 'driftfix=slew' if $tdf;
3422 push @$machineFlags, 'accel=tcg';
3425 if ($machine_type) {
3426 push @$machineFlags, "type=${machine_type}";
3429 if ($conf->{startdate
}) {
3430 push @$rtcFlags, "base=$conf->{startdate}";
3431 } elsif ($useLocaltime) {
3432 push @$rtcFlags, 'base=localtime';
3435 my $cpu = $kvm ?
"kvm64" : "qemu64";
3436 if (my $cputype = $conf->{cpu
}) {
3437 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
3438 or die "Cannot parse cpu description: $cputype\n";
3439 $cpu = $cpuconf->{cputype
};
3440 $kvm_off = 1 if $cpuconf->{hidden
};
3442 if (defined(my $flags = $cpuconf->{flags
})) {
3443 push @$cpuFlags, split(";", $flags);
3447 push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3449 push @$cpuFlags , '-x2apic'
3450 if $conf->{ostype
} && $conf->{ostype
} eq 'solaris';
3452 push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3454 push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3456 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3)) {
3458 push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3459 push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3462 add_hyperv_enlightenments
($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios
}, $gpu_passthrough) if $kvm;
3464 push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3466 push @$cpuFlags, 'kvm=off' if $kvm_off;
3468 my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3469 die "internal error"; # should not happen
3471 push @$cpuFlags, "vendor=${cpu_vendor}"
3472 if $cpu_vendor ne 'default';
3474 $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3476 push @$cmd, '-cpu', $cpu;
3478 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3480 push @$cmd, '-S' if $conf->{freeze
};
3482 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3485 #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3486 #push @$cmd, '-soundhw', 'es1370';
3487 #push @$cmd, '-soundhw', $soundhw if $soundhw;
3489 if (parse_guest_agent
($conf)->{enabled
}) {
3490 my $qgasocket = qmp_socket
($vmid, 1);
3491 my $pciaddr = print_pci_addr
("qga0", $bridges);
3492 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3493 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3494 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3502 for(my $i = 1; $i < $qxlnum; $i++){
3503 my $pciaddr = print_pci_addr
("vga$i", $bridges);
3504 push @$devices, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3507 # assume other OS works like Linux
3508 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3509 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3513 my $pciaddr = print_pci_addr
("spice", $bridges);
3515 my $nodename = PVE
::INotify
::nodename
();
3516 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3517 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3518 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3519 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3520 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3522 push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3524 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3525 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3526 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3529 # enable balloon by default, unless explicitly disabled
3530 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3531 $pciaddr = print_pci_addr
("balloon0", $bridges);
3532 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3535 if ($conf->{watchdog
}) {
3536 my $wdopts = parse_watchdog
($conf->{watchdog
});
3537 $pciaddr = print_pci_addr
("watchdog", $bridges);
3538 my $watchdog = $wdopts->{model
} || 'i6300esb';
3539 push @$devices, '-device', "$watchdog$pciaddr";
3540 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3544 my $scsicontroller = {};
3545 my $ahcicontroller = {};
3546 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3548 # Add iscsi initiator name if available
3549 if (my $initiator = get_initiator_name
()) {
3550 push @$devices, '-iscsi', "initiator-name=$initiator";
3553 foreach_drive
($conf, sub {
3554 my ($ds, $drive) = @_;
3556 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3557 push @$vollist, $drive->{file
};
3560 # ignore efidisk here, already added in bios/fw handling code above
3561 return if $drive->{interface
} eq 'efidisk';
3563 $use_virtio = 1 if $ds =~ m/^virtio/;
3565 if (drive_is_cdrom
($drive)) {
3566 if ($bootindex_hash->{d
}) {
3567 $drive->{bootindex
} = $bootindex_hash->{d
};
3568 $bootindex_hash->{d
} += 1;
3571 if ($bootindex_hash->{c
}) {
3572 $drive->{bootindex
} = $bootindex_hash->{c
} if $conf->{bootdisk
} && ($conf->{bootdisk
} eq $ds);
3573 $bootindex_hash->{c
} += 1;
3577 if($drive->{interface
} eq 'virtio'){
3578 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3581 if ($drive->{interface
} eq 'scsi') {
3583 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3585 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges);
3586 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3589 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3590 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3591 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3592 } elsif ($drive->{iothread
}) {
3593 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3597 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3598 $queues = ",num_queues=$drive->{queues}";
3601 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3602 $scsicontroller->{$controller}=1;
3605 if ($drive->{interface
} eq 'sata') {
3606 my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3607 $pciaddr = print_pci_addr
("ahci$controller", $bridges);
3608 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3609 $ahcicontroller->{$controller}=1;
3612 my $drive_cmd = print_drive_full
($storecfg, $vmid, $drive);
3613 push @$devices, '-drive',$drive_cmd;
3614 push @$devices, '-device', print_drivedevice_full
($storecfg, $conf, $vmid, $drive, $bridges);
3617 for (my $i = 0; $i < $MAX_NETS; $i++) {
3618 next if !$conf->{"net$i"};
3619 my $d = parse_net
($conf->{"net$i"});
3622 $use_virtio = 1 if $d->{model
} eq 'virtio';
3624 if ($bootindex_hash->{n
}) {
3625 $d->{bootindex
} = $bootindex_hash->{n
};
3626 $bootindex_hash->{n
} += 1;
3629 my $netdevfull = print_netdev_full
($vmid,$conf,$d,"net$i");
3630 push @$devices, '-netdev', $netdevfull;
3632 my $netdevicefull = print_netdevice_full
($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3633 push @$devices, '-device', $netdevicefull;
3638 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3)) {
3643 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3645 while (my ($k, $v) = each %$bridges) {
3646 $pciaddr = print_pci_addr
("pci.$k");
3647 unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3652 if ($conf->{args
}) {
3653 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3657 push @$cmd, @$devices;
3658 push @$cmd, '-rtc', join(',', @$rtcFlags)
3659 if scalar(@$rtcFlags);
3660 push @$cmd, '-machine', join(',', @$machineFlags)
3661 if scalar(@$machineFlags);
3662 push @$cmd, '-global', join(',', @$globalFlags)
3663 if scalar(@$globalFlags);
3665 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3670 return "${var_run_tmpdir}/$vmid.vnc";
3676 my $res = vm_mon_cmd
($vmid, 'query-spice');
3678 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3682 my ($vmid, $qga) = @_;
3683 my $sockettype = $qga ?
'qga' : 'qmp';
3684 return "${var_run_tmpdir}/$vmid.$sockettype";
3689 return "${var_run_tmpdir}/$vmid.pid";
3692 sub vm_devices_list
{
3695 my $res = vm_mon_cmd
($vmid, 'query-pci');
3696 my $devices_to_check = [];
3698 foreach my $pcibus (@$res) {
3699 push @$devices_to_check, @{$pcibus->{devices
}},
3702 while (@$devices_to_check) {
3704 for my $d (@$devices_to_check) {
3705 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3706 next if !$d->{'pci_bridge'};
3708 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
3709 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
3711 $devices_to_check = $to_check;
3714 my $resblock = vm_mon_cmd
($vmid, 'query-block');
3715 foreach my $block (@$resblock) {
3716 if($block->{device
} =~ m/^drive-(\S+)/){
3721 my $resmice = vm_mon_cmd
($vmid, 'query-mice');
3722 foreach my $mice (@$resmice) {
3723 if ($mice->{name
} eq 'QEMU HID Tablet') {
3724 $devices->{tablet
} = 1;
3729 # for usb devices there is no query-usb
3730 # but we can iterate over the entries in
3731 # qom-list path=/machine/peripheral
3732 my $resperipheral = vm_mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
3733 foreach my $per (@$resperipheral) {
3734 if ($per->{name
} =~ m/^usb\d+$/) {
3735 $devices->{$per->{name
}} = 1;
3743 my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3745 my $q35 = machine_type_is_q35
($conf);
3747 my $devices_list = vm_devices_list
($vmid);
3748 return 1 if defined($devices_list->{$deviceid});
3750 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3752 if ($deviceid eq 'tablet') {
3754 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf));
3756 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3758 die "usb hotplug currently not reliable\n";
3759 # since we can't reliably hot unplug all added usb devices
3760 # and usb passthrough disables live migration
3761 # we disable usb hotplugging for now
3762 qemu_deviceadd
($vmid, PVE
::QemuServer
::USB
::print_usbdevice_full
($conf, $deviceid, $device));
3764 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3766 qemu_iothread_add
($vmid, $deviceid, $device);
3768 qemu_driveadd
($storecfg, $vmid, $device);
3769 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device);
3771 qemu_deviceadd
($vmid, $devicefull);
3772 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3774 eval { qemu_drivedel
($vmid, $deviceid); };
3779 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3782 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3783 my $pciaddr = print_pci_addr
($deviceid);
3784 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3786 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3788 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3789 qemu_iothread_add
($vmid, $deviceid, $device);
3790 $devicefull .= ",iothread=iothread-$deviceid";
3793 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3794 $devicefull .= ",num_queues=$device->{queues}";
3797 qemu_deviceadd
($vmid, $devicefull);
3798 qemu_deviceaddverify
($vmid, $deviceid);
3800 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3802 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device);
3803 qemu_driveadd
($storecfg, $vmid, $device);
3805 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device);
3806 eval { qemu_deviceadd
($vmid, $devicefull); };
3808 eval { qemu_drivedel
($vmid, $deviceid); };
3813 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3815 return undef if !qemu_netdevadd
($vmid, $conf, $device, $deviceid);
3817 my $machine_type = PVE
::QemuServer
::qemu_machine_pxe
($vmid, $conf);
3818 my $use_old_bios_files = undef;
3819 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3821 my $netdevicefull = print_netdevice_full
($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3822 qemu_deviceadd
($vmid, $netdevicefull);
3823 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3825 eval { qemu_netdevdel
($vmid, $deviceid); };
3830 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3833 my $pciaddr = print_pci_addr
($deviceid);
3834 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3836 qemu_deviceadd
($vmid, $devicefull);
3837 qemu_deviceaddverify
($vmid, $deviceid);
3840 die "can't hotplug device '$deviceid'\n";
3846 # fixme: this should raise exceptions on error!
3847 sub vm_deviceunplug
{
3848 my ($vmid, $conf, $deviceid) = @_;
3850 my $devices_list = vm_devices_list
($vmid);
3851 return 1 if !defined($devices_list->{$deviceid});
3853 die "can't unplug bootdisk" if $conf->{bootdisk
} && $conf->{bootdisk
} eq $deviceid;
3855 if ($deviceid eq 'tablet') {
3857 qemu_devicedel
($vmid, $deviceid);
3859 } elsif ($deviceid =~ m/^usb\d+$/) {
3861 die "usb hotplug currently not reliable\n";
3862 # when unplugging usb devices this way,
3863 # there may be remaining usb controllers/hubs
3864 # so we disable it for now
3865 qemu_devicedel
($vmid, $deviceid);
3866 qemu_devicedelverify
($vmid, $deviceid);
3868 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3870 qemu_devicedel
($vmid, $deviceid);
3871 qemu_devicedelverify
($vmid, $deviceid);
3872 qemu_drivedel
($vmid, $deviceid);
3873 qemu_iothread_del
($conf, $vmid, $deviceid);
3875 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3877 qemu_devicedel
($vmid, $deviceid);
3878 qemu_devicedelverify
($vmid, $deviceid);
3879 qemu_iothread_del
($conf, $vmid, $deviceid);
3881 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3883 qemu_devicedel
($vmid, $deviceid);
3884 qemu_drivedel
($vmid, $deviceid);
3885 qemu_deletescsihw
($conf, $vmid, $deviceid);
3887 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3889 qemu_devicedel
($vmid, $deviceid);
3890 qemu_devicedelverify
($vmid, $deviceid);
3891 qemu_netdevdel
($vmid, $deviceid);
3894 die "can't unplug device '$deviceid'\n";
3900 sub qemu_deviceadd
{
3901 my ($vmid, $devicefull) = @_;
3903 $devicefull = "driver=".$devicefull;
3904 my %options = split(/[=,]/, $devicefull);
3906 vm_mon_cmd
($vmid, "device_add" , %options);
3909 sub qemu_devicedel
{
3910 my ($vmid, $deviceid) = @_;
3912 my $ret = vm_mon_cmd
($vmid, "device_del", id
=> $deviceid);
3915 sub qemu_iothread_add
{
3916 my($vmid, $deviceid, $device) = @_;
3918 if ($device->{iothread
}) {
3919 my $iothreads = vm_iothreads_list
($vmid);
3920 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3924 sub qemu_iothread_del
{
3925 my($conf, $vmid, $deviceid) = @_;
3927 my $device = parse_drive
($deviceid, $conf->{$deviceid});
3928 if ($device->{iothread
}) {
3929 my $iothreads = vm_iothreads_list
($vmid);
3930 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3934 sub qemu_objectadd
{
3935 my($vmid, $objectid, $qomtype) = @_;
3937 vm_mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
3942 sub qemu_objectdel
{
3943 my($vmid, $objectid) = @_;
3945 vm_mon_cmd
($vmid, "object-del", id
=> $objectid);
3951 my ($storecfg, $vmid, $device) = @_;
3953 my $drive = print_drive_full
($storecfg, $vmid, $device);
3954 $drive =~ s/\\/\\\\/g;
3955 my $ret = vm_human_monitor_command
($vmid, "drive_add auto \"$drive\"");
3957 # If the command succeeds qemu prints: "OK
"
3958 return 1 if $ret =~ m/OK/s;
3960 die "adding drive failed
: $ret\n";
3964 my($vmid, $deviceid) = @_;
3966 my $ret = vm_human_monitor_command($vmid, "drive_del drive-
$deviceid");
3969 return 1 if $ret eq "";
3971 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3972 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3974 die "deleting drive
$deviceid failed
: $ret\n";
3977 sub qemu_deviceaddverify {
3978 my ($vmid, $deviceid) = @_;
3980 for (my $i = 0; $i <= 5; $i++) {
3981 my $devices_list = vm_devices_list($vmid);
3982 return 1 if defined($devices_list->{$deviceid});
3986 die "error on hotplug device
'$deviceid'\n";
3990 sub qemu_devicedelverify {
3991 my ($vmid, $deviceid) = @_;
3993 # need to verify that the device is correctly removed as device_del
3994 # is async and empty return is not reliable
3996 for (my $i = 0; $i <= 5; $i++) {
3997 my $devices_list = vm_devices_list($vmid);
3998 return 1 if !defined($devices_list->{$deviceid});
4002 die "error on hot-unplugging device
'$deviceid'\n";
4005 sub qemu_findorcreatescsihw {
4006 my ($storecfg, $conf, $vmid, $device) = @_;
4008 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4010 my $scsihwid="$controller_prefix$controller";
4011 my $devices_list = vm_devices_list($vmid);
4013 if(!defined($devices_list->{$scsihwid})) {
4014 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
4020 sub qemu_deletescsihw {
4021 my ($conf, $vmid, $opt) = @_;
4023 my $device = parse_drive($opt, $conf->{$opt});
4025 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4026 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4030 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4032 my $devices_list = vm_devices_list($vmid);
4033 foreach my $opt (keys %{$devices_list}) {
4034 if (PVE::QemuServer::is_valid_drivename($opt)) {
4035 my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4036 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4042 my $scsihwid="scsihw
$controller";
4044 vm_deviceunplug($vmid, $conf, $scsihwid);
4049 sub qemu_add_pci_bridge {
4050 my ($storecfg, $conf, $vmid, $device) = @_;
4056 print_pci_addr($device, $bridges);
4058 while (my ($k, $v) = each %$bridges) {
4061 return 1 if !defined($bridgeid) || $bridgeid < 1;
4063 my $bridge = "pci
.$bridgeid";
4064 my $devices_list = vm_devices_list($vmid);
4066 if (!defined($devices_list->{$bridge})) {
4067 vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4073 sub qemu_set_link_status {
4074 my ($vmid, $device, $up) = @_;
4076 vm_mon_cmd($vmid, "set_link
", name => $device,
4077 up => $up ? JSON::true : JSON::false);
4080 sub qemu_netdevadd {
4081 my ($vmid, $conf, $device, $deviceid) = @_;
4083 my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4084 my %options = split(/[=,]/, $netdev);
4086 vm_mon_cmd($vmid, "netdev_add
", %options);
4090 sub qemu_netdevdel {
4091 my ($vmid, $deviceid) = @_;
4093 vm_mon_cmd($vmid, "netdev_del
", id => $deviceid);
4096 sub qemu_usb_hotplug {
4097 my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4101 # remove the old one first
4102 vm_deviceunplug($vmid, $conf, $deviceid);
4104 # check if xhci controller is necessary and available
4105 if ($device->{usb3}) {
4107 my $devicelist = vm_devices_list($vmid);
4109 if (!$devicelist->{xhci}) {
4110 my $pciaddr = print_pci_addr("xhci
");
4111 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4114 my $d = parse_usb_device($device->{host});
4115 $d->{usb3} = $device->{usb3};
4118 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4121 sub qemu_cpu_hotplug {
4122 my ($vmid, $conf, $vcpus) = @_;
4124 my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4127 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4128 $sockets = $conf->{sockets} if $conf->{sockets};
4129 my $cores = $conf->{cores} || 1;
4130 my $maxcpus = $sockets * $cores;
4132 $vcpus = $maxcpus if !$vcpus;
4134 die "you can
't add more vcpus than maxcpus\n"
4135 if $vcpus > $maxcpus;
4137 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4139 if ($vcpus < $currentvcpus) {
4141 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4143 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4144 qemu_devicedel($vmid, "cpu$i");
4146 my $currentrunningvcpus = undef;
4148 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4149 last if scalar(@{$currentrunningvcpus}) == $i-1;
4150 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4154 #update conf after each succesfull cpu unplug
4155 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4156 PVE::QemuConfig->write_config($vmid, $conf);
4159 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4165 my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4166 die "vcpus in running vm does not match its configuration\n"
4167 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4169 if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4171 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4172 my $cpustr = print_cpu_device($conf, $i);
4173 qemu_deviceadd($vmid, $cpustr);
4176 my $currentrunningvcpus = undef;
4178 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4179 last if scalar(@{$currentrunningvcpus}) == $i;
4180 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4184 #update conf after each succesfull cpu hotplug
4185 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4186 PVE::QemuConfig->write_config($vmid, $conf);
4190 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4191 vm_mon_cmd($vmid, "cpu-add", id => int($i));
4196 sub qemu_block_set_io_throttle {
4197 my ($vmid, $deviceid,
4198 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4199 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4200 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4201 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4203 return if !check_running($vmid) ;
4205 vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4207 bps_rd => int($bps_rd),
4208 bps_wr => int($bps_wr),
4210 iops_rd => int($iops_rd),
4211 iops_wr => int($iops_wr),
4212 bps_max => int($bps_max),
4213 bps_rd_max => int($bps_rd_max),
4214 bps_wr_max => int($bps_wr_max),
4215 iops_max => int($iops_max),
4216 iops_rd_max => int($iops_rd_max),
4217 iops_wr_max => int($iops_wr_max),
4218 bps_max_length => int($bps_max_length),
4219 bps_rd_max_length => int($bps_rd_max_length),
4220 bps_wr_max_length => int($bps_wr_max_length),
4221 iops_max_length => int($iops_max_length),
4222 iops_rd_max_length => int($iops_rd_max_length),
4223 iops_wr_max_length => int($iops_wr_max_length),
4228 # old code, only used to shutdown old VM after update
4230 my ($fh, $timeout) = @_;
4232 my $sel = new IO::Select;
4239 while (scalar (@ready = $sel->can_read($timeout))) {
4241 if ($count = $fh->sysread($buf, 8192)) {
4242 if ($buf =~ /^(.*)\(qemu\) $/s) {
4249 if (!defined($count)) {
4256 die "monitor read timeout\n" if !scalar(@ready);
4261 sub qemu_block_resize {
4262 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4264 my $running = check_running($vmid);
4266 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4268 return if !$running;
4270 vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4274 sub qemu_volume_snapshot {
4275 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4277 my $running = check_running($vmid);
4279 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4280 vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4282 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4286 sub qemu_volume_snapshot_delete {
4287 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4289 my $running = check_running($vmid);
4294 my $conf = PVE::QemuConfig->load_config($vmid);
4295 foreach_drive($conf, sub {
4296 my ($ds, $drive) = @_;
4297 $running = 1 if $drive->{file} eq $volid;
4301 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4302 vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4304 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4308 sub set_migration_caps {
4314 "auto-converge" => 1,
4316 "x-rdma-pin-all" => 0,
4321 my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4323 for my $supported_capability (@$supported_capabilities) {
4325 capability => $supported_capability->{capability},
4326 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4330 vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4333 my $fast_plug_option = {
4341 'vmstatestorage
' => 1,
4344 # hotplug changes in [PENDING]
4345 # $selection hash can be used to only apply specified options, for
4346 # example: { cores => 1 } (only apply changed 'cores
')
4347 # $errors ref is used to return error messages
4348 sub vmconfig_hotplug_pending {
4349 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4351 my $defaults = load_defaults();
4353 # commit values which do not have any impact on running VM first
4354 # Note: those option cannot raise errors, we we do not care about
4355 # $selection and always apply them.
4357 my $add_error = sub {
4358 my ($opt, $msg) = @_;
4359 $errors->{$opt} = "hotplug problem - $msg";
4363 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4364 if ($fast_plug_option->{$opt}) {
4365 $conf->{$opt} = $conf->{pending}->{$opt};
4366 delete $conf->{pending}->{$opt};
4372 PVE::QemuConfig->write_config($vmid, $conf);
4373 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4376 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4378 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4379 while (my ($opt, $force) = each %$pending_delete_hash) {
4380 next if $selection && !$selection->{$opt};
4382 if ($opt eq 'hotplug
') {
4383 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4384 } elsif ($opt eq 'tablet
') {
4385 die "skip\n" if !$hotplug_features->{usb};
4386 if ($defaults->{tablet}) {
4387 vm_deviceplug($storecfg, $conf, $vmid, $opt);
4389 vm_deviceunplug($vmid, $conf, $opt);
4391 } elsif ($opt =~ m/^usb\d+/) {
4393 # since we cannot reliably hot unplug usb devices
4394 # we are disabling it
4395 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4396 vm_deviceunplug($vmid, $conf, $opt);
4397 } elsif ($opt eq 'vcpus
') {
4398 die "skip\n" if !$hotplug_features->{cpu};
4399 qemu_cpu_hotplug($vmid, $conf, undef);
4400 } elsif ($opt eq 'balloon
') {
4401 # enable balloon device is not hotpluggable
4402 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4403 # here we reset the ballooning value to memory
4404 my $balloon = $conf->{memory} || $defaults->{memory};
4405 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4406 } elsif ($fast_plug_option->{$opt}) {
4408 } elsif ($opt =~ m/^net(\d+)$/) {
4409 die "skip\n" if !$hotplug_features->{network};
4410 vm_deviceunplug($vmid, $conf, $opt);
4411 } elsif (is_valid_drivename($opt)) {
4412 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4413 vm_deviceunplug($vmid, $conf, $opt);
4414 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4415 } elsif ($opt =~ m/^memory$/) {
4416 die "skip\n" if !$hotplug_features->{memory};
4417 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4418 } elsif ($opt eq 'cpuunits
') {
4419 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4420 } elsif ($opt eq 'cpulimit
') {
4421 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4427 &$add_error($opt, $err) if $err ne "skip\n";
4429 # save new config if hotplug was successful
4430 delete $conf->{$opt};
4431 vmconfig_undelete_pending_option($conf, $opt);
4432 PVE::QemuConfig->write_config($vmid, $conf);
4433 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4437 my $apply_pending_cloudinit;
4438 $apply_pending_cloudinit = sub {
4439 my ($key, $value) = @_;
4440 $apply_pending_cloudinit = sub {}; # once is enough
4442 my @cloudinit_opts = keys %$confdesc_cloudinit;
4443 foreach my $opt (keys %{$conf->{pending}}) {
4444 next if !grep { $_ eq $opt } @cloudinit_opts;
4445 $conf->{$opt} = delete $conf->{pending}->{$opt};
4448 my $new_conf = { %$conf };
4449 $new_conf->{$key} = $value;
4450 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4453 foreach my $opt (keys %{$conf->{pending}}) {
4454 next if $selection && !$selection->{$opt};
4455 my $value = $conf->{pending}->{$opt};
4457 if ($opt eq 'hotplug
') {
4458 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4459 } elsif ($opt eq 'tablet
') {
4460 die "skip\n" if !$hotplug_features->{usb};
4462 vm_deviceplug($storecfg, $conf, $vmid, $opt);
4463 } elsif ($value == 0) {
4464 vm_deviceunplug($vmid, $conf, $opt);
4466 } elsif ($opt =~ m/^usb\d+$/) {
4468 # since we cannot reliably hot unplug usb devices
4469 # we are disabling it
4470 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4471 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4472 die "skip\n" if !$d;
4473 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4474 } elsif ($opt eq 'vcpus
') {
4475 die "skip\n" if !$hotplug_features->{cpu};
4476 qemu_cpu_hotplug($vmid, $conf, $value);
4477 } elsif ($opt eq 'balloon
') {
4478 # enable/disable balloning device is not hotpluggable
4479 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4480 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4481 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4483 # allow manual ballooning if shares is set to zero
4484 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4485 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4486 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4488 } elsif ($opt =~ m/^net(\d+)$/) {
4489 # some changes can be done without hotplug
4490 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4491 $vmid, $opt, $value);
4492 } elsif (is_valid_drivename($opt)) {
4493 # some changes can be done without hotplug
4494 my $drive = parse_drive($opt, $value);
4495 if (drive_is_cloudinit($drive)) {
4496 &$apply_pending_cloudinit($opt, $value);
4498 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4499 $vmid, $opt, $value, 1);
4500 } elsif ($opt =~ m/^memory$/) { #dimms
4501 die "skip\n" if !$hotplug_features->{memory};
4502 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4503 } elsif ($opt eq 'cpuunits
') {
4504 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4505 } elsif ($opt eq 'cpulimit
') {
4506 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4507 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4509 die "skip\n"; # skip non-hot-pluggable options
4513 &$add_error($opt, $err) if $err ne "skip\n";
4515 # save new config if hotplug was successful
4516 $conf->{$opt} = $value;
4517 delete $conf->{pending}->{$opt};
4518 PVE::QemuConfig->write_config($vmid, $conf);
4519 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4524 sub try_deallocate_drive {
4525 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4527 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4528 my $volid = $drive->{file};
4529 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4530 my $sid = PVE::Storage::parse_volume_id($volid);
4531 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
4533 # check if the disk is really unused
4534 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4535 if is_volume_in_use($storecfg, $conf, $key, $volid);
4536 PVE::Storage::vdisk_free($storecfg, $volid);
4539 # If vm is not owner of this disk remove from config
4547 sub vmconfig_delete_or_detach_drive {
4548 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4550 my $drive = parse_drive($opt, $conf->{$opt});
4552 my $rpcenv = PVE::RPCEnvironment::get();
4553 my $authuser = $rpcenv->get_user();
4556 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
4557 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4559 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4563 sub vmconfig_apply_pending {
4564 my ($vmid, $conf, $storecfg) = @_;
4568 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4569 while (my ($opt, $force) = each %$pending_delete_hash) {
4570 die "internal error" if $opt =~ m/^unused/;
4571 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4572 if (!defined($conf->{$opt})) {
4573 vmconfig_undelete_pending_option($conf, $opt);
4574 PVE::QemuConfig->write_config($vmid, $conf);
4575 } elsif (is_valid_drivename($opt)) {
4576 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4577 vmconfig_undelete_pending_option($conf, $opt);
4578 delete $conf->{$opt};
4579 PVE::QemuConfig->write_config($vmid, $conf);
4581 vmconfig_undelete_pending_option($conf, $opt);
4582 delete $conf->{$opt};
4583 PVE::QemuConfig->write_config($vmid, $conf);
4587 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4589 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4590 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4592 if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4593 # skip if nothing changed
4594 } elsif (is_valid_drivename($opt)) {
4595 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4596 if defined($conf->{$opt});
4597 $conf->{$opt} = $conf->{pending}->{$opt};
4599 $conf->{$opt} = $conf->{pending}->{$opt};
4602 delete $conf->{pending}->{$opt};
4603 PVE::QemuConfig->write_config($vmid, $conf);
4607 my $safe_num_ne = sub {
4610 return 0 if !defined($a) && !defined($b);
4611 return 1 if !defined($a);
4612 return 1 if !defined($b);
4617 my $safe_string_ne = sub {
4620 return 0 if !defined($a) && !defined($b);
4621 return 1 if !defined($a);
4622 return 1 if !defined($b);
4627 sub vmconfig_update_net {
4628 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4630 my $newnet = parse_net($value);
4632 if ($conf->{$opt}) {
4633 my $oldnet = parse_net($conf->{$opt});
4635 if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4636 &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4637 &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4638 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4640 # for non online change, we try to hot-unplug
4641 die "skip\n" if !$hotplug;
4642 vm_deviceunplug($vmid, $conf, $opt);
4645 die "internal error" if $opt !~ m/net(\d+)/;
4646 my $iface = "tap${vmid}i$1";
4648 if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4649 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4650 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4651 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4652 PVE::Network::tap_unplug($iface);
4653 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4654 } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4655 # Rate can be applied on its own but any change above needs to
4656 # include the rate in tap_plug since OVS resets everything.
4657 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4660 if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4661 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4669 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4675 sub vmconfig_update_disk {
4676 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4678 # fixme: do we need force?
4680 my $drive = parse_drive($opt, $value);
4682 if ($conf->{$opt}) {
4684 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
4686 my $media = $drive->{media} || 'disk
';
4687 my $oldmedia = $old_drive->{media} || 'disk
';
4688 die "unable to change media type\n" if $media ne $oldmedia;
4690 if (!drive_is_cdrom($old_drive)) {
4692 if ($drive->{file} ne $old_drive->{file}) {
4694 die "skip\n" if !$hotplug;
4696 # unplug and register as unused
4697 vm_deviceunplug($vmid, $conf, $opt);
4698 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4701 # update existing disk
4703 # skip non hotpluggable value
4704 if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4705 &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4706 &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4707 &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4712 if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4713 &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4714 &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4715 &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4716 &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4717 &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4718 &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4719 &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4720 &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4721 &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4722 &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4723 &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4724 &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4725 &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4726 &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4727 &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4728 &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4729 &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4731 qemu_block_set_io_throttle($vmid,"drive-$opt",
4732 ($drive->{mbps} || 0)*1024*1024,
4733 ($drive->{mbps_rd} || 0)*1024*1024,
4734 ($drive->{mbps_wr} || 0)*1024*1024,
4735 $drive->{iops} || 0,
4736 $drive->{iops_rd} || 0,
4737 $drive->{iops_wr} || 0,
4738 ($drive->{mbps_max} || 0)*1024*1024,
4739 ($drive->{mbps_rd_max} || 0)*1024*1024,
4740 ($drive->{mbps_wr_max} || 0)*1024*1024,
4741 $drive->{iops_max} || 0,
4742 $drive->{iops_rd_max} || 0,
4743 $drive->{iops_wr_max} || 0,
4744 $drive->{bps_max_length} || 1,
4745 $drive->{bps_rd_max_length} || 1,
4746 $drive->{bps_wr_max_length} || 1,
4747 $drive->{iops_max_length} || 1,
4748 $drive->{iops_rd_max_length} || 1,
4749 $drive->{iops_wr_max_length} || 1);
4758 if ($drive->{file} eq 'none
') {
4759 vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4760 if (drive_is_cloudinit($old_drive)) {
4761 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4764 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4765 vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4766 vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4774 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4776 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4777 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4781 my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4782 $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4784 PVE::QemuConfig->lock_config($vmid, sub {
4785 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4787 die "you can't start a vm
if it
's a template\n" if PVE::QemuConfig->is_template($conf);
4789 PVE::QemuConfig->check_lock($conf) if !$skiplock;
4791 die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4793 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4794 vmconfig_apply_pending($vmid, $conf, $storecfg);
4795 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4798 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4800 my $defaults = load_defaults();
4802 # set environment variable useful inside network script
4803 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4805 my $local_volumes = {};
4807 if ($targetstorage) {
4808 foreach_drive($conf, sub {
4809 my ($ds, $drive) = @_;
4811 return if drive_is_cdrom($drive);
4813 my $volid = $drive->{file};
4817 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4819 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4820 return if $scfg->{shared};
4821 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4826 foreach my $opt (sort keys %$local_volumes) {
4828 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4829 my $drive = parse_drive($opt, $conf->{$opt});
4831 #if remote storage is specified, use default format
4832 if ($targetstorage && $targetstorage ne "1") {
4833 $storeid = $targetstorage;
4834 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4835 $format = $defFormat;
4837 #else we use same format than original
4838 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4839 $format = qemu_img_format($scfg, $volid);
4842 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4843 my $newdrive = $drive;
4844 $newdrive->{format} = $format;
4845 $newdrive->{file} = $newvolid;
4846 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4847 $local_volumes->{$opt} = $drivestr;
4848 #pass drive to conf for command line
4849 $conf->{$opt} = $drivestr;
4853 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4855 my $migrate_port = 0;
4858 if ($statefile eq 'tcp
') {
4859 my $localip = "localhost";
4860 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter
.cfg
');
4861 my $nodename = PVE::INotify::nodename();
4863 if (!defined($migration_type)) {
4864 if (defined($datacenterconf->{migration}->{type})) {
4865 $migration_type = $datacenterconf->{migration}->{type};
4867 $migration_type = 'secure
';
4871 if ($migration_type eq 'insecure
') {
4872 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4873 if ($migrate_network_addr) {
4874 $localip = $migrate_network_addr;
4876 $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4879 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4882 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4883 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4884 $migrate_uri = "tcp:${localip}:${migrate_port}";
4885 push @$cmd, '-incoming
', $migrate_uri;
4888 } elsif ($statefile eq 'unix
') {
4889 # should be default for secure migrations as a ssh TCP forward
4890 # tunnel is not deterministic reliable ready and fails regurarly
4891 # to set up in time, so use UNIX socket forwards
4892 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4893 unlink $socket_addr;
4895 $migrate_uri = "unix:$socket_addr";
4897 push @$cmd, '-incoming
', $migrate_uri;
4901 push @$cmd, '-loadstate
', $statefile;
4908 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
4909 my $d = parse_hostpci($conf->{"hostpci$i"});
4911 my $pcidevices = $d->{pciid};
4912 foreach my $pcidevice (@$pcidevices) {
4913 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4915 my $info = pci_device_info("0000:$pciid");
4916 die "IOMMU not present\n" if !check_iommu_support();
4917 die "no pci device info for device '$pciid'\n" if !$info;
4918 die "can't unbind
/bind pci group to vfio
'$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4919 die "can
't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4923 PVE::Storage::activate_volumes($storecfg, $vollist);
4925 if (!check_running($vmid, 1)) {
4927 run_command(['/bin/systemctl
', 'stop
', "$vmid.scope"],
4928 outfunc => sub {}, errfunc => sub {});
4932 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4933 : $defaults->{cpuunits};
4935 my $start_timeout = $conf->{hugepages} ? 300 : 30;
4936 my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4939 Slice => 'qemu
.slice
',
4941 CPUShares => $cpuunits
4944 if (my $cpulimit = $conf->{cpulimit}) {
4945 $properties{CPUQuota} = int($cpulimit * 100);
4947 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4949 my $run_qemu = sub {
4950 PVE::Tools::run_fork sub {
4951 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4952 run_command($cmd, %run_params);
4956 if ($conf->{hugepages}) {
4959 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4960 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4962 PVE::QemuServer::Memory::hugepages_mount();
4963 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4965 eval { $run_qemu->() };
4967 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4971 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4973 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4976 eval { $run_qemu->() };
4980 # deactivate volumes if start fails
4981 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4982 die "start failed: $err";
4985 print "migration listens on $migrate_uri\n" if $migrate_uri;
4987 if ($statefile && $statefile ne 'tcp
' && $statefile ne 'unix
') {
4988 eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4992 #start nbd server for storage migration
4993 if ($targetstorage) {
4994 my $nodename = PVE::INotify::nodename();
4995 my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4996 my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4997 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4998 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5000 vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet
', data => { host => "${localip}", port => "${migrate_port}" } } );
5002 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5004 foreach my $opt (sort keys %$local_volumes) {
5005 my $volid = $local_volumes->{$opt};
5006 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5007 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5008 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5012 if ($migratedfrom) {
5014 set_migration_caps($vmid);
5019 print "spice listens on port $spice_port\n";
5020 if ($spice_ticket) {
5021 vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice
', password => $spice_ticket);
5022 vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice
', time => "+30");
5027 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5028 if !$statefile && $conf->{balloon};
5030 foreach my $opt (keys %$conf) {
5031 next if $opt !~ m/^net\d+$/;
5032 my $nicconf = parse_net($conf->{$opt});
5033 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5037 vm_mon_cmd_nocheck($vmid, 'qom-set
',
5038 path => "machine/peripheral/balloon0",
5039 property => "guest-stats-polling-interval",
5040 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5046 my ($vmid, $execute, %params) = @_;
5048 my $cmd = { execute => $execute, arguments => \%params };
5049 vm_qmp_command($vmid, $cmd);
5052 sub vm_mon_cmd_nocheck {
5053 my ($vmid, $execute, %params) = @_;
5055 my $cmd = { execute => $execute, arguments => \%params };
5056 vm_qmp_command($vmid, $cmd, 1);
5059 sub vm_qmp_command {
5060 my ($vmid, $cmd, $nocheck) = @_;
5065 if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5066 $timeout = $cmd->{arguments}->{timeout};
5067 delete $cmd->{arguments}->{timeout};
5071 die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5072 my $sname = qmp_socket($vmid);
5073 if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5074 my $qmpclient = PVE::QMPClient->new();
5076 $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5078 die "unable to open monitor socket\n";
5082 syslog("err", "VM $vmid qmp command failed - $err");
5089 sub vm_human_monitor_command {
5090 my ($vmid, $cmdline) = @_;
5095 execute => 'human-monitor-command
',
5096 arguments => { 'command-line
' => $cmdline},
5099 return vm_qmp_command($vmid, $cmd);
5102 sub vm_commandline {
5103 my ($storecfg, $vmid) = @_;
5105 my $conf = PVE::QemuConfig->load_config($vmid);
5107 my $defaults = load_defaults();
5109 my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5111 return PVE::Tools::cmd2string($cmd);
5115 my ($vmid, $skiplock) = @_;
5117 PVE::QemuConfig->lock_config($vmid, sub {
5119 my $conf = PVE::QemuConfig->load_config($vmid);
5121 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5123 vm_mon_cmd($vmid, "system_reset");
5127 sub get_vm_volumes {
5131 foreach_volid($conf, sub {
5132 my ($volid, $attr) = @_;
5134 return if $volid =~ m|^/|;
5136 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5139 push @$vollist, $volid;
5145 sub vm_stop_cleanup {
5146 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5151 my $vollist = get_vm_volumes($conf);
5152 PVE::Storage::deactivate_volumes($storecfg, $vollist);
5155 foreach my $ext (qw(mon qmp pid vnc qga)) {
5156 unlink "/var/run/qemu-server/${vmid}.$ext";
5159 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5161 warn $@ if $@; # avoid errors - just warn
5164 # Note: use $nockeck to skip tests if VM configuration file exists.
5165 # We need that when migration VMs to other nodes (files already moved)
5166 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5168 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5170 $force = 1 if !defined($force) && !$shutdown;
5173 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5174 kill 15, $pid if $pid;
5175 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5176 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5180 PVE
::QemuConfig-
>lock_config($vmid, sub {
5182 my $pid = check_running
($vmid, $nocheck);
5187 $conf = PVE
::QemuConfig-
>load_config($vmid);
5188 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5189 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5190 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5191 $timeout = $opts->{down
} if $opts->{down
};
5195 $timeout = 60 if !defined($timeout);
5199 if (defined($conf) && parse_guest_agent
($conf)->{enabled
}) {
5200 vm_qmp_command
($vmid, { execute
=> "guest-shutdown" }, $nocheck);
5202 vm_qmp_command
($vmid, { execute
=> "system_powerdown" }, $nocheck);
5205 vm_qmp_command
($vmid, { execute
=> "quit" }, $nocheck);
5212 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5217 if ($count >= $timeout) {
5219 warn "VM still running - terminating now with SIGTERM\n";
5222 die "VM quit/powerdown failed - got timeout\n";
5225 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5230 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5233 die "VM quit/powerdown failed\n";
5241 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5246 if ($count >= $timeout) {
5247 warn "VM still running - terminating now with SIGKILL\n";
5252 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5257 my ($vmid, $skiplock) = @_;
5259 PVE
::QemuConfig-
>lock_config($vmid, sub {
5261 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5263 PVE
::QemuConfig-
>check_lock($conf)
5264 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5266 vm_mon_cmd
($vmid, "stop");
5271 my ($vmid, $skiplock, $nocheck) = @_;
5273 PVE
::QemuConfig-
>lock_config($vmid, sub {
5275 my $res = vm_mon_cmd
($vmid, 'query-status');
5276 my $resume_cmd = 'cont';
5278 if ($res->{status
} && $res->{status
} eq 'suspended') {
5279 $resume_cmd = 'system_wakeup';
5284 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5286 PVE
::QemuConfig-
>check_lock($conf)
5287 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
5289 vm_mon_cmd
($vmid, $resume_cmd);
5292 vm_mon_cmd_nocheck
($vmid, $resume_cmd);
5298 my ($vmid, $skiplock, $key) = @_;
5300 PVE
::QemuConfig-
>lock_config($vmid, sub {
5302 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5304 # there is no qmp command, so we use the human monitor command
5305 vm_human_monitor_command
($vmid, "sendkey $key");
5310 my ($storecfg, $vmid, $skiplock) = @_;
5312 PVE
::QemuConfig-
>lock_config($vmid, sub {
5314 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5316 if (!check_running
($vmid)) {
5317 destroy_vm
($storecfg, $vmid, undef, $skiplock);
5319 die "VM $vmid is running - destroy failed\n";
5327 my ($filename, $buf) = @_;
5329 my $fh = IO
::File-
>new($filename, "w");
5330 return undef if !$fh;
5332 my $res = print $fh $buf;
5339 sub pci_device_info
{
5344 return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5345 my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5347 my $irq = file_read_firstline
("$pcisysfs/devices/$name/irq");
5348 return undef if !defined($irq) || $irq !~ m/^\d+$/;
5350 my $vendor = file_read_firstline
("$pcisysfs/devices/$name/vendor");
5351 return undef if !defined($vendor) || $vendor !~ s/^0x//;
5353 my $product = file_read_firstline
("$pcisysfs/devices/$name/device");
5354 return undef if !defined($product) || $product !~ s/^0x//;
5359 product
=> $product,
5365 has_fl_reset
=> -f
"$pcisysfs/devices/$name/reset" || 0,
5374 my $name = $dev->{name
};
5376 my $fn = "$pcisysfs/devices/$name/reset";
5378 return file_write
($fn, "1");
5381 sub pci_dev_bind_to_vfio
{
5384 my $name = $dev->{name
};
5386 my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5388 if (!-d
$vfio_basedir) {
5389 system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5391 die "Cannot find vfio-pci module!\n" if !-d
$vfio_basedir;
5393 my $testdir = "$vfio_basedir/$name";
5394 return 1 if -d
$testdir;
5396 my $data = "$dev->{vendor} $dev->{product}";
5397 return undef if !file_write
("$vfio_basedir/new_id", $data);
5399 my $fn = "$pcisysfs/devices/$name/driver/unbind";
5400 if (!file_write
($fn, $name)) {
5401 return undef if -f
$fn;
5404 $fn = "$vfio_basedir/bind";
5405 if (! -d
$testdir) {
5406 return undef if !file_write
($fn, $name);
5412 sub pci_dev_group_bind_to_vfio
{
5415 my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5417 if (!-d
$vfio_basedir) {
5418 system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5420 die "Cannot find vfio-pci module!\n" if !-d
$vfio_basedir;
5422 # get IOMMU group devices
5423 opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5424 my @devs = grep /^0000:/, readdir($D);
5427 foreach my $pciid (@devs) {
5428 $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5430 # pci bridges, switches or root ports are not supported
5431 # they have a pci_bus subdirectory so skip them
5432 next if (-e
"$pcisysfs/devices/$pciid/pci_bus");
5434 my $info = pci_device_info
($1);
5435 pci_dev_bind_to_vfio
($info) || die "Cannot bind $pciid to vfio\n";
5441 # vzdump restore implementaion
5443 sub tar_archive_read_firstfile
{
5444 my $archive = shift;
5446 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5448 # try to detect archive type first
5449 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5450 die "unable to open file '$archive'\n";
5451 my $firstfile = <$fh>;
5455 die "ERROR: archive contaions no data\n" if !$firstfile;
5461 sub tar_restore_cleanup
{
5462 my ($storecfg, $statfile) = @_;
5464 print STDERR
"starting cleanup\n";
5466 if (my $fd = IO
::File-
>new($statfile, "r")) {
5467 while (defined(my $line = <$fd>)) {
5468 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5471 if ($volid =~ m
|^/|) {
5472 unlink $volid || die 'unlink failed\n';
5474 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5476 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5478 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5480 print STDERR
"unable to parse line in statfile - $line";
5487 sub restore_archive
{
5488 my ($archive, $vmid, $user, $opts) = @_;
5490 my $format = $opts->{format
};
5493 if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5494 $format = 'tar' if !$format;
5496 } elsif ($archive =~ m/\.tar$/) {
5497 $format = 'tar' if !$format;
5498 } elsif ($archive =~ m/.tar.lzo$/) {
5499 $format = 'tar' if !$format;
5501 } elsif ($archive =~ m/\.vma$/) {
5502 $format = 'vma' if !$format;
5503 } elsif ($archive =~ m/\.vma\.gz$/) {
5504 $format = 'vma' if !$format;
5506 } elsif ($archive =~ m/\.vma\.lzo$/) {
5507 $format = 'vma' if !$format;
5510 $format = 'vma' if !$format; # default
5513 # try to detect archive format
5514 if ($format eq 'tar') {
5515 return restore_tar_archive
($archive, $vmid, $user, $opts);
5517 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5521 sub restore_update_config_line
{
5522 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5524 return if $line =~ m/^\#qmdump\#/;
5525 return if $line =~ m/^\#vzdump\#/;
5526 return if $line =~ m/^lock:/;
5527 return if $line =~ m/^unused\d+:/;
5528 return if $line =~ m/^parent:/;
5529 return if $line =~ m/^template:/; # restored VM is never a template
5531 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5532 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5533 # try to convert old 1.X settings
5534 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5535 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5536 my ($model, $macaddr) = split(/\=/, $devconfig);
5537 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
5540 bridge
=> "vmbr$ind",
5541 macaddr
=> $macaddr,
5543 my $netstr = print_net
($net);
5545 print $outfd "net$cookie->{netcount}: $netstr\n";
5546 $cookie->{netcount
}++;
5548 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5549 my ($id, $netstr) = ($1, $2);
5550 my $net = parse_net
($netstr);
5551 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
5552 $netstr = print_net
($net);
5553 print $outfd "$id: $netstr\n";
5554 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5557 my $di = parse_drive
($virtdev, $value);
5558 if (defined($di->{backup
}) && !$di->{backup
}) {
5559 print $outfd "#$line";
5560 } elsif ($map->{$virtdev}) {
5561 delete $di->{format
}; # format can change on restore
5562 $di->{file
} = $map->{$virtdev};
5563 $value = print_drive
($vmid, $di);
5564 print $outfd "$virtdev: $value\n";
5568 } elsif (($line =~ m/^(vmgenid: )(.*)/)) {
5569 # always generate a new vmgenid
5571 if ($vmgenid ne '0') {
5572 $vmgenid = generate_uuid
();
5574 print $outfd $1.$vmgenid."\n";
5575 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5576 my ($uuid, $uuid_str);
5577 UUID
::generate
($uuid);
5578 UUID
::unparse
($uuid, $uuid_str);
5579 my $smbios1 = parse_smbios1
($2);
5580 $smbios1->{uuid
} = $uuid_str;
5581 print $outfd $1.print_smbios1
($smbios1)."\n";
5588 my ($cfg, $vmid) = @_;
5590 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5592 my $volid_hash = {};
5593 foreach my $storeid (keys %$info) {
5594 foreach my $item (@{$info->{$storeid}}) {
5595 next if !($item->{volid
} && $item->{size
});
5596 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5597 $volid_hash->{$item->{volid
}} = $item;
5604 sub is_volume_in_use
{
5605 my ($storecfg, $conf, $skip_drive, $volid) = @_;
5607 my $path = PVE
::Storage
::path
($storecfg, $volid);
5609 my $scan_config = sub {
5610 my ($cref, $snapname) = @_;
5612 foreach my $key (keys %$cref) {
5613 my $value = $cref->{$key};
5614 if (is_valid_drivename
($key)) {
5615 next if $skip_drive && $key eq $skip_drive;
5616 my $drive = parse_drive
($key, $value);
5617 next if !$drive || !$drive->{file
} || drive_is_cdrom
($drive);
5618 return 1 if $volid eq $drive->{file
};
5619 if ($drive->{file
} =~ m!^/!) {
5620 return 1 if $drive->{file
} eq $path;
5622 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
}, 1);
5624 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid, 1);
5626 return 1 if $path eq PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
5634 return 1 if &$scan_config($conf);
5638 foreach my $snapname (keys %{$conf->{snapshots
}}) {
5639 return 1 if &$scan_config($conf->{snapshots
}->{$snapname}, $snapname);
5645 sub update_disksize
{
5646 my ($vmid, $conf, $volid_hash) = @_;
5649 my $prefix = "VM $vmid:";
5651 # used and unused disks
5652 my $referenced = {};
5654 # Note: it is allowed to define multiple storages with same path (alias), so
5655 # we need to check both 'volid' and real 'path' (two different volid can point
5656 # to the same path).
5658 my $referencedpath = {};
5661 foreach my $opt (keys %$conf) {
5662 if (is_valid_drivename
($opt)) {
5663 my $drive = parse_drive
($opt, $conf->{$opt});
5664 my $volid = $drive->{file
};
5667 $referenced->{$volid} = 1;
5668 if ($volid_hash->{$volid} &&
5669 (my $path = $volid_hash->{$volid}->{path
})) {
5670 $referencedpath->{$path} = 1;
5673 next if drive_is_cdrom
($drive);
5674 next if !$volid_hash->{$volid};
5676 $drive->{size
} = $volid_hash->{$volid}->{size
};
5677 my $new = print_drive
($vmid, $drive);
5678 if ($new ne $conf->{$opt}) {
5680 $conf->{$opt} = $new;
5681 print "$prefix update disk '$opt' information.\n";
5686 # remove 'unusedX' entry if volume is used
5687 foreach my $opt (keys %$conf) {
5688 next if $opt !~ m/^unused\d+$/;
5689 my $volid = $conf->{$opt};
5690 my $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
5691 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5692 print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5694 delete $conf->{$opt};
5697 $referenced->{$volid} = 1;
5698 $referencedpath->{$path} = 1 if $path;
5701 foreach my $volid (sort keys %$volid_hash) {
5702 next if $volid =~ m/vm-$vmid-state-/;
5703 next if $referenced->{$volid};
5704 my $path = $volid_hash->{$volid}->{path
};
5705 next if !$path; # just to be sure
5706 next if $referencedpath->{$path};
5708 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
5709 print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5710 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5717 my ($vmid, $nolock, $dryrun) = @_;
5719 my $cfg = PVE
::Storage
::config
();
5721 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5722 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5723 foreach my $stor (keys %{$cfg->{ids
}}) {
5724 delete($cfg->{ids
}->{$stor}) if ! $cfg->{ids
}->{$stor}->{content
}->{images
};
5727 print "rescan volumes...\n";
5728 my $volid_hash = scan_volids
($cfg, $vmid);
5730 my $updatefn = sub {
5733 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5735 PVE
::QemuConfig-
>check_lock($conf);
5738 foreach my $volid (keys %$volid_hash) {
5739 my $info = $volid_hash->{$volid};
5740 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
5743 my $changes = update_disksize
($vmid, $conf, $vm_volids);
5745 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
5748 if (defined($vmid)) {
5752 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
5755 my $vmlist = config_list
();
5756 foreach my $vmid (keys %$vmlist) {
5760 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
5766 sub restore_vma_archive
{
5767 my ($archive, $vmid, $user, $opts, $comp) = @_;
5769 my $readfrom = $archive;
5771 my $cfg = PVE
::Storage
::config
();
5773 my $bwlimit = $opts->{bwlimit
};
5775 my $dbg_cmdstring = '';
5776 my $add_pipe = sub {
5778 push @$commands, $cmd;
5779 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5780 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
5785 if ($archive eq '-') {
5788 # If we use a backup from a PVE defined storage we also consider that
5789 # storage's rate limit:
5790 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
5791 if (defined($volid)) {
5792 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
5793 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
5795 print STDERR
"applying read rate limit: $readlimit\n";
5796 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5797 $add_pipe->($cstream);
5804 if ($comp eq 'gzip') {
5805 $cmd = ['zcat', $readfrom];
5806 } elsif ($comp eq 'lzop') {
5807 $cmd = ['lzop', '-d', '-c', $readfrom];
5809 die "unknown compression method '$comp'\n";
5814 my $tmpdir = "/var/tmp/vzdumptmp$$";
5817 # disable interrupts (always do cleanups)
5821 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
5823 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5824 POSIX
::mkfifo
($mapfifo, 0600);
5827 my $openfifo = sub {
5828 open($fifofh, '>', $mapfifo) || die $!;
5831 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5838 my $rpcenv = PVE
::RPCEnvironment
::get
();
5840 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
5841 my $tmpfn = "$conffile.$$.tmp";
5843 # Note: $oldconf is undef if VM does not exists
5844 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
5845 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
5849 my $print_devmap = sub {
5850 my $virtdev_hash = {};
5852 my $cfgfn = "$tmpdir/qemu-server.conf";
5854 # we can read the config - that is already extracted
5855 my $fh = IO
::File-
>new($cfgfn, "r") ||
5856 "unable to read qemu-server.conf - $!\n";
5858 my $fwcfgfn = "$tmpdir/qemu-server.fw";
5860 my $pve_firewall_dir = '/etc/pve/firewall';
5861 mkdir $pve_firewall_dir; # make sure the dir exists
5862 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5865 while (defined(my $line = <$fh>)) {
5866 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5867 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5868 die "archive does not contain data for drive '$virtdev'\n"
5869 if !$devinfo->{$devname};
5870 if (defined($opts->{storage
})) {
5871 $storeid = $opts->{storage
} || 'local';
5872 } elsif (!$storeid) {
5875 $format = 'raw' if !$format;
5876 $devinfo->{$devname}->{devname
} = $devname;
5877 $devinfo->{$devname}->{virtdev
} = $virtdev;
5878 $devinfo->{$devname}->{format
} = $format;
5879 $devinfo->{$devname}->{storeid
} = $storeid;
5881 # check permission on storage
5882 my $pool = $opts->{pool
}; # todo: do we need that?
5883 if ($user ne 'root@pam') {
5884 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5887 $storage_limits{$storeid} = $bwlimit;
5889 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5893 foreach my $key (keys %storage_limits) {
5894 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$key], $bwlimit);
5896 print STDERR
"rate limit for storage $key: $limit KiB/s\n";
5897 $storage_limits{$key} = $limit * 1024;
5900 foreach my $devname (keys %$devinfo) {
5901 die "found no device mapping information for device '$devname'\n"
5902 if !$devinfo->{$devname}->{virtdev
};
5905 # create empty/temp config
5907 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
5908 foreach_drive
($oldconf, sub {
5909 my ($ds, $drive) = @_;
5911 return if drive_is_cdrom
($drive);
5913 my $volid = $drive->{file
};
5915 return if !$volid || $volid =~ m
|^/|;
5917 my ($path, $owner) = PVE
::Storage
::path
($cfg, $volid);
5918 return if !$path || !$owner || ($owner != $vmid);
5920 # Note: only delete disk we want to restore
5921 # other volumes will become unused
5922 if ($virtdev_hash->{$ds}) {
5923 eval { PVE
::Storage
::vdisk_free
($cfg, $volid); };
5930 # delete vmstate files
5931 # since after the restore we have no snapshots anymore
5932 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5933 my $snap = $oldconf->{snapshots
}->{$snapname};
5934 if ($snap->{vmstate
}) {
5935 eval { PVE
::Storage
::vdisk_free
($cfg, $snap->{vmstate
}); };
5944 foreach my $virtdev (sort keys %$virtdev_hash) {
5945 my $d = $virtdev_hash->{$virtdev};
5946 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5947 my $storeid = $d->{storeid
};
5948 my $scfg = PVE
::Storage
::storage_config
($cfg, $storeid);
5951 if (my $limit = $storage_limits{$storeid}) {
5952 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5955 # test if requested format is supported
5956 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($cfg, $storeid);
5957 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
5958 $d->{format
} = $defFormat if !$supported;
5960 my $volid = PVE
::Storage
::vdisk_alloc
($cfg, $storeid, $vmid,
5961 $d->{format
}, undef, $alloc_size);
5962 print STDERR
"new volume ID is '$volid'\n";
5963 $d->{volid
} = $volid;
5964 my $path = PVE
::Storage
::path
($cfg, $volid);
5966 PVE
::Storage
::activate_volumes
($cfg,[$volid]);
5968 my $write_zeros = 1;
5969 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
5973 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5975 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5976 $map->{$virtdev} = $volid;
5979 $fh->seek(0, 0) || die "seek failed - $!\n";
5981 my $outfd = new IO
::File
($tmpfn, "w") ||
5982 die "unable to write config for VM $vmid\n";
5984 my $cookie = { netcount
=> 0 };
5985 while (defined(my $line = <$fh>)) {
5986 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
5999 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6000 local $SIG{ALRM
} = sub { die "got timeout\n"; };
6002 $oldtimeout = alarm($timeout);
6009 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6010 my ($dev_id, $size, $devname) = ($1, $2, $3);
6011 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
6012 } elsif ($line =~ m/^CTIME: /) {
6013 # we correctly received the vma config, so we can disable
6014 # the timeout now for disk allocation (set to 10 minutes, so
6015 # that we always timeout if something goes wrong)
6018 print $fifofh "done\n";
6019 my $tmp = $oldtimeout || 0;
6020 $oldtimeout = undef;
6026 print "restore vma archive: $dbg_cmdstring\n";
6027 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
6031 alarm($oldtimeout) if $oldtimeout;
6034 foreach my $devname (keys %$devinfo) {
6035 my $volid = $devinfo->{$devname}->{volid
};
6036 push @$vollist, $volid if $volid;
6039 PVE
::Storage
::deactivate_volumes
($cfg, $vollist);
6047 foreach my $devname (keys %$devinfo) {
6048 my $volid = $devinfo->{$devname}->{volid
};
6051 if ($volid =~ m
|^/|) {
6052 unlink $volid || die 'unlink failed\n';
6054 PVE
::Storage
::vdisk_free
($cfg, $volid);
6056 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6058 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6065 rename($tmpfn, $conffile) ||
6066 die "unable to commit configuration file '$conffile'\n";
6068 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6070 eval { rescan
($vmid, 1); };
6074 sub restore_tar_archive
{
6075 my ($archive, $vmid, $user, $opts) = @_;
6077 if ($archive ne '-') {
6078 my $firstfile = tar_archive_read_firstfile
($archive);
6079 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6080 if $firstfile ne 'qemu-server.conf';
6083 my $storecfg = PVE
::Storage
::config
();
6085 # destroy existing data - keep empty config
6086 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
6087 destroy_vm
($storecfg, $vmid, 1) if -f
$vmcfgfn;
6089 my $tocmd = "/usr/lib/qemu-server/qmextract";
6091 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
6092 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
6093 $tocmd .= ' --prealloc' if $opts->{prealloc
};
6094 $tocmd .= ' --info' if $opts->{info
};
6096 # tar option "xf" does not autodetect compression when read from STDIN,
6097 # so we pipe to zcat
6098 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
6099 PVE
::Tools
::shellquote
("--to-command=$tocmd");
6101 my $tmpdir = "/var/tmp/vzdumptmp$$";
6104 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
6105 local $ENV{VZDUMP_VMID
} = $vmid;
6106 local $ENV{VZDUMP_USER
} = $user;
6108 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6109 my $tmpfn = "$conffile.$$.tmp";
6111 # disable interrupts (always do cleanups)
6115 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6123 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6125 if ($archive eq '-') {
6126 print "extracting archive from STDIN\n";
6127 run_command
($cmd, input
=> "<&STDIN");
6129 print "extracting archive '$archive'\n";
6133 return if $opts->{info
};
6137 my $statfile = "$tmpdir/qmrestore.stat";
6138 if (my $fd = IO
::File-
>new($statfile, "r")) {
6139 while (defined (my $line = <$fd>)) {
6140 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6141 $map->{$1} = $2 if $1;
6143 print STDERR
"unable to parse line in statfile - $line\n";
6149 my $confsrc = "$tmpdir/qemu-server.conf";
6151 my $srcfd = new IO
::File
($confsrc, "r") ||
6152 die "unable to open file '$confsrc'\n";
6154 my $outfd = new IO
::File
($tmpfn, "w") ||
6155 die "unable to write config for VM $vmid\n";
6157 my $cookie = { netcount
=> 0 };
6158 while (defined (my $line = <$srcfd>)) {
6159 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
6171 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
6178 rename $tmpfn, $conffile ||
6179 die "unable to commit configuration file '$conffile'\n";
6181 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6183 eval { rescan
($vmid, 1); };
6187 sub foreach_storage_used_by_vm
{
6188 my ($conf, $func) = @_;
6192 foreach_drive
($conf, sub {
6193 my ($ds, $drive) = @_;
6194 return if drive_is_cdrom
($drive);
6196 my $volid = $drive->{file
};
6198 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
6199 $sidhash->{$sid} = $sid if $sid;
6202 foreach my $sid (sort keys %$sidhash) {
6207 sub do_snapshots_with_qemu
{
6208 my ($storecfg, $volid) = @_;
6210 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
6212 if ($qemu_snap_storage->{$storecfg->{ids
}->{$storage_name}->{type
}}
6213 && !$storecfg->{ids
}->{$storage_name}->{krbd
}){
6217 if ($volid =~ m/\.(qcow2|qed)$/){
6224 sub qga_check_running
{
6225 my ($vmid, $nowarn) = @_;
6227 eval { vm_mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
6229 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6235 sub template_create
{
6236 my ($vmid, $conf, $disk) = @_;
6238 my $storecfg = PVE
::Storage
::config
();
6240 foreach_drive
($conf, sub {
6241 my ($ds, $drive) = @_;
6243 return if drive_is_cdrom
($drive);
6244 return if $disk && $ds ne $disk;
6246 my $volid = $drive->{file
};
6247 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
6249 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
6250 $drive->{file
} = $voliddst;
6251 $conf->{$ds} = print_drive
($vmid, $drive);
6252 PVE
::QemuConfig-
>write_config($vmid, $conf);
6256 sub qemu_img_convert
{
6257 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6259 my $storecfg = PVE
::Storage
::config
();
6260 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
6261 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
6263 if ($src_storeid && $dst_storeid) {
6265 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
6267 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
6268 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6270 my $src_format = qemu_img_format
($src_scfg, $src_volname);
6271 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
6273 my $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
6274 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6277 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6278 push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6279 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
6280 push @$cmd, '-T', 'none' if $src_scfg->{type
} eq 'zfspool';
6281 push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6282 if ($is_zero_initialized) {
6283 push @$cmd, "zeroinit:$dst_path";
6285 push @$cmd, $dst_path;
6290 if($line =~ m/\((\S+)\/100\
%\)/){
6292 my $transferred = int($size * $percent / 100);
6293 my $remaining = $size - $transferred;
6295 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6300 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
6302 die "copy failed: $err" if $err;
6306 sub qemu_img_format
{
6307 my ($scfg, $volname) = @_;
6309 if ($scfg->{path
} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6316 sub qemu_drive_mirror
{
6317 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6319 $jobs = {} if !$jobs;
6323 $jobs->{"drive-$drive"} = {};
6325 if ($dst_volid =~ /^nbd:/) {
6326 $qemu_target = $dst_volid;
6329 my $storecfg = PVE
::Storage
::config
();
6330 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
6332 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
6334 $format = qemu_img_format
($dst_scfg, $dst_volname);
6336 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
6338 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
6341 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
6342 $opts->{format
} = $format if $format;
6344 print "drive mirror is starting for drive-$drive\n";
6346 eval { vm_mon_cmd
($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6349 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6350 die "mirroring error: $err";
6353 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6356 sub qemu_drive_mirror_monitor
{
6357 my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6360 my $err_complete = 0;
6363 die "storage migration timed out\n" if $err_complete > 300;
6365 my $stats = vm_mon_cmd
($vmid, "query-block-jobs");
6367 my $running_mirror_jobs = {};
6368 foreach my $stat (@$stats) {
6369 next if $stat->{type
} ne 'mirror';
6370 $running_mirror_jobs->{$stat->{device
}} = $stat;
6373 my $readycounter = 0;
6375 foreach my $job (keys %$jobs) {
6377 if(defined($jobs->{$job}->{complete
}) && !defined($running_mirror_jobs->{$job})) {
6378 print "$job : finished\n";
6379 delete $jobs->{$job};
6383 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6385 my $busy = $running_mirror_jobs->{$job}->{busy
};
6386 my $ready = $running_mirror_jobs->{$job}->{ready
};
6387 if (my $total = $running_mirror_jobs->{$job}->{len
}) {
6388 my $transferred = $running_mirror_jobs->{$job}->{offset
} || 0;
6389 my $remaining = $total - $transferred;
6390 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6392 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6395 $readycounter++ if $running_mirror_jobs->{$job}->{ready
};
6398 last if scalar(keys %$jobs) == 0;
6400 if ($readycounter == scalar(keys %$jobs)) {
6401 print "all mirroring jobs are ready \n";
6402 last if $skipcomplete; #do the complete later
6404 if ($vmiddst && $vmiddst != $vmid) {
6405 my $agent_running = $qga && qga_check_running
($vmid);
6406 if ($agent_running) {
6407 print "freeze filesystem\n";
6408 eval { PVE
::QemuServer
::vm_mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
6410 print "suspend vm\n";
6411 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
6414 # if we clone a disk for a new target vm, we don't switch the disk
6415 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
6417 if ($agent_running) {
6418 print "unfreeze filesystem\n";
6419 eval { PVE
::QemuServer
::vm_mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
6421 print "resume vm\n";
6422 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
6428 foreach my $job (keys %$jobs) {
6429 # try to switch the disk if source and destination are on the same guest
6430 print "$job: Completing block job...\n";
6432 eval { vm_mon_cmd
($vmid, "block-job-complete", device
=> $job) };
6433 if ($@ =~ m/cannot be completed/) {
6434 print "$job: Block job cannot be completed, try again.\n";
6437 print "$job: Completed successfully.\n";
6438 $jobs->{$job}->{complete
} = 1;
6449 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
6450 die "mirroring error: $err";
6455 sub qemu_blockjobs_cancel
{
6456 my ($vmid, $jobs) = @_;
6458 foreach my $job (keys %$jobs) {
6459 print "$job: Cancelling block job\n";
6460 eval { vm_mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
6461 $jobs->{$job}->{cancel
} = 1;
6465 my $stats = vm_mon_cmd
($vmid, "query-block-jobs");
6467 my $running_jobs = {};
6468 foreach my $stat (@$stats) {
6469 $running_jobs->{$stat->{device
}} = $stat;
6472 foreach my $job (keys %$jobs) {
6474 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
6475 print "$job: Done.\n";
6476 delete $jobs->{$job};
6480 last if scalar(keys %$jobs) == 0;
6487 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6488 $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6493 print "create linked clone of drive $drivename ($drive->{file})\n";
6494 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6495 push @$newvollist, $newvolid;
6498 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6499 $storeid = $storage if $storage;
6501 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
6502 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
6504 print "create full clone of drive $drivename ($drive->{file})\n";
6506 if (drive_is_cloudinit
($drive)) {
6507 $name = "vm-$newvmid-cloudinit";
6508 # cloudinit only supports raw and qcow2 atm:
6509 if ($dst_format eq 'qcow2') {
6511 } elsif ($dst_format ne 'raw') {
6512 die "clone: unhandled format for cloudinit image\n";
6515 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6516 push @$newvollist, $newvolid;
6518 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
6520 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6521 if (!$running || $snapname) {
6522 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
6525 my $kvmver = get_running_qemu_version
($vmid);
6526 if (!qemu_machine_feature_enabled
(undef, $kvmver, 2, 7)) {
6527 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6528 if $drive->{iothread
};
6531 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6535 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
6538 $disk->{format
} = undef;
6539 $disk->{file
} = $newvolid;
6540 $disk->{size
} = $size;
6545 # this only works if VM is running
6546 sub get_current_qemu_machine
{
6549 my $cmd = { execute
=> 'query-machines', arguments
=> {} };
6550 my $res = vm_qmp_command
($vmid, $cmd);
6552 my ($current, $default);
6553 foreach my $e (@$res) {
6554 $default = $e->{name
} if $e->{'is-default'};
6555 $current = $e->{name
} if $e->{'is-current'};
6558 # fallback to the default machine if current is not supported by qemu
6559 return $current || $default || 'pc';
6562 sub get_running_qemu_version
{
6564 my $cmd = { execute
=> 'query-version', arguments
=> {} };
6565 my $res = vm_qmp_command
($vmid, $cmd);
6566 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6569 sub qemu_machine_feature_enabled
{
6570 my ($machine, $kvmver, $version_major, $version_minor) = @_;
6575 if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6577 $current_major = $3;
6578 $current_minor = $4;
6580 } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6582 $current_major = $1;
6583 $current_minor = $2;
6586 return 1 if $current_major > $version_major ||
6587 ($current_major == $version_major &&
6588 $current_minor >= $version_minor);
6591 sub qemu_machine_pxe
{
6592 my ($vmid, $conf, $machine) = @_;
6594 $machine = PVE
::QemuServer
::get_current_qemu_machine
($vmid) if !$machine;
6596 if ($conf->{machine
} && $conf->{machine
} =~ m/\.pxe$/) {
6603 sub qemu_use_old_bios_files
{
6604 my ($machine_type) = @_;
6606 return if !$machine_type;
6608 my $use_old_bios_files = undef;
6610 if ($machine_type =~ m/^(\S+)\.pxe$/) {
6612 $use_old_bios_files = 1;
6614 my $kvmver = kvm_user_version
();
6615 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6616 # load new efi bios files on migration. So this hack is required to allow
6617 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6618 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6619 $use_old_bios_files = !qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 4);
6622 return ($use_old_bios_files, $machine_type);
6625 sub create_efidisk
{
6626 my ($storecfg, $storeid, $vmid, $fmt) = @_;
6628 die "EFI vars default image not found\n" if ! -f
$OVMF_VARS;
6630 my $vars_size = PVE
::Tools
::convert_size
(-s
$OVMF_VARS, 'b' => 'kb');
6631 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6632 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6634 my $path = PVE
::Storage
::path
($storecfg, $volid);
6636 run_command
(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6638 die "Copying EFI vars image failed: $@" if $@;
6640 return ($volid, $vars_size);
6647 dir_glob_foreach
("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6648 my (undef, $id, $function) = @_;
6649 my $res = { id
=> $id, function
=> $function};
6650 push @{$devices->{$id}}, $res;
6653 # Entries should be sorted by functions.
6654 foreach my $id (keys %$devices) {
6655 my $dev = $devices->{$id};
6656 $devices->{$id} = [ sort { $a->{function
} <=> $b->{function
} } @$dev ];
6662 sub vm_iothreads_list
{
6665 my $res = vm_mon_cmd
($vmid, 'query-iothreads');
6668 foreach my $iothread (@$res) {
6669 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
6676 my ($conf, $drive) = @_;
6680 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
6682 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
6688 my $controller = int($drive->{index} / $maxdev);
6689 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single') ?
"virtioscsi" : "scsihw";
6691 return ($maxdev, $controller, $controller_prefix);
6694 sub add_hyperv_enlightenments
{
6695 my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6697 return if $winversion < 6;
6698 return if $bios && $bios eq 'ovmf' && $winversion < 8;
6700 push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6702 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3)) {
6703 push @$cpuFlags , 'hv_spinlocks=0x1fff';
6704 push @$cpuFlags , 'hv_vapic';
6705 push @$cpuFlags , 'hv_time';
6707 push @$cpuFlags , 'hv_spinlocks=0xffff';
6710 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 6)) {
6711 push @$cpuFlags , 'hv_reset';
6712 push @$cpuFlags , 'hv_vpindex';
6713 push @$cpuFlags , 'hv_runtime';
6716 if ($winversion >= 7) {
6717 push @$cpuFlags , 'hv_relaxed';
6719 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 3, 0)) {
6720 push @$cpuFlags , 'hv_synic';
6721 push @$cpuFlags , 'hv_stimer';
6726 sub windows_version
{
6729 return 0 if !$ostype;
6733 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6735 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6737 } elsif ($ostype =~ m/^win(\d+)$/) {
6744 sub resolve_dst_disk_format
{
6745 my ($storecfg, $storeid, $src_volname, $format) = @_;
6746 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6749 # if no target format is specified, use the source disk format as hint
6751 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6752 $format = qemu_img_format
($scfg, $src_volname);
6758 # test if requested format is supported - else use default
6759 my $supported = grep { $_ eq $format } @$validFormats;
6760 $format = $defFormat if !$supported;
6764 sub resolve_first_disk
{
6766 my @disks = PVE
::QemuServer
::valid_drive_names
();
6768 foreach my $ds (reverse @disks) {
6769 next if !$conf->{$ds};
6770 my $disk = PVE
::QemuServer
::parse_drive
($ds, $conf->{$ds});
6771 next if PVE
::QemuServer
::drive_is_cdrom
($disk);
6778 my ($uuid, $uuid_str);
6779 UUID
::generate
($uuid);
6780 UUID
::unparse
($uuid, $uuid_str);
6784 sub generate_smbios1_uuid
{
6785 return "uuid=".generate_uuid
();
6788 # bash completion helper
6790 sub complete_backup_archives
{
6791 my ($cmdname, $pname, $cvalue) = @_;
6793 my $cfg = PVE
::Storage
::config
();
6797 if ($cvalue =~ m/^([^:]+):/) {
6801 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
6804 foreach my $id (keys %$data) {
6805 foreach my $item (@{$data->{$id}}) {
6806 next if $item->{format
} !~ m/^vma\.(gz|lzo)$/;
6807 push @$res, $item->{volid
} if defined($item->{volid
});
6814 my $complete_vmid_full = sub {
6817 my $idlist = vmstatus
();
6821 foreach my $id (keys %$idlist) {
6822 my $d = $idlist->{$id};
6823 if (defined($running)) {
6824 next if $d->{template
};
6825 next if $running && $d->{status
} ne 'running';
6826 next if !$running && $d->{status
} eq 'running';
6835 return &$complete_vmid_full();
6838 sub complete_vmid_stopped
{
6839 return &$complete_vmid_full(0);
6842 sub complete_vmid_running
{
6843 return &$complete_vmid_full(1);
6846 sub complete_storage
{
6848 my $cfg = PVE
::Storage
::config
();
6849 my $ids = $cfg->{ids
};
6852 foreach my $sid (keys %$ids) {
6853 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
6854 next if !$ids->{$sid}->{content
}->{images
};
6864 vm_mon_cmd
($vmid, 'nbd-server-stop');