1 package PVE
::QemuServer
;
22 use Storable
qw(dclone);
23 use PVE
::Exception
qw(raise raise_param_exc);
25 use PVE
::Tools
qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
26 use PVE
::JSONSchema
qw(get_standard_option);
27 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
32 use PVE
::RPCEnvironment
;
33 use Time
::HiRes
qw(gettimeofday);
34 use File
::Copy
qw(copy);
37 my $qemu_snap_storage = {rbd
=> 1, sheepdog
=> 1};
39 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
41 # Note about locking: we use flock on the config file protect
42 # against concurent actions.
43 # Aditionaly, we have a 'lock' setting in the config file. This
44 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
45 # allowed when such lock is set. But you can ignore this kind of
46 # lock with the --skiplock flag.
48 cfs_register_file
('/qemu-server/',
52 PVE
::JSONSchema
::register_standard_option
('skiplock', {
53 description
=> "Ignore locks - only root is allowed to use this option.",
58 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
59 description
=> "Some command save/restore state from this location.",
65 PVE
::JSONSchema
::register_standard_option
('pve-snapshot-name', {
66 description
=> "The name of the snapshot.",
67 type
=> 'string', format
=> 'pve-configid',
71 #no warnings 'redefine';
74 my ($controller, $vmid, $option, $value) = @_;
76 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
77 PVE
::ProcFSTools
::write_proc_entry
($path, $value);
81 my $nodename = PVE
::INotify
::nodename
();
83 mkdir "/etc/pve/nodes/$nodename";
84 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
87 my $var_run_tmpdir = "/var/run/qemu-server";
88 mkdir $var_run_tmpdir;
90 my $lock_dir = "/var/lock/qemu-server";
93 my $pcisysfs = "/sys/bus/pci";
97 description
=> "Emulated CPU type.",
99 enum
=> [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Haswell-noTSX Broadwell Broadwell-noTSX Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
100 format_description
=> 'cputype',
105 description
=> "Do not identify as a KVM virtual machine.",
116 enum
=> [qw(i6300esb ib700)],
117 description
=> "Watchdog type to emulate.",
118 default => 'i6300esb',
123 enum
=> [qw(reset shutdown poweroff pause debug none)],
124 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
128 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
134 description
=> "Specifies whether a VM will be started during system bootup.",
140 description
=> "Automatic restart after crash (currently ignored).",
145 type
=> 'string', format
=> 'pve-hotplug-features',
146 description
=> "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
147 default => 'network,disk,usb',
152 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
158 description
=> "Lock/unlock the VM.",
159 enum
=> [qw(migrate backup snapshot rollback)],
164 description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
172 description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
180 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
187 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
193 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
201 description
=> "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
202 enum
=> PVE
::Tools
::kvmkeymaplist
(),
207 type
=> 'string', format
=> 'dns-name',
208 description
=> "Set a name for the VM. Only used on the configuration web interface.",
213 description
=> "scsi controller model",
214 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
220 description
=> "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
225 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
226 description
=> <<EODESC,
227 Used to enable special optimization/features for specific
230 other => unspecified OS
231 wxp => Microsoft Windows XP
232 w2k => Microsoft Windows 2000
233 w2k3 => Microsoft Windows 2003
234 w2k8 => Microsoft Windows 2008
235 wvista => Microsoft Windows Vista
236 win7 => Microsoft Windows 7
237 win8 => Microsoft Windows 8/2012
238 l24 => Linux 2.4 Kernel
239 l26 => Linux 2.6/3.X Kernel
240 solaris => solaris/opensolaris/openindiania kernel
242 other|l24|l26|solaris ... no special behaviour
243 wxp|w2k|w2k3|w2k8|wvista|win7|win8 ... use --localtime switch
249 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
250 pattern
=> '[acdn]{1,4}',
255 type
=> 'string', format
=> 'pve-qm-bootdisk',
256 description
=> "Enable booting from specified disk.",
257 pattern
=> '(ide|sata|scsi|virtio)\d+',
262 description
=> "The number of CPUs. Please use option -sockets instead.",
269 description
=> "The number of CPU sockets.",
276 description
=> "The number of cores per socket.",
283 description
=> "Enable/disable NUMA.",
289 description
=> "Number of hotplugged vcpus.",
296 description
=> "Enable/disable ACPI.",
302 description
=> "Enable/disable Qemu GuestAgent.",
308 description
=> "Enable/disable KVM hardware virtualization.",
314 description
=> "Enable/disable time drift fix.",
320 description
=> "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
325 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
330 description
=> "Select the VGA type. If you want to use high resolution" .
331 " modes (>= 1280x1024x16) then you should use the options " .
332 "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
333 "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
334 "display sever. For win* OS you can select how many independent " .
335 "displays you want, Linux guests can add displays them self. " .
336 "You can also run without any graphic card, using a serial device" .
338 enum
=> [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
342 type
=> 'string', format
=> 'pve-qm-watchdog',
343 description
=> "Create a virtual hardware watchdog device. Once enabled" .
344 " (by a guest action), the watchdog must be periodically polled " .
345 "by an agent inside the guest or else the watchdog will reset " .
346 "the guest (or execute the respective action specified)",
351 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
352 description
=> "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
353 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
356 startup
=> get_standard_option
('pve-startup-order'),
360 description
=> "Enable/disable Template.",
366 description
=> <<EODESCR,
367 Arbitrary arguments passed to kvm, for example:
369 args: -no-reboot -no-hpet
371 NOTE: this option is for experts only.
378 description
=> "Enable/disable the USB tablet device. This device is " .
379 "usually needed to allow absolute mouse positioning with VNC. " .
380 "Else the mouse runs out of sync with normal VNC clients. " .
381 "If you're running lots of console-only guests on one host, " .
382 "you may consider disabling this to save some context switches. " .
383 "This is turned off by default if you use spice (-vga=qxl).",
388 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
392 migrate_downtime
=> {
395 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
401 type
=> 'string', format
=> 'pve-qm-drive',
402 typetext
=> 'volume',
403 description
=> "This is an alias for option -ide2",
407 description
=> "Emulated CPU type.",
411 parent
=> get_standard_option
('pve-snapshot-name', {
413 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
417 description
=> "Timestamp for snapshots.",
423 type
=> 'string', format
=> 'pve-volume-id',
424 description
=> "Reference to a volume which stores the VM state. This is used internally for snapshots.",
427 description
=> "Specific the Qemu machine type.",
429 pattern
=> '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
434 description
=> "Specify SMBIOS type 1 fields.",
435 type
=> 'string', format
=> 'pve-qm-smbios1',
442 description
=> "Sets the protection flag of the VM. This will prevent the remove operation.",
448 enum
=> [ qw(seabios ovmf) ],
449 description
=> "Select BIOS implementation.",
450 default => 'seabios',
454 # what about other qemu settings ?
456 #machine => 'string',
469 ##soundhw => 'string',
471 while (my ($k, $v) = each %$confdesc) {
472 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
475 my $MAX_IDE_DISKS = 4;
476 my $MAX_SCSI_DISKS = 14;
477 my $MAX_VIRTIO_DISKS = 16;
478 my $MAX_SATA_DISKS = 6;
479 my $MAX_USB_DEVICES = 5;
481 my $MAX_UNUSED_DISKS = 8;
482 my $MAX_HOSTPCI_DEVICES = 4;
483 my $MAX_SERIAL_PORTS = 4;
484 my $MAX_PARALLEL_PORTS = 3;
486 my $MAX_MEM = 4194304;
487 my $STATICMEM = 1024;
492 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
493 description
=> "CPUs accessing this numa node.",
494 format_description
=> "id[-id];...",
498 description
=> "Amount of memory this numa node provides.",
499 format_description
=> "mb",
504 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
505 description
=> "host numa nodes to use",
506 format_description
=> "id[-id];...",
511 enum
=> [qw(preferred bind interleave)],
512 format_description
=> 'preferred|bind|interleave',
513 description
=> "numa allocation policy.",
517 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
520 type
=> 'string', format
=> $numa_fmt,
521 description
=> "numa topology",
523 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
525 for (my $i = 0; $i < $MAX_NUMA; $i++) {
526 $confdesc->{"numa$i"} = $numadesc;
529 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
530 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
531 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
532 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
537 pattern
=> qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
538 description
=> "MAC address",
539 format_description
=> "XX:XX:XX:XX:XX:XX",
542 model
=> { alias
=> 'macaddr', default_key
=> 1 },
543 (map { $_ => { group
=> 'model' } } @$nic_model_list),
546 description
=> 'Bridge to attach the network device to.',
547 format_description
=> 'bridge',
552 minimum
=> 0, maximum
=> 16,
553 description
=> 'Number of packet queues to be used on the device.',
554 format_description
=> 'number',
560 description
=> 'Rate limit in mbps as floating point number.',
561 format_description
=> 'mbps',
566 minimum
=> 2, maximum
=> 4094,
567 description
=> 'VLAN tag to apply to packets on this interface.',
568 format_description
=> 'vlanid',
573 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
574 description
=> 'VLAN trunks to pass through this interface.',
575 format_description
=> 'id;id...',
580 description
=> 'Whether this interface should be protected by the firewall.',
581 format_description
=> '0|1',
586 description
=> 'Whether this interface should be DISconnected (like pulling the plug).',
587 format_description
=> '0|1',
593 type
=> 'string', format
=> 'pve-qm-net',
594 description
=> <<EODESCR,
595 Specify network devices.
597 MODEL is one of: $nic_model_list_txt
599 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
600 automatically generated if not specified.
602 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
604 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
606 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
612 The DHCP server assign addresses to the guest starting from 10.0.2.15.
616 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
618 for (my $i = 0; $i < $MAX_NETS; $i++) {
619 $confdesc->{"net$i"} = $netdesc;
622 PVE
::JSONSchema
::register_format
('pve-volume-id-or-none', \
&verify_volume_id_or_none
);
623 sub verify_volume_id_or_none
{
624 my ($volid, $noerr) = @_;
626 return $volid if $volid eq 'none';
627 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
629 return undef if $noerr;
637 my %drivedesc_base = (
638 volume
=> { alias
=> 'file' },
641 format
=> 'pve-volume-id-or-none',
643 format_description
=> 'volume',
644 description
=> "The drive's backing volume.",
648 format_description
=> 'cdrom|disk',
649 enum
=> [qw(cdrom disk)],
650 description
=> "The drive's media type.",
656 format_description
=> 'count',
657 description
=> "Force the drive's physical geometry to have a specific cylinder count.",
662 format_description
=> 'count',
663 description
=> "Force the drive's physical geometry to have a specific head count.",
668 format_description
=> 'count',
669 description
=> "Force the drive's physical geometry to have a specific sector count.",
674 format_description
=> 'none|lba|auto',
675 enum
=> [qw(none lba auto)],
676 description
=> "Force disk geometry bios translation mode.",
681 format_description
=> 'on|off',
682 description
=> "Whether the drive should be included when making snapshots.",
687 format_description
=> 'none|writethrough|writeback|unsafe|directsync',
688 enum
=> [qw(none writethrough writeback unsafe directsync)],
689 description
=> "The drive's cache mode",
694 format_description
=> 'drive format',
695 enum
=> [qw(raw cow qcow qed qcow2 vmdk cloop)],
696 description
=> "The drive's backing file's data format.",
701 format
=> 'disk-size',
702 description
=> "Disk size. This is purely informational and has no effect.",
707 format_description
=> 'on|off',
708 description
=> "Whether the drive should be included when making backups.",
713 format_description
=> 'enospc|ignore|report|stop',
714 enum
=> [qw(enospc ignore report stop)],
715 description
=> 'Write error action.',
720 format_description
=> 'native|threads',
721 enum
=> [qw(native threads)],
722 description
=> 'AIO type to use.',
727 format_description
=> 'ignore|on',
728 enum
=> [qw(ignore on)],
729 description
=> 'Controls whether to pass discard/trim requests to the underlying storage.',
734 description
=> 'Controls whether to detect and try to optimize writes of zeroes.',
739 format
=> 'urlencoded',
740 format_description
=> 'serial',
741 maxLength
=> 20*3, # *3 since it's %xx url enoded
742 description
=> "The drive's reported serial number, url-encoded, up to 20 bytes long.",
750 format_description
=> 'ignore|report|stop',
751 enum
=> [qw(ignore report stop)],
752 description
=> 'Read error action.',
757 my %iothread_fmt = ( iothread
=> {
759 format_description
=> 'off|on',
760 description
=> "Whether to use iothreads for this drive",
767 format
=> 'urlencoded',
768 format_description
=> 'model',
769 maxLength
=> 40*3, # *3 since it's %xx url enoded
770 description
=> "The drive's reported model name, url-encoded, up to 40 bytes long.",
778 format_description
=> 'nbqueues',
779 description
=> "Number of queues.",
785 my $add_throttle_desc = sub {
786 my ($key, $type, $what, $size, $longsize) = @_;
787 $drivedesc_base{$key} = {
789 format_description
=> $size,
790 description
=> "Maximum $what speed in $longsize per second.",
794 # throughput: (leaky bucket)
795 $add_throttle_desc->('bps', 'integer', 'r/w speed', 'bps', 'bytes');
796 $add_throttle_desc->('bps_rd', 'integer', 'read speed', 'bps', 'bytes');
797 $add_throttle_desc->('bps_wr', 'integer', 'write speed', 'bps', 'bytes');
798 $add_throttle_desc->('mbps', 'number', 'r/w speed', 'mbps', 'megabytes');
799 $add_throttle_desc->('mbps_rd', 'number', 'read speed', 'mbps', 'megabytes');
800 $add_throttle_desc->('mbps_wr', 'number', 'write speed', 'mbps', 'megabytes');
801 $add_throttle_desc->('iops', 'integer', 'r/w I/O', 'iops', 'operations');
802 $add_throttle_desc->('iops_rd', 'integer', 'read I/O', 'iops', 'operations');
803 $add_throttle_desc->('iops_wr', 'integer', 'write I/O', 'iops', 'operations');
805 # pools: (pool of IO before throttling starts taking effect)
806 $add_throttle_desc->('mbps_max', 'number', 'unthrottled r/w pool', 'mbps', 'megabytes');
807 $add_throttle_desc->('mbps_rd_max', 'number', 'unthrottled read pool', 'mbps', 'megabytes');
808 $add_throttle_desc->('mbps_wr_max', 'number', 'unthrottled write pool', 'mbps', 'megabytes');
809 $add_throttle_desc->('iops_max', 'integer', 'unthrottled r/w I/O pool', 'iops', 'operations');
810 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool', 'iops', 'operations');
811 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
821 type
=> 'string', format
=> $ide_fmt,
822 description
=> "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
824 PVE
::JSONSchema
::register_standard_option
("pve-qm-ide", $idedesc);
833 type
=> 'string', format
=> $scsi_fmt,
834 description
=> "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
836 PVE
::JSONSchema
::register_standard_option
("pve-qm-scsi", $scsidesc);
844 type
=> 'string', format
=> $sata_fmt,
845 description
=> "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
847 PVE
::JSONSchema
::register_standard_option
("pve-qm-sata", $satadesc);
856 type
=> 'string', format
=> $virtio_fmt,
857 description
=> "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
859 PVE
::JSONSchema
::register_standard_option
("pve-qm-virtio", $virtiodesc);
872 type
=> 'string', format
=> 'pve-qm-usb-device',
873 format_description
=> 'HOSTUSBDEVICE|spice',
874 description
=> 'The Host USB device or port or the value spice',
879 format_description
=> 'yes|no',
880 description
=> 'Specifies whether if given host option is a USB3 device or port',
886 type
=> 'string', format
=> $usb_fmt,
887 description
=> <<EODESCR,
888 Configure an USB device (n is 0 to 4). This can be used to
889 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
891 'bus-port(.port)*' (decimal numbers) or
892 'vendor_id:product_id' (hexadeciaml numbers) or
895 You can use the 'lsusb -t' command to list existing usb devices.
897 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
899 The value 'spice' can be used to add a usb redirection devices for spice.
901 The 'usb3' option determines whether the device is a USB3 device or not (this does currently not work reliably with spice redirection and is then ignored).
905 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
907 # NOTE: the match-groups of this regex are used in parse_hostpci
908 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
913 pattern
=> qr/$PCIRE(;$PCIRE)*/,
914 format_description
=> 'HOSTPCIID[;HOSTPCIID2...]',
915 description
=> "The PCI ID of a host's PCI device or a list of PCI virtual functions of the host.",
933 PVE
::JSONSchema
::register_format
('pve-qm-hostpci', $hostpci_fmt);
937 type
=> 'string', format
=> 'pve-qm-hostpci',
938 description
=> <<EODESCR,
939 Map host pci devices. HOSTPCIDEVICE syntax is:
941 'bus:dev.func' (hexadecimal numbers)
943 You can us the 'lspci' command to list existing pci devices.
945 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
947 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
949 Experimental: user reported problems with this option.
952 PVE
::JSONSchema
::register_standard_option
("pve-qm-hostpci", $hostpcidesc);
957 pattern
=> '(/dev/.+|socket)',
958 description
=> <<EODESCR,
959 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
961 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
963 Experimental: user reported problems with this option.
970 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
971 description
=> <<EODESCR,
972 Map host parallel devices (n is 0 to 2).
974 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
976 Experimental: user reported problems with this option.
980 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
981 $confdesc->{"parallel$i"} = $paralleldesc;
984 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
985 $confdesc->{"serial$i"} = $serialdesc;
988 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
989 $confdesc->{"hostpci$i"} = $hostpcidesc;
992 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) {
993 $drivename_hash->{"ide$i"} = 1;
994 $confdesc->{"ide$i"} = $idedesc;
997 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) {
998 $drivename_hash->{"sata$i"} = 1;
999 $confdesc->{"sata$i"} = $satadesc;
1002 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) {
1003 $drivename_hash->{"scsi$i"} = 1;
1004 $confdesc->{"scsi$i"} = $scsidesc ;
1007 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) {
1008 $drivename_hash->{"virtio$i"} = 1;
1009 $confdesc->{"virtio$i"} = $virtiodesc;
1012 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1013 $confdesc->{"usb$i"} = $usbdesc;
1018 type
=> 'string', format
=> 'pve-volume-id',
1019 description
=> "Reference to unused volumes.",
1022 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
1023 $confdesc->{"unused$i"} = $unuseddesc;
1026 my $kvm_api_version = 0;
1030 return $kvm_api_version if $kvm_api_version;
1032 my $fh = IO
::File-
>new("</dev/kvm") ||
1035 if (my $v = $fh->ioctl(KVM_GET_API_VERSION
(), 0)) {
1036 $kvm_api_version = $v;
1041 return $kvm_api_version;
1044 my $kvm_user_version;
1046 sub kvm_user_version
{
1048 return $kvm_user_version if $kvm_user_version;
1050 $kvm_user_version = 'unknown';
1054 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1055 $kvm_user_version = $2;
1059 eval { run_command
("kvm -version", outfunc
=> $code); };
1062 return $kvm_user_version;
1066 my $kernel_has_vhost_net = -c
'/dev/vhost-net';
1068 sub valid_drive_names
{
1069 # order is important - used to autoselect boot disk
1070 return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1071 (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1072 (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1073 (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1076 sub is_valid_drivename
{
1079 return defined($drivename_hash->{$dev});
1084 return defined($confdesc->{$key});
1088 return $nic_model_list;
1091 sub os_list_description
{
1095 wxp
=> 'Windows XP',
1096 w2k
=> 'Windows 2000',
1097 w2k3
=>, 'Windows 2003',
1098 w2k8
=> 'Windows 2008',
1099 wvista
=> 'Windows Vista',
1100 win7
=> 'Windows 7',
1101 win8
=> 'Windows 8/2012',
1109 sub get_cdrom_path
{
1111 return $cdrom_path if $cdrom_path;
1113 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1114 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1115 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1119 my ($storecfg, $vmid, $cdrom) = @_;
1121 if ($cdrom eq 'cdrom') {
1122 return get_cdrom_path
();
1123 } elsif ($cdrom eq 'none') {
1125 } elsif ($cdrom =~ m
|^/|) {
1128 return PVE
::Storage
::path
($storecfg, $cdrom);
1132 # try to convert old style file names to volume IDs
1133 sub filename_to_volume_id
{
1134 my ($vmid, $file, $media) = @_;
1136 if (!($file eq 'none' || $file eq 'cdrom' ||
1137 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1139 return undef if $file =~ m
|/|;
1141 if ($media && $media eq 'cdrom') {
1142 $file = "local:iso/$file";
1144 $file = "local:$vmid/$file";
1151 sub verify_media_type
{
1152 my ($opt, $vtype, $media) = @_;
1157 if ($media eq 'disk') {
1159 } elsif ($media eq 'cdrom') {
1162 die "internal error";
1165 return if ($vtype eq $etype);
1167 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1170 sub cleanup_drive_path
{
1171 my ($opt, $storecfg, $drive) = @_;
1173 # try to convert filesystem paths to volume IDs
1175 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1176 ($drive->{file
} !~ m
|^/dev/.+|) &&
1177 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1178 ($drive->{file
} !~ m/^\d+$/)) {
1179 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1180 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1181 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1182 verify_media_type
($opt, $vtype, $drive->{media
});
1183 $drive->{file
} = $volid;
1186 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1189 sub parse_hotplug_features
{
1194 return $res if $data eq '0';
1196 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1198 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1199 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1202 warn "ignoring unknown hotplug feature '$feature'\n";
1208 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1209 sub pve_verify_hotplug_features
{
1210 my ($value, $noerr) = @_;
1212 return $value if parse_hotplug_features
($value);
1214 return undef if $noerr;
1216 die "unable to parse hotplug option\n";
1219 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1220 # [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1221 # [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1222 # [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1223 # [,iothread=on][,serial=serial][,model=model]
1226 my ($key, $data) = @_;
1228 my ($interface, $index);
1230 if ($key =~ m/^([^\d]+)(\d+)$/) {
1237 my $desc = $key =~ /^unused\d+$/ ?
$alldrive_fmt
1238 : $confdesc->{$key}->{format
};
1240 warn "invalid drive key: $key\n";
1243 my $res = eval { PVE
::JSONSchema
::parse_property_string
($desc, $data) };
1244 return undef if !$res;
1245 $res->{interface
} = $interface;
1246 $res->{index} = $index;
1249 foreach my $opt (qw(bps bps_rd bps_wr)) {
1250 if (my $bps = defined(delete $res->{$opt})) {
1251 if (defined($res->{"m$opt"})) {
1252 warn "both $opt and m$opt specified\n";
1256 $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1259 return undef if $error;
1261 return undef if $res->{mbps_rd
} && $res->{mbps
};
1262 return undef if $res->{mbps_wr
} && $res->{mbps
};
1263 return undef if $res->{iops_rd
} && $res->{iops
};
1264 return undef if $res->{iops_wr
} && $res->{iops
};
1266 if ($res->{media
} && ($res->{media
} eq 'cdrom')) {
1267 return undef if $res->{snapshot
} || $res->{trans
} || $res->{format
};
1268 return undef if $res->{heads
} || $res->{secs
} || $res->{cyls
};
1269 return undef if $res->{interface
} eq 'virtio';
1272 if (my $size = $res->{size
}) {
1273 return undef if !defined($res->{size
} = PVE
::JSONSchema
::parse_size
($size));
1280 my ($vmid, $drive) = @_;
1281 my $data = { %$drive };
1282 delete $data->{$_} for qw(index interface);
1283 return PVE
::JSONSchema
::print_property_string
($data, $alldrive_fmt);
1287 my($fh, $noerr) = @_;
1290 my $SG_GET_VERSION_NUM = 0x2282;
1292 my $versionbuf = "\x00" x
8;
1293 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1295 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1298 my $version = unpack("I", $versionbuf);
1299 if ($version < 30000) {
1300 die "scsi generic interface too old\n" if !$noerr;
1304 my $buf = "\x00" x
36;
1305 my $sensebuf = "\x00" x
8;
1306 my $cmd = pack("C x3 C x1", 0x12, 36);
1308 # see /usr/include/scsi/sg.h
1309 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1311 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1312 length($sensebuf), 0, length($buf), $buf,
1313 $cmd, $sensebuf, 6000);
1315 $ret = ioctl($fh, $SG_IO, $packet);
1317 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1321 my @res = unpack($sg_io_hdr_t, $packet);
1322 if ($res[17] || $res[18]) {
1323 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1328 (my $byte0, my $byte1, $res->{vendor
},
1329 $res->{product
}, $res->{revision
}) = unpack("C C x6 A8 A16 A4", $buf);
1331 $res->{removable
} = $byte1 & 128 ?
1 : 0;
1332 $res->{type
} = $byte0 & 31;
1340 my $fh = IO
::File-
>new("+<$path") || return undef;
1341 my $res = scsi_inquiry
($fh, 1);
1347 sub machine_type_is_q35
{
1350 return $conf->{machine
} && ($conf->{machine
} =~ m/q35/) ?
1 : 0;
1353 sub print_tabletdevice_full
{
1356 my $q35 = machine_type_is_q35
($conf);
1358 # we use uhci for old VMs because tablet driver was buggy in older qemu
1359 my $usbbus = $q35 ?
"ehci" : "uhci";
1361 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1364 sub print_drivedevice_full
{
1365 my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1370 if ($drive->{interface
} eq 'virtio') {
1371 my $pciaddr = print_pci_addr
("$drive->{interface}$drive->{index}", $bridges);
1372 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1373 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread
};
1374 } elsif ($drive->{interface
} eq 'scsi') {
1376 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1377 my $unit = $drive->{index} % $maxdev;
1378 my $devicetype = 'hd';
1380 if (drive_is_cdrom
($drive)) {
1383 if ($drive->{file
} =~ m
|^/|) {
1384 $path = $drive->{file
};
1385 if (my $info = path_is_scsi
($path)) {
1386 if ($info->{type
} == 0) {
1387 $devicetype = 'block';
1388 } elsif ($info->{type
} == 1) { # tape
1389 $devicetype = 'generic';
1393 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1396 if($path =~ m/^iscsi\:\/\
//){
1397 $devicetype = 'generic';
1401 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)){
1402 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1404 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1407 } elsif ($drive->{interface
} eq 'ide'){
1409 my $controller = int($drive->{index} / $maxdev);
1410 my $unit = $drive->{index} % $maxdev;
1411 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1413 $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1414 if ($devicetype eq 'hd' && (my $model = $drive->{model
})) {
1415 $model = URI
::Escape
::uri_unescape
($model);
1416 $device .= ",model=$model";
1418 } elsif ($drive->{interface
} eq 'sata'){
1419 my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1420 my $unit = $drive->{index} % $MAX_SATA_DISKS;
1421 $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1422 } elsif ($drive->{interface
} eq 'usb') {
1424 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1426 die "unsupported interface type";
1429 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1434 sub get_initiator_name
{
1437 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return undef;
1438 while (defined(my $line = <$fh>)) {
1439 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1448 sub print_drive_full
{
1449 my ($storecfg, $vmid, $drive) = @_;
1452 my $volid = $drive->{file
};
1455 if (drive_is_cdrom
($drive)) {
1456 $path = get_iso_path
($storecfg, $vmid, $volid);
1458 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1460 $path = PVE
::Storage
::path
($storecfg, $volid);
1461 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
1462 $format = qemu_img_format
($scfg, $volname);
1470 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1471 foreach my $o (@qemu_drive_options) {
1472 $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1474 if (my $serial = $drive->{serial
}) {
1475 $serial = URI
::Escape
::uri_unescape
($serial);
1476 $opts .= ",serial=$serial";
1479 $opts .= ",format=$format" if $format && !$drive->{format
};
1481 foreach my $o (qw(bps bps_rd bps_wr)) {
1482 my $v = $drive->{"m$o"};
1483 $opts .= ",$o=" . int($v*1024*1024) if $v;
1486 my $cache_direct = 0;
1488 if (my $cache = $drive->{cache
}) {
1489 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1490 } elsif (!drive_is_cdrom
($drive)) {
1491 $opts .= ",cache=none";
1495 # aio native works only with O_DIRECT
1496 if (!$drive->{aio
}) {
1498 $opts .= ",aio=native";
1500 $opts .= ",aio=threads";
1504 if (!drive_is_cdrom
($drive)) {
1506 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1507 $detectzeroes = 'off';
1508 } elsif ($drive->{discard
}) {
1509 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1511 # This used to be our default with discard not being specified:
1512 $detectzeroes = 'on';
1514 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1517 my $pathinfo = $path ?
"file=$path," : '';
1519 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1522 sub print_netdevice_full
{
1523 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1525 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
1527 my $device = $net->{model
};
1528 if ($net->{model
} eq 'virtio') {
1529 $device = 'virtio-net-pci';
1532 my $pciaddr = print_pci_addr
("$netid", $bridges);
1533 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1534 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1535 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1536 my $vectors = $net->{queues
} * 2 + 2;
1537 $tmpstr .= ",vectors=$vectors,mq=on";
1539 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1541 if ($use_old_bios_files) {
1543 if ($device eq 'virtio-net-pci') {
1544 $romfile = 'pxe-virtio.rom';
1545 } elsif ($device eq 'e1000') {
1546 $romfile = 'pxe-e1000.rom';
1547 } elsif ($device eq 'ne2k') {
1548 $romfile = 'pxe-ne2k_pci.rom';
1549 } elsif ($device eq 'pcnet') {
1550 $romfile = 'pxe-pcnet.rom';
1551 } elsif ($device eq 'rtl8139') {
1552 $romfile = 'pxe-rtl8139.rom';
1554 $tmpstr .= ",romfile=$romfile" if $romfile;
1560 sub print_netdev_full
{
1561 my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1564 if ($netid =~ m/^net(\d+)$/) {
1568 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1570 my $ifname = "tap${vmid}i$i";
1572 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1573 die "interface name '$ifname' is too long (max 15 character)\n"
1574 if length($ifname) >= 16;
1576 my $vhostparam = '';
1577 $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model
} eq 'virtio';
1579 my $vmname = $conf->{name
} || "vm$vmid";
1582 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1584 if ($net->{bridge
}) {
1585 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1587 $netdev = "type=user,id=$netid,hostname=$vmname";
1590 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1595 sub drive_is_cdrom
{
1598 return $drive && $drive->{media
} && ($drive->{media
} eq 'cdrom');
1602 sub parse_number_sets
{
1605 foreach my $part (split(/;/, $set)) {
1606 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1607 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1608 push @$res, [ $1, $2 ];
1610 die "invalid range: $part\n";
1619 my $res = PVE
::JSONSchema
::parse_property_string
($numa_fmt, $data);
1620 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1621 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1628 return undef if !$value;
1630 my $res = PVE
::JSONSchema
::parse_property_string
($hostpci_fmt, $value);
1632 my @idlist = split(/;/, $res->{host
});
1633 delete $res->{host
};
1634 foreach my $id (@idlist) {
1635 if ($id =~ /^$PCIRE$/) {
1636 push @{$res->{pciid
}}, { id
=> $1, function
=> ($2//'0') };
1638 # should have been caught by parse_property_string already
1639 die "failed to parse PCI id: $id\n";
1645 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1649 my $res = eval { PVE
::JSONSchema
::parse_property_string
($net_fmt, $data) };
1654 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
() if !defined($res->{macaddr
});
1661 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1664 sub add_random_macs
{
1665 my ($settings) = @_;
1667 foreach my $opt (keys %$settings) {
1668 next if $opt !~ m/^net(\d+)$/;
1669 my $net = parse_net
($settings->{$opt});
1671 $settings->{$opt} = print_net
($net);
1675 sub vm_is_volid_owner
{
1676 my ($storecfg, $vmid, $volid) = @_;
1678 if ($volid !~ m
|^/|) {
1680 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1681 if ($owner && ($owner == $vmid)) {
1689 sub split_flagged_list
{
1690 my $text = shift || '';
1691 $text =~ s/[,;]/ /g;
1693 return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1696 sub join_flagged_list
{
1697 my ($how, $lst) = @_;
1698 join $how, map { $lst->{$_} . $_ } keys %$lst;
1701 sub vmconfig_delete_pending_option
{
1702 my ($conf, $key, $force) = @_;
1704 delete $conf->{pending
}->{$key};
1705 my $pending_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
1706 $pending_delete_hash->{$key} = $force ?
'!' : '';
1707 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
1710 sub vmconfig_undelete_pending_option
{
1711 my ($conf, $key) = @_;
1713 my $pending_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
1714 delete $pending_delete_hash->{$key};
1716 if (%$pending_delete_hash) {
1717 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
1719 delete $conf->{pending
}->{delete};
1723 sub vmconfig_register_unused_drive
{
1724 my ($storecfg, $vmid, $conf, $drive) = @_;
1726 if (!drive_is_cdrom
($drive)) {
1727 my $volid = $drive->{file
};
1728 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1729 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
1734 sub vmconfig_cleanup_pending
{
1737 # remove pending changes when nothing changed
1739 foreach my $opt (keys %{$conf->{pending
}}) {
1740 if (defined($conf->{$opt}) && ($conf->{pending
}->{$opt} eq $conf->{$opt})) {
1742 delete $conf->{pending
}->{$opt};
1746 my $current_delete_hash = split_flagged_list
($conf->{pending
}->{delete});
1747 my $pending_delete_hash = {};
1748 while (my ($opt, $force) = each %$current_delete_hash) {
1749 if (defined($conf->{$opt})) {
1750 $pending_delete_hash->{$opt} = $force;
1756 if (%$pending_delete_hash) {
1757 $conf->{pending
}->{delete} = join_flagged_list
(',', $pending_delete_hash);
1759 delete $conf->{pending
}->{delete};
1765 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1769 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1770 format_description
=> 'UUID',
1776 format_description
=> 'str',
1782 format_description
=> 'str',
1788 format_description
=> 'name',
1794 format_description
=> 'name',
1800 format_description
=> 'str',
1806 format_description
=> 'str',
1814 my $res = eval { PVE
::JSONSchema
::parse_property_string
($smbios1_fmt, $data) };
1821 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
1824 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
1826 PVE
::JSONSchema
::register_format
('pve-qm-bootdisk', \
&verify_bootdisk
);
1827 sub verify_bootdisk
{
1828 my ($value, $noerr) = @_;
1830 return $value if is_valid_drivename
($value);
1832 return undef if $noerr;
1834 die "invalid boot disk '$value'\n";
1837 PVE
::JSONSchema
::register_format
('pve-qm-net', \
&verify_net
);
1839 my ($value, $noerr) = @_;
1841 return $value if parse_net
($value);
1843 return undef if $noerr;
1845 die "unable to parse network options\n";
1848 sub parse_watchdog
{
1851 return undef if !$value;
1853 my $res = eval { PVE
::JSONSchema
::parse_property_string
($watchdog_fmt, $value) };
1858 sub parse_usb_device
{
1861 return undef if !$value;
1864 if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1865 $res->{vendorid
} = $2;
1866 $res->{productid
} = $4;
1867 } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1868 $res->{hostbus
} = $1;
1869 $res->{hostport
} = $2;
1870 } elsif ($value =~ m/^spice$/i) {
1879 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
1880 sub verify_usb_device
{
1881 my ($value, $noerr) = @_;
1883 return $value if parse_usb_device
($value);
1885 return undef if $noerr;
1887 die "unable to parse usb device\n";
1890 # add JSON properties for create and set function
1891 sub json_config_properties
{
1894 foreach my $opt (keys %$confdesc) {
1895 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1896 $prop->{$opt} = $confdesc->{$opt};
1903 my ($key, $value) = @_;
1905 die "unknown setting '$key'\n" if !$confdesc->{$key};
1907 my $type = $confdesc->{$key}->{type
};
1909 if (!defined($value)) {
1910 die "got undefined value\n";
1913 if ($value =~ m/[\n\r]/) {
1914 die "property contains a line feed\n";
1917 if ($type eq 'boolean') {
1918 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1919 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1920 die "type check ('boolean') failed - got '$value'\n";
1921 } elsif ($type eq 'integer') {
1922 return int($1) if $value =~ m/^(\d+)$/;
1923 die "type check ('integer') failed - got '$value'\n";
1924 } elsif ($type eq 'number') {
1925 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1926 die "type check ('number') failed - got '$value'\n";
1927 } elsif ($type eq 'string') {
1928 if (my $fmt = $confdesc->{$key}->{format
}) {
1929 if ($fmt eq 'pve-qm-drive') {
1930 # special case - we need to pass $key to parse_drive()
1931 my $drive = parse_drive
($key, $value);
1932 return $value if $drive;
1933 die "unable to parse drive options\n";
1935 PVE
::JSONSchema
::check_format
($fmt, $value);
1938 $value =~ s/^\"(.*)\"$/$1/;
1941 die "internal error"
1945 sub check_iommu_support
{
1946 #fixme : need to check IOMMU support
1947 #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1957 my $conf = PVE
::QemuConfig-
>config_file($vmid);
1958 utime undef, undef, $conf;
1962 my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1964 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
1966 my $conf = PVE
::QemuConfig-
>load_config($vmid);
1968 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
1970 # only remove disks owned by this VM
1971 foreach_drive
($conf, sub {
1972 my ($ds, $drive) = @_;
1974 return if drive_is_cdrom
($drive);
1976 my $volid = $drive->{file
};
1978 return if !$volid || $volid =~ m
|^/|;
1980 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
1981 return if !$path || !$owner || ($owner != $vmid);
1983 PVE
::Storage
::vdisk_free
($storecfg, $volid);
1986 if ($keep_empty_config) {
1987 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
1992 # also remove unused disk
1994 my $dl = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid);
1997 PVE
::Storage
::foreach_volid
($dl, sub {
1998 my ($volid, $sid, $volname, $d) = @_;
1999 PVE
::Storage
::vdisk_free
($storecfg, $volid);
2008 sub parse_vm_config
{
2009 my ($filename, $raw) = @_;
2011 return undef if !defined($raw);
2014 digest
=> Digest
::SHA
::sha1_hex
($raw),
2019 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2020 || die "got strange filename '$filename'";
2028 my @lines = split(/\n/, $raw);
2029 foreach my $line (@lines) {
2030 next if $line =~ m/^\s*$/;
2032 if ($line =~ m/^\[PENDING\]\s*$/i) {
2033 $section = 'pending';
2034 if (defined($descr)) {
2036 $conf->{description
} = $descr;
2039 $conf = $res->{$section} = {};
2042 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2044 if (defined($descr)) {
2046 $conf->{description
} = $descr;
2049 $conf = $res->{snapshots
}->{$section} = {};
2053 if ($line =~ m/^\#(.*)\s*$/) {
2054 $descr = '' if !defined($descr);
2055 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2059 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2060 $descr = '' if !defined($descr);
2061 $descr .= PVE
::Tools
::decode_text
($2);
2062 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2063 $conf->{snapstate
} = $1;
2064 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2067 $conf->{$key} = $value;
2068 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2070 if ($section eq 'pending') {
2071 $conf->{delete} = $value; # we parse this later
2073 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2075 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2078 eval { $value = check_type
($key, $value); };
2080 warn "vm $vmid - unable to parse value of '$key' - $@";
2082 my $fmt = $confdesc->{$key}->{format
};
2083 if ($fmt && $fmt eq 'pve-qm-drive') {
2084 my $v = parse_drive
($key, $value);
2085 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2086 $v->{file
} = $volid;
2087 $value = print_drive
($vmid, $v);
2089 warn "vm $vmid - unable to parse value of '$key'\n";
2094 if ($key eq 'cdrom') {
2095 $conf->{ide2
} = $value;
2097 $conf->{$key} = $value;
2103 if (defined($descr)) {
2105 $conf->{description
} = $descr;
2107 delete $res->{snapstate
}; # just to be sure
2112 sub write_vm_config
{
2113 my ($filename, $conf) = @_;
2115 delete $conf->{snapstate
}; # just to be sure
2117 if ($conf->{cdrom
}) {
2118 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2119 $conf->{ide2
} = $conf->{cdrom
};
2120 delete $conf->{cdrom
};
2123 # we do not use 'smp' any longer
2124 if ($conf->{sockets
}) {
2125 delete $conf->{smp
};
2126 } elsif ($conf->{smp
}) {
2127 $conf->{sockets
} = $conf->{smp
};
2128 delete $conf->{cores
};
2129 delete $conf->{smp
};
2132 my $used_volids = {};
2134 my $cleanup_config = sub {
2135 my ($cref, $pending, $snapname) = @_;
2137 foreach my $key (keys %$cref) {
2138 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2139 $key eq 'snapstate' || $key eq 'pending';
2140 my $value = $cref->{$key};
2141 if ($key eq 'delete') {
2142 die "propertry 'delete' is only allowed in [PENDING]\n"
2144 # fixme: check syntax?
2147 eval { $value = check_type
($key, $value); };
2148 die "unable to parse value of '$key' - $@" if $@;
2150 $cref->{$key} = $value;
2152 if (!$snapname && is_valid_drivename
($key)) {
2153 my $drive = parse_drive
($key, $value);
2154 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2159 &$cleanup_config($conf);
2161 &$cleanup_config($conf->{pending
}, 1);
2163 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2164 die "internal error" if $snapname eq 'pending';
2165 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2168 # remove 'unusedX' settings if we re-add a volume
2169 foreach my $key (keys %$conf) {
2170 my $value = $conf->{$key};
2171 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2172 delete $conf->{$key};
2176 my $generate_raw_config = sub {
2177 my ($conf, $pending) = @_;
2181 # add description as comment to top of file
2182 if (defined(my $descr = $conf->{description
})) {
2184 foreach my $cl (split(/\n/, $descr)) {
2185 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2188 $raw .= "#\n" if $pending;
2192 foreach my $key (sort keys %$conf) {
2193 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2194 $raw .= "$key: $conf->{$key}\n";
2199 my $raw = &$generate_raw_config($conf);
2201 if (scalar(keys %{$conf->{pending
}})){
2202 $raw .= "\n[PENDING]\n";
2203 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2206 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2207 $raw .= "\n[$snapname]\n";
2208 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2218 # we use static defaults from our JSON schema configuration
2219 foreach my $key (keys %$confdesc) {
2220 if (defined(my $default = $confdesc->{$key}->{default})) {
2221 $res->{$key} = $default;
2225 my $conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
2226 $res->{keyboard
} = $conf->{keyboard
} if $conf->{keyboard
};
2232 my $vmlist = PVE
::Cluster
::get_vmlist
();
2234 return $res if !$vmlist || !$vmlist->{ids
};
2235 my $ids = $vmlist->{ids
};
2237 foreach my $vmid (keys %$ids) {
2238 my $d = $ids->{$vmid};
2239 next if !$d->{node
} || $d->{node
} ne $nodename;
2240 next if !$d->{type
} || $d->{type
} ne 'qemu';
2241 $res->{$vmid}->{exists} = 1;
2246 # test if VM uses local resources (to prevent migration)
2247 sub check_local_resources
{
2248 my ($conf, $noerr) = @_;
2252 $loc_res = 1 if $conf->{hostusb
}; # old syntax
2253 $loc_res = 1 if $conf->{hostpci
}; # old syntax
2255 foreach my $k (keys %$conf) {
2256 next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2257 # sockets are safe: they will recreated be on the target side post-migrate
2258 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2259 $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2262 die "VM uses local resources\n" if $loc_res && !$noerr;
2267 # check if used storages are available on all nodes (use by migrate)
2268 sub check_storage_availability
{
2269 my ($storecfg, $conf, $node) = @_;
2271 foreach_drive
($conf, sub {
2272 my ($ds, $drive) = @_;
2274 my $volid = $drive->{file
};
2277 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2280 # check if storage is available on both nodes
2281 my $scfg = PVE
::Storage
::storage_check_node
($storecfg, $sid);
2282 PVE
::Storage
::storage_check_node
($storecfg, $sid, $node);
2286 # list nodes where all VM images are available (used by has_feature API)
2288 my ($conf, $storecfg) = @_;
2290 my $nodelist = PVE
::Cluster
::get_nodelist
();
2291 my $nodehash = { map { $_ => 1 } @$nodelist };
2292 my $nodename = PVE
::INotify
::nodename
();
2294 foreach_drive
($conf, sub {
2295 my ($ds, $drive) = @_;
2297 my $volid = $drive->{file
};
2300 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2302 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2303 if ($scfg->{disable
}) {
2305 } elsif (my $avail = $scfg->{nodes
}) {
2306 foreach my $node (keys %$nodehash) {
2307 delete $nodehash->{$node} if !$avail->{$node};
2309 } elsif (!$scfg->{shared
}) {
2310 foreach my $node (keys %$nodehash) {
2311 delete $nodehash->{$node} if $node ne $nodename
2321 my ($pidfile, $pid) = @_;
2323 my $fh = IO
::File-
>new("/proc/$pid/cmdline", "r");
2327 return undef if !$line;
2328 my @param = split(/\0/, $line);
2330 my $cmd = $param[0];
2331 return if !$cmd || ($cmd !~ m
|kvm
$| && $cmd !~ m
|qemu-system-x86_64
$|);
2333 for (my $i = 0; $i < scalar (@param); $i++) {
2336 if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2337 my $p = $param[$i+1];
2338 return 1 if $p && ($p eq $pidfile);
2347 my ($vmid, $nocheck, $node) = @_;
2349 my $filename = PVE
::QemuConfig-
>config_file($vmid, $node);
2351 die "unable to find configuration file for VM $vmid - no such machine\n"
2352 if !$nocheck && ! -f
$filename;
2354 my $pidfile = pidfile_name
($vmid);
2356 if (my $fd = IO
::File-
>new("<$pidfile")) {
2361 my $mtime = $st->mtime;
2362 if ($mtime > time()) {
2363 warn "file '$filename' modified in future\n";
2366 if ($line =~ m/^(\d+)$/) {
2368 if (check_cmdline
($pidfile, $pid)) {
2369 if (my $pinfo = PVE
::ProcFSTools
::check_process_running
($pid)) {
2381 my $vzlist = config_list
();
2383 my $fd = IO
::Dir-
>new($var_run_tmpdir) || return $vzlist;
2385 while (defined(my $de = $fd->read)) {
2386 next if $de !~ m/^(\d+)\.pid$/;
2388 next if !defined($vzlist->{$vmid});
2389 if (my $pid = check_running
($vmid)) {
2390 $vzlist->{$vmid}->{pid
} = $pid;
2398 my ($storecfg, $conf) = @_;
2400 my $bootdisk = $conf->{bootdisk
};
2401 return undef if !$bootdisk;
2402 return undef if !is_valid_drivename
($bootdisk);
2404 return undef if !$conf->{$bootdisk};
2406 my $drive = parse_drive
($bootdisk, $conf->{$bootdisk});
2407 return undef if !defined($drive);
2409 return undef if drive_is_cdrom
($drive);
2411 my $volid = $drive->{file
};
2412 return undef if !$volid;
2414 return $drive->{size
};
2417 my $last_proc_pid_stat;
2419 # get VM status information
2420 # This must be fast and should not block ($full == false)
2421 # We only query KVM using QMP if $full == true (this can be slow)
2423 my ($opt_vmid, $full) = @_;
2427 my $storecfg = PVE
::Storage
::config
();
2429 my $list = vzlist
();
2430 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2432 my $cpucount = $cpuinfo->{cpus
} || 1;
2434 foreach my $vmid (keys %$list) {
2435 next if $opt_vmid && ($vmid ne $opt_vmid);
2437 my $cfspath = PVE
::QemuConfig-
>cfs_config_path($vmid);
2438 my $conf = PVE
::Cluster
::cfs_read_file
($cfspath) || {};
2441 $d->{pid
} = $list->{$vmid}->{pid
};
2443 # fixme: better status?
2444 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2446 my $size = disksize
($storecfg, $conf);
2447 if (defined($size)) {
2448 $d->{disk
} = 0; # no info available
2449 $d->{maxdisk
} = $size;
2455 $d->{cpus
} = ($conf->{sockets
} || 1) * ($conf->{cores
} || 1);
2456 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2457 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2459 $d->{name
} = $conf->{name
} || "VM $vmid";
2460 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024) : 0;
2462 if ($conf->{balloon
}) {
2463 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2464 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
} : 1000;
2475 $d->{diskwrite
} = 0;
2477 $d->{template
} = PVE
::QemuConfig-
>is_template($conf);
2482 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2483 foreach my $dev (keys %$netdev) {
2484 next if $dev !~ m/^tap([1-9]\d*)i/;
2486 my $d = $res->{$vmid};
2489 $d->{netout
} += $netdev->{$dev}->{receive
};
2490 $d->{netin
} += $netdev->{$dev}->{transmit
};
2493 $d->{nics
}->{$dev}->{netout
} = $netdev->{$dev}->{receive
};
2494 $d->{nics
}->{$dev}->{netin
} = $netdev->{$dev}->{transmit
};
2499 my $ctime = gettimeofday
;
2501 foreach my $vmid (keys %$list) {
2503 my $d = $res->{$vmid};
2504 my $pid = $d->{pid
};
2507 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2508 next if !$pstat; # not running
2510 my $used = $pstat->{utime} + $pstat->{stime
};
2512 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2514 if ($pstat->{vsize
}) {
2515 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2518 my $old = $last_proc_pid_stat->{$pid};
2520 $last_proc_pid_stat->{$pid} = {
2528 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2530 if ($dtime > 1000) {
2531 my $dutime = $used - $old->{used
};
2533 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2534 $last_proc_pid_stat->{$pid} = {
2540 $d->{cpu
} = $old->{cpu
};
2544 return $res if !$full;
2546 my $qmpclient = PVE
::QMPClient-
>new();
2548 my $ballooncb = sub {
2549 my ($vmid, $resp) = @_;
2551 my $info = $resp->{'return'};
2552 return if !$info->{max_mem
};
2554 my $d = $res->{$vmid};
2556 # use memory assigned to VM
2557 $d->{maxmem
} = $info->{max_mem
};
2558 $d->{balloon
} = $info->{actual
};
2560 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2561 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2562 $d->{freemem
} = $info->{free_mem
};
2565 $d->{ballooninfo
} = $info;
2568 my $blockstatscb = sub {
2569 my ($vmid, $resp) = @_;
2570 my $data = $resp->{'return'} || [];
2571 my $totalrdbytes = 0;
2572 my $totalwrbytes = 0;
2574 for my $blockstat (@$data) {
2575 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2576 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2578 $blockstat->{device
} =~ s/drive-//;
2579 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2581 $res->{$vmid}->{diskread
} = $totalrdbytes;
2582 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2585 my $statuscb = sub {
2586 my ($vmid, $resp) = @_;
2588 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2589 # this fails if ballon driver is not loaded, so this must be
2590 # the last commnand (following command are aborted if this fails).
2591 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2593 my $status = 'unknown';
2594 if (!defined($status = $resp->{'return'}->{status
})) {
2595 warn "unable to get VM status\n";
2599 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
2602 foreach my $vmid (keys %$list) {
2603 next if $opt_vmid && ($vmid ne $opt_vmid);
2604 next if !$res->{$vmid}->{pid
}; # not running
2605 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2608 $qmpclient->queue_execute(undef, 1);
2610 foreach my $vmid (keys %$list) {
2611 next if $opt_vmid && ($vmid ne $opt_vmid);
2612 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
2619 my ($conf, $vmid, $memory, $sockets, $func) = @_;
2622 my $current_size = 1024;
2623 my $dimm_size = 512;
2624 return if $current_size == $memory;
2626 for (my $j = 0; $j < 8; $j++) {
2627 for (my $i = 0; $i < 32; $i++) {
2628 my $name = "dimm${dimm_id}";
2630 my $numanode = $i % $sockets;
2631 $current_size += $dimm_size;
2632 &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2633 return $current_size if $current_size >= $memory;
2639 sub foreach_reverse_dimm
{
2640 my ($conf, $vmid, $memory, $sockets, $func) = @_;
2643 my $current_size = 4177920;
2644 my $dimm_size = 65536;
2645 return if $current_size == $memory;
2647 for (my $j = 0; $j < 8; $j++) {
2648 for (my $i = 0; $i < 32; $i++) {
2649 my $name = "dimm${dimm_id}";
2651 my $numanode = $i % $sockets;
2652 $current_size -= $dimm_size;
2653 &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2654 return $current_size if $current_size <= $memory;
2661 my ($conf, $func) = @_;
2663 foreach my $ds (valid_drive_names
()) {
2664 next if !defined($conf->{$ds});
2666 my $drive = parse_drive
($ds, $conf->{$ds});
2669 &$func($ds, $drive);
2674 my ($conf, $func) = @_;
2678 my $test_volid = sub {
2679 my ($volid, $is_cdrom) = @_;
2683 $volhash->{$volid} = $is_cdrom || 0;
2686 foreach_drive
($conf, sub {
2687 my ($ds, $drive) = @_;
2688 &$test_volid($drive->{file
}, drive_is_cdrom
($drive));
2691 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2692 my $snap = $conf->{snapshots
}->{$snapname};
2693 &$test_volid($snap->{vmstate
}, 0);
2694 foreach_drive
($snap, sub {
2695 my ($ds, $drive) = @_;
2696 &$test_volid($drive->{file
}, drive_is_cdrom
($drive));
2700 foreach my $volid (keys %$volhash) {
2701 &$func($volid, $volhash->{$volid});
2705 sub vga_conf_has_spice
{
2708 return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2713 sub config_to_command
{
2714 my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2717 my $globalFlags = [];
2718 my $machineFlags = [];
2724 my $kvmver = kvm_user_version
();
2725 my $vernum = 0; # unknown
2726 my $ostype = $conf->{ostype
};
2727 if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2728 $vernum = $1*1000000+$2*1000;
2729 } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2730 $vernum = $1*1000000+$2*1000+$3;
2733 die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2735 my $have_ovz = -f
'/proc/vz/vestat';
2737 my $q35 = machine_type_is_q35
($conf);
2738 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
2739 my $machine_type = $forcemachine || $conf->{machine
};
2740 my $use_old_bios_files = undef;
2741 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
2743 my $cpuunits = defined($conf->{cpuunits
}) ?
2744 $conf->{cpuunits
} : $defaults->{cpuunits
};
2746 push @$cmd, '/usr/bin/systemd-run';
2747 push @$cmd, '--scope';
2748 push @$cmd, '--slice', "qemu";
2749 push @$cmd, '--unit', $vmid;
2750 # set KillMode=none, so that systemd don't kill those scopes
2751 # at shutdown (pve-manager service should stop the VMs instead)
2752 push @$cmd, '-p', "KillMode=none";
2753 push @$cmd, '-p', "CPUShares=$cpuunits";
2754 if ($conf->{cpulimit
}) {
2755 my $cpulimit = int($conf->{cpulimit
} * 100);
2756 push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2759 push @$cmd, '/usr/bin/kvm';
2761 push @$cmd, '-id', $vmid;
2765 my $qmpsocket = qmp_socket
($vmid);
2766 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2767 push @$cmd, '-mon', "chardev=qmp,mode=control";
2770 push @$cmd, '-pidfile' , pidfile_name
($vmid);
2772 push @$cmd, '-daemonize';
2774 if ($conf->{smbios1
}) {
2775 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2778 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
2779 my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2780 my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2781 my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2782 PVE
::Tools
::file_copy
($ovmfvar_src, $ovmfvar_dst, 256*1024);
2783 push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2784 push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2788 # the q35 chipset support native usb2, so we enable usb controller
2789 # by default for this machine type
2790 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2792 $pciaddr = print_pci_addr
("piix3", $bridges);
2793 push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2796 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
2797 next if !$conf->{"usb$i"};
2798 my $d = eval { PVE
::JSONSchema
::parse_property_string
($usbdesc->{format
},$conf->{"usb$i"}) };
2799 next if !$d || $d->{usb3
}; # do not add usb2 controller if we have only usb3 devices
2802 # include usb device config
2803 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2806 # add usb3 controller if needed
2809 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
2810 next if !$conf->{"usb$i"};
2811 my $d = eval { PVE
::JSONSchema
::parse_property_string
($usbdesc->{format
},$conf->{"usb$i"}) };
2812 next if !$d || !$d->{usb3
};
2816 $pciaddr = print_pci_addr
("xhci", $bridges);
2817 push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2819 my $vga = $conf->{vga
};
2821 my $qxlnum = vga_conf_has_spice
($vga);
2822 $vga = 'qxl' if $qxlnum;
2825 if ($conf->{ostype
} && ($conf->{ostype
} eq 'win8' ||
2826 $conf->{ostype
} eq 'win7' ||
2827 $conf->{ostype
} eq 'w2k8')) {
2834 # enable absolute mouse coordinates (needed by vnc)
2836 if (defined($conf->{tablet
})) {
2837 $tablet = $conf->{tablet
};
2839 $tablet = $defaults->{tablet
};
2840 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2841 $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2844 push @$devices, '-device', print_tabletdevice_full
($conf) if $tablet;
2848 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
2849 my $d = parse_hostpci
($conf->{"hostpci$i"});
2852 my $pcie = $d->{pcie
};
2854 die "q35 machine model is not enabled" if !$q35;
2855 $pciaddr = print_pcie_addr
("hostpci$i");
2857 $pciaddr = print_pci_addr
("hostpci$i", $bridges);
2860 my $rombar = defined($d->{rombar
}) && !$d->{rombar
} ?
',rombar=0' : '';
2862 if ($d->{'x-vga'}) {
2863 $xvga = ',x-vga=on';
2866 if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2867 push @$cpuFlags , 'hv_vendor_id=proxmox';
2869 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
2873 my $pcidevices = $d->{pciid
};
2874 my $multifunction = 1 if @$pcidevices > 1;
2877 foreach my $pcidevice (@$pcidevices) {
2879 my $id = "hostpci$i";
2880 $id .= ".$j" if $multifunction;
2881 my $addr = $pciaddr;
2882 $addr .= ".$j" if $multifunction;
2883 my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2886 $devicestr .= "$rombar$xvga";
2887 $devicestr .= ",multifunction=on" if $multifunction;
2890 push @$devices, '-device', $devicestr;
2896 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
2897 next if !$conf->{"usb$i"};
2898 my $d = eval { PVE
::JSONSchema
::parse_property_string
($usbdesc->{format
},$conf->{"usb$i"}) };
2901 # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2903 if (defined($d->{usb3
}) && $d->{usb3
}) {
2904 $usbbus = ',bus=xhci.0';
2907 if (defined($d->{host
})) {
2908 $d = parse_usb_device
($d->{host
});
2909 if (defined($d->{vendorid
}) && defined($d->{productid
})) {
2910 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2911 } elsif (defined($d->{hostbus
}) && defined($d->{hostport
})) {
2912 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2913 } elsif (defined($d->{spice
}) && $d->{spice
}) {
2914 # usb redir support for spice, currently no usb3
2915 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2916 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2922 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2923 if (my $path = $conf->{"serial$i"}) {
2924 if ($path eq 'socket') {
2925 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2926 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2927 push @$devices, '-device', "isa-serial,chardev=serial$i";
2929 die "no such serial device\n" if ! -c
$path;
2930 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2931 push @$devices, '-device', "isa-serial,chardev=serial$i";
2937 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
2938 if (my $path = $conf->{"parallel$i"}) {
2939 die "no such parallel device\n" if ! -c
$path;
2940 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
2941 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2942 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2946 my $vmname = $conf->{name
} || "vm$vmid";
2948 push @$cmd, '-name', $vmname;
2951 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
2952 $sockets = $conf->{sockets
} if $conf->{sockets
};
2954 my $cores = $conf->{cores
} || 1;
2956 my $maxcpus = $sockets * $cores;
2958 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
2960 my $allowed_vcpus = $cpuinfo->{cpus
};
2962 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2963 if ($allowed_vcpus < $maxcpus);
2965 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2967 push @$cmd, '-nodefaults';
2969 my $bootorder = $conf->{boot
} || $confdesc->{boot
}->{default};
2971 my $bootindex_hash = {};
2973 foreach my $o (split(//, $bootorder)) {
2974 $bootindex_hash->{$o} = $i*100;
2978 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2980 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
2982 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
2984 push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2986 if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2987 my $socket = vnc_socket
($vmid);
2988 push @$cmd, '-vnc', "unix:$socket,x509,password";
2990 push @$cmd, '-nographic';
2994 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
2996 my $nokvm = defined($conf->{kvm
}) && $conf->{kvm
} == 0 ?
1 : 0;
2997 my $useLocaltime = $conf->{localtime};
3000 # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3002 if ($ostype =~ m/^w/) { # windows
3003 $useLocaltime = 1 if !defined($conf->{localtime});
3005 # use time drift fix when acpi is enabled
3006 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3007 $tdf = 1 if !defined($conf->{tdf
});
3011 if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3012 $ostype eq 'wvista') {
3013 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3014 push @$cmd, '-no-hpet';
3015 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3)) {
3016 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3017 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3018 push @$cpuFlags , 'hv_time' if !$nokvm;
3021 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3025 if ($ostype eq 'win7' || $ostype eq 'win8') {
3026 push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3030 push @$rtcFlags, 'driftfix=slew' if $tdf;
3033 push @$machineFlags, 'accel=tcg';
3035 die "No accelerator found!\n" if !$cpuinfo->{hvm
};
3038 if ($machine_type) {
3039 push @$machineFlags, "type=${machine_type}";
3042 if ($conf->{startdate
}) {
3043 push @$rtcFlags, "base=$conf->{startdate}";
3044 } elsif ($useLocaltime) {
3045 push @$rtcFlags, 'base=localtime';
3048 my $cpu = $nokvm ?
"qemu64" : "kvm64";
3049 if (my $cputype = $conf->{cpu
}) {
3050 my $cpuconf = PVE
::JSONSchema
::parse_property_string
($cpu_fmt, $cputype)
3051 or die "Cannot parse cpu description: $cputype\n";
3052 $cpu = $cpuconf->{cputype
};
3053 $kvm_off = 1 if $cpuconf->{hidden
};
3056 push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3058 push @$cpuFlags , '-x2apic'
3059 if $conf->{ostype
} && $conf->{ostype
} eq 'solaris';
3061 push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3063 push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3065 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3)) {
3067 push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3068 push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3071 push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3073 push @$cpuFlags, 'kvm=off' if $kvm_off;
3075 $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3077 push @$cmd, '-cpu', $cpu;
3079 my $memory = $conf->{memory
} || $defaults->{memory
};
3080 my $static_memory = 0;
3081 my $dimm_memory = 0;
3083 if ($hotplug_features->{memory
}) {
3084 die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa
};
3085 die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
3086 $static_memory = $STATICMEM;
3087 die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
3088 $dimm_memory = $memory - $static_memory;
3089 push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
3093 $static_memory = $memory;
3094 push @$cmd, '-m', $static_memory;
3097 if ($conf->{numa
}) {
3099 my $numa_totalmemory = undef;
3100 for (my $i = 0; $i < $MAX_NUMA; $i++) {
3101 next if !$conf->{"numa$i"};
3102 my $numa = parse_numa
($conf->{"numa$i"});
3105 die "missing numa node$i memory value\n" if !$numa->{memory
};
3106 my $numa_memory = $numa->{memory
};
3107 $numa_totalmemory += $numa_memory;
3108 my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
3111 my $cpulists = $numa->{cpus
};
3112 die "missing numa node$i cpus\n" if !defined($cpulists);
3113 my $cpus = join(',', map {
3114 my ($start, $end) = @$_;
3115 defined($end) ?
"$start-$end" : $start
3119 my $hostnodelists = $numa->{hostnodes
};
3120 if (defined($hostnodelists)) {
3122 foreach my $hostnoderange (@$hostnodelists) {
3123 my ($start, $end) = @$hostnoderange;
3124 $hostnodes .= ',' if $hostnodes;
3125 $hostnodes .= $start;
3126 $hostnodes .= "-$end" if defined($end);
3128 for (my $i = $start; $i <= $end; ++$i ) {
3129 die "host numa node$i don't exist\n" if ! -d
"/sys/devices/system/node/node$i/";
3134 my $policy = $numa->{policy
};
3135 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3136 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3139 push @$cmd, '-object', $numa_object;
3140 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3143 die "total memory for NUMA nodes must be equal to vm static memory\n"
3144 if $numa_totalmemory && $numa_totalmemory != $static_memory;
3146 #if no custom tology, we split memory and cores across numa nodes
3147 if(!$numa_totalmemory) {
3149 my $numa_memory = ($static_memory / $sockets) . "M";
3151 for (my $i = 0; $i < $sockets; $i++) {
3153 my $cpustart = ($cores * $i);
3154 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3155 my $cpus = $cpustart;
3156 $cpus .= "-$cpuend" if $cpuend;
3158 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3159 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3164 if ($hotplug_features->{memory
}) {
3165 foreach_dimm
($conf, $vmid, $memory, $sockets, sub {
3166 my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3167 push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3168 push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3170 #if dimm_memory is not aligned to dimm map
3171 if($current_size > $memory) {
3172 $conf->{memory
} = $current_size;
3173 PVE
::QemuConfig-
>write_config($vmid, $conf);
3178 push @$cmd, '-S' if $conf->{freeze
};
3180 # set keyboard layout
3181 my $kb = $conf->{keyboard
} || $defaults->{keyboard
};
3182 push @$cmd, '-k', $kb if $kb;
3185 #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3186 #push @$cmd, '-soundhw', 'es1370';
3187 #push @$cmd, '-soundhw', $soundhw if $soundhw;
3189 if($conf->{agent
}) {
3190 my $qgasocket = qmp_socket
($vmid, 1);
3191 my $pciaddr = print_pci_addr
("qga0", $bridges);
3192 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3193 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3194 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3201 if ($conf->{ostype
} && $conf->{ostype
} =~ m/^w/){
3202 for(my $i = 1; $i < $qxlnum; $i++){
3203 my $pciaddr = print_pci_addr
("vga$i", $bridges);
3204 push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3207 # assume other OS works like Linux
3208 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3209 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3213 my $pciaddr = print_pci_addr
("spice", $bridges);
3215 my $nodename = PVE
::INotify
::nodename
();
3216 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3217 $spice_port = PVE
::Tools
::next_spice_port
($pfamily);
3219 push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3221 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3222 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3223 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3226 # enable balloon by default, unless explicitly disabled
3227 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3228 $pciaddr = print_pci_addr
("balloon0", $bridges);
3229 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3232 if ($conf->{watchdog
}) {
3233 my $wdopts = parse_watchdog
($conf->{watchdog
});
3234 $pciaddr = print_pci_addr
("watchdog", $bridges);
3235 my $watchdog = $wdopts->{model
} || 'i6300esb';
3236 push @$devices, '-device', "$watchdog$pciaddr";
3237 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3241 my $scsicontroller = {};
3242 my $ahcicontroller = {};
3243 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3245 # Add iscsi initiator name if available
3246 if (my $initiator = get_initiator_name
()) {
3247 push @$devices, '-iscsi', "initiator-name=$initiator";
3250 foreach_drive
($conf, sub {
3251 my ($ds, $drive) = @_;
3253 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3254 push @$vollist, $drive->{file
};
3257 $use_virtio = 1 if $ds =~ m/^virtio/;
3259 if (drive_is_cdrom
($drive)) {
3260 if ($bootindex_hash->{d
}) {
3261 $drive->{bootindex
} = $bootindex_hash->{d
};
3262 $bootindex_hash->{d
} += 1;
3265 if ($bootindex_hash->{c
}) {
3266 $drive->{bootindex
} = $bootindex_hash->{c
} if $conf->{bootdisk
} && ($conf->{bootdisk
} eq $ds);
3267 $bootindex_hash->{c
} += 1;
3271 if($drive->{interface
} eq 'virtio'){
3272 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3275 if ($drive->{interface
} eq 'scsi') {
3277 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3279 $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges);
3280 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3283 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3284 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3285 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3289 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3290 $queues = ",num_queues=$drive->{queues}";
3293 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3294 $scsicontroller->{$controller}=1;
3297 if ($drive->{interface
} eq 'sata') {
3298 my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3299 $pciaddr = print_pci_addr
("ahci$controller", $bridges);
3300 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3301 $ahcicontroller->{$controller}=1;
3304 my $drive_cmd = print_drive_full
($storecfg, $vmid, $drive);
3305 push @$devices, '-drive',$drive_cmd;
3306 push @$devices, '-device', print_drivedevice_full
($storecfg, $conf, $vmid, $drive, $bridges);
3309 for (my $i = 0; $i < $MAX_NETS; $i++) {
3310 next if !$conf->{"net$i"};
3311 my $d = parse_net
($conf->{"net$i"});
3314 $use_virtio = 1 if $d->{model
} eq 'virtio';
3316 if ($bootindex_hash->{n
}) {
3317 $d->{bootindex
} = $bootindex_hash->{n
};
3318 $bootindex_hash->{n
} += 1;
3321 my $netdevfull = print_netdev_full
($vmid,$conf,$d,"net$i");
3322 push @$devices, '-netdev', $netdevfull;
3324 my $netdevicefull = print_netdevice_full
($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3325 push @$devices, '-device', $netdevicefull;
3330 if (qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 3)) {
3335 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3337 while (my ($k, $v) = each %$bridges) {
3338 $pciaddr = print_pci_addr
("pci.$k");
3339 unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3344 if ($conf->{args
}) {
3345 my $aa = PVE
::Tools
::split_args
($conf->{args
});
3349 push @$cmd, @$devices;
3350 push @$cmd, '-rtc', join(',', @$rtcFlags)
3351 if scalar(@$rtcFlags);
3352 push @$cmd, '-machine', join(',', @$machineFlags)
3353 if scalar(@$machineFlags);
3354 push @$cmd, '-global', join(',', @$globalFlags)
3355 if scalar(@$globalFlags);
3357 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
3362 return "${var_run_tmpdir}/$vmid.vnc";
3368 my $res = vm_mon_cmd
($vmid, 'query-spice');
3370 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3374 my ($vmid, $qga) = @_;
3375 my $sockettype = $qga ?
'qga' : 'qmp';
3376 return "${var_run_tmpdir}/$vmid.$sockettype";
3381 return "${var_run_tmpdir}/$vmid.pid";
3384 sub vm_devices_list
{
3387 my $res = vm_mon_cmd
($vmid, 'query-pci');
3389 foreach my $pcibus (@$res) {
3390 foreach my $device (@{$pcibus->{devices
}}) {
3391 next if !$device->{'qdev_id'};
3392 if ($device->{'pci_bridge'}) {
3393 $devices->{$device->{'qdev_id'}} = 1;
3394 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices
}}) {
3395 next if !$bridge_device->{'qdev_id'};
3396 $devices->{$bridge_device->{'qdev_id'}} = 1;
3397 $devices->{$device->{'qdev_id'}}++;
3400 $devices->{$device->{'qdev_id'}} = 1;
3405 my $resblock = vm_mon_cmd
($vmid, 'query-block');
3406 foreach my $block (@$resblock) {
3407 if($block->{device
} =~ m/^drive-(\S+)/){
3412 my $resmice = vm_mon_cmd
($vmid, 'query-mice');
3413 foreach my $mice (@$resmice) {
3414 if ($mice->{name
} eq 'QEMU HID Tablet') {
3415 $devices->{tablet
} = 1;
3424 my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3426 my $q35 = machine_type_is_q35
($conf);
3428 my $devices_list = vm_devices_list
($vmid);
3429 return 1 if defined($devices_list->{$deviceid});
3431 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3433 if ($deviceid eq 'tablet') {
3435 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf));
3437 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3439 qemu_iothread_add
($vmid, $deviceid, $device);
3441 qemu_driveadd
($storecfg, $vmid, $device);
3442 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device);
3444 qemu_deviceadd
($vmid, $devicefull);
3445 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3447 eval { qemu_drivedel
($vmid, $deviceid); };
3452 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3455 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
3456 my $pciaddr = print_pci_addr
($deviceid);
3457 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
3459 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3461 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
3462 qemu_iothread_add
($vmid, $deviceid, $device);
3463 $devicefull .= ",iothread=iothread-$deviceid";
3466 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
3467 $devicefull .= ",num_queues=$device->{queues}";
3470 qemu_deviceadd
($vmid, $devicefull);
3471 qemu_deviceaddverify
($vmid, $deviceid);
3473 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3475 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device);
3476 qemu_driveadd
($storecfg, $vmid, $device);
3478 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device);
3479 eval { qemu_deviceadd
($vmid, $devicefull); };
3481 eval { qemu_drivedel
($vmid, $deviceid); };
3486 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3488 return undef if !qemu_netdevadd
($vmid, $conf, $device, $deviceid);
3490 my $machine_type = PVE
::QemuServer
::qemu_machine_pxe
($vmid, $conf);
3491 my $use_old_bios_files = undef;
3492 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3494 my $netdevicefull = print_netdevice_full
($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3495 qemu_deviceadd
($vmid, $netdevicefull);
3496 eval { qemu_deviceaddverify
($vmid, $deviceid); };
3498 eval { qemu_netdevdel
($vmid, $deviceid); };
3503 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3506 my $pciaddr = print_pci_addr
($deviceid);
3507 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3509 qemu_deviceadd
($vmid, $devicefull);
3510 qemu_deviceaddverify
($vmid, $deviceid);
3513 die "can't hotplug device '$deviceid'\n";
3519 # fixme: this should raise exceptions on error!
3520 sub vm_deviceunplug
{
3521 my ($vmid, $conf, $deviceid) = @_;
3523 my $devices_list = vm_devices_list
($vmid);
3524 return 1 if !defined($devices_list->{$deviceid});
3526 die "can't unplug bootdisk" if $conf->{bootdisk
} && $conf->{bootdisk
} eq $deviceid;
3528 if ($deviceid eq 'tablet') {
3530 qemu_devicedel
($vmid, $deviceid);
3532 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3534 qemu_devicedel
($vmid, $deviceid);
3535 qemu_devicedelverify
($vmid, $deviceid);
3536 qemu_drivedel
($vmid, $deviceid);
3537 qemu_iothread_del
($conf, $vmid, $deviceid);
3539 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3541 qemu_devicedel
($vmid, $deviceid);
3542 qemu_devicedelverify
($vmid, $deviceid);
3543 qemu_iothread_del
($conf, $vmid, $deviceid);
3545 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3547 #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3548 my $device = parse_drive
($deviceid, $conf->{$deviceid});
3549 die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread
};
3551 qemu_devicedel
($vmid, $deviceid);
3552 qemu_drivedel
($vmid, $deviceid);
3553 qemu_deletescsihw
($conf, $vmid, $deviceid);
3555 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3557 qemu_devicedel
($vmid, $deviceid);
3558 qemu_devicedelverify
($vmid, $deviceid);
3559 qemu_netdevdel
($vmid, $deviceid);
3562 die "can't unplug device '$deviceid'\n";
3568 sub qemu_deviceadd
{
3569 my ($vmid, $devicefull) = @_;
3571 $devicefull = "driver=".$devicefull;
3572 my %options = split(/[=,]/, $devicefull);
3574 vm_mon_cmd
($vmid, "device_add" , %options);
3577 sub qemu_devicedel
{
3578 my ($vmid, $deviceid) = @_;
3580 my $ret = vm_mon_cmd
($vmid, "device_del", id
=> $deviceid);
3583 sub qemu_iothread_add
{
3584 my($vmid, $deviceid, $device) = @_;
3586 if ($device->{iothread
}) {
3587 my $iothreads = vm_iothreads_list
($vmid);
3588 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3592 sub qemu_iothread_del
{
3593 my($conf, $vmid, $deviceid) = @_;
3595 my $device = parse_drive
($deviceid, $conf->{$deviceid});
3596 if ($device->{iothread
}) {
3597 my $iothreads = vm_iothreads_list
($vmid);
3598 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3602 sub qemu_objectadd
{
3603 my($vmid, $objectid, $qomtype) = @_;
3605 vm_mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
3610 sub qemu_objectdel
{
3611 my($vmid, $objectid) = @_;
3613 vm_mon_cmd
($vmid, "object-del", id
=> $objectid);
3619 my ($storecfg, $vmid, $device) = @_;
3621 my $drive = print_drive_full
($storecfg, $vmid, $device);
3622 $drive =~ s/\\/\\\\/g;
3623 my $ret = vm_human_monitor_command
($vmid, "drive_add auto \"$drive\"");
3625 # If the command succeeds qemu prints: "OK
"
3626 return 1 if $ret =~ m/OK/s;
3628 die "adding drive failed
: $ret\n";
3632 my($vmid, $deviceid) = @_;
3634 my $ret = vm_human_monitor_command($vmid, "drive_del drive-
$deviceid");
3637 return 1 if $ret eq "";
3639 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3640 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3642 die "deleting drive
$deviceid failed
: $ret\n";
3645 sub qemu_deviceaddverify {
3646 my ($vmid, $deviceid) = @_;
3648 for (my $i = 0; $i <= 5; $i++) {
3649 my $devices_list = vm_devices_list($vmid);
3650 return 1 if defined($devices_list->{$deviceid});
3654 die "error on hotplug device
'$deviceid'\n";
3658 sub qemu_devicedelverify {
3659 my ($vmid, $deviceid) = @_;
3661 # need to verify that the device is correctly removed as device_del
3662 # is async and empty return is not reliable
3664 for (my $i = 0; $i <= 5; $i++) {
3665 my $devices_list = vm_devices_list($vmid);
3666 return 1 if !defined($devices_list->{$deviceid});
3670 die "error on hot-unplugging device
'$deviceid'\n";
3673 sub qemu_findorcreatescsihw {
3674 my ($storecfg, $conf, $vmid, $device) = @_;
3676 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3678 my $scsihwid="$controller_prefix$controller";
3679 my $devices_list = vm_devices_list($vmid);
3681 if(!defined($devices_list->{$scsihwid})) {
3682 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3688 sub qemu_deletescsihw {
3689 my ($conf, $vmid, $opt) = @_;
3691 my $device = parse_drive($opt, $conf->{$opt});
3693 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3694 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
3698 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3700 my $devices_list = vm_devices_list($vmid);
3701 foreach my $opt (keys %{$devices_list}) {
3702 if (PVE::QemuServer::is_valid_drivename($opt)) {
3703 my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3704 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3710 my $scsihwid="scsihw
$controller";
3712 vm_deviceunplug($vmid, $conf, $scsihwid);
3717 sub qemu_add_pci_bridge {
3718 my ($storecfg, $conf, $vmid, $device) = @_;
3724 print_pci_addr($device, $bridges);
3726 while (my ($k, $v) = each %$bridges) {
3729 return 1 if !defined($bridgeid) || $bridgeid < 1;
3731 my $bridge = "pci
.$bridgeid";
3732 my $devices_list = vm_devices_list($vmid);
3734 if (!defined($devices_list->{$bridge})) {
3735 vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3741 sub qemu_set_link_status {
3742 my ($vmid, $device, $up) = @_;
3744 vm_mon_cmd($vmid, "set_link
", name => $device,
3745 up => $up ? JSON::true : JSON::false);
3748 sub qemu_netdevadd {
3749 my ($vmid, $conf, $device, $deviceid) = @_;
3751 my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3752 my %options = split(/[=,]/, $netdev);
3754 vm_mon_cmd($vmid, "netdev_add
", %options);
3758 sub qemu_netdevdel {
3759 my ($vmid, $deviceid) = @_;
3761 vm_mon_cmd($vmid, "netdev_del
", id => $deviceid);
3764 sub qemu_cpu_hotplug {
3765 my ($vmid, $conf, $vcpus) = @_;
3768 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3769 $sockets = $conf->{sockets} if $conf->{sockets};
3770 my $cores = $conf->{cores} || 1;
3771 my $maxcpus = $sockets * $cores;
3773 $vcpus = $maxcpus if !$vcpus;
3775 die "you can
't add more vcpus than maxcpus\n"
3776 if $vcpus > $maxcpus;
3778 my $currentvcpus = $conf->{vcpus} || $maxcpus;
3779 die "online cpu unplug is not yet possible\n"
3780 if $vcpus < $currentvcpus;
3782 my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3783 die "vcpus in running vm is different than configuration\n"
3784 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3786 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3787 vm_mon_cmd($vmid, "cpu-add", id => int($i));
3791 sub qemu_memory_hotplug {
3792 my ($vmid, $conf, $defaults, $opt, $value) = @_;
3794 return $value if !check_running($vmid);
3796 my $memory = $conf->{memory} || $defaults->{memory};
3797 $value = $defaults->{memory} if !$value;
3798 return $value if $value == $memory;
3800 my $static_memory = $STATICMEM;
3801 my $dimm_memory = $memory - $static_memory;
3803 die "memory can't be lower than
$static_memory MB
" if $value < $static_memory;
3804 die "you cannot add more memory than
$MAX_MEM MB
!\n" if $memory > $MAX_MEM;
3808 $sockets = $conf->{sockets} if $conf->{sockets};
3810 if($value > $memory) {
3812 foreach_dimm($conf, $vmid, $value, $sockets, sub {
3813 my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3815 return if $current_size <= $conf->{memory};
3817 eval { vm_mon_cmd($vmid, "object-add
", 'qom-type' => "memory-backend-ram
", id => "mem-
$name", props => { size => int($dimm_size*1024*1024) } ) };
3819 eval { qemu_objectdel($vmid, "mem-
$name"); };
3823 eval { vm_mon_cmd($vmid, "device_add
", driver => "pc-dimm
", id => "$name", memdev => "mem-
$name", node => $numanode) };
3825 eval { qemu_objectdel($vmid, "mem-
$name"); };
3828 #update conf after each succesful module hotplug
3829 $conf->{memory} = $current_size;
3830 PVE::QemuConfig->write_config($vmid, $conf);
3835 foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3836 my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3838 return if $current_size >= $conf->{memory};
3839 print "try to unplug memory dimm
$name\n";
3843 eval { qemu_devicedel($vmid, $name) };
3845 my $dimm_list = qemu_dimm_list($vmid);
3846 last if !$dimm_list->{$name};
3847 raise_param_exc({ $name => "error unplug memory module
" }) if $retry > 5;
3851 #update conf after each succesful module unplug
3852 $conf->{memory} = $current_size;
3854 eval { qemu_objectdel($vmid, "mem-
$name"); };
3855 PVE::QemuConfig->write_config($vmid, $conf);
3860 sub qemu_dimm_list {
3863 my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices
");
3866 foreach my $dimm (@$dimmarray) {
3868 $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3869 $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3870 $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3871 $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3872 $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3877 sub qemu_block_set_io_throttle {
3878 my ($vmid, $deviceid,
3879 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3880 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3882 return if !check_running($vmid) ;
3884 vm_mon_cmd($vmid, "block_set_io_throttle
", device => $deviceid,
3886 bps_rd => int($bps_rd),
3887 bps_wr => int($bps_wr),
3889 iops_rd => int($iops_rd),
3890 iops_wr => int($iops_wr),
3891 bps_max => int($bps_max),
3892 bps_rd_max => int($bps_rd_max),
3893 bps_wr_max => int($bps_wr_max),
3894 iops_max => int($iops_max),
3895 iops_rd_max => int($iops_rd_max),
3896 iops_wr_max => int($iops_wr_max)
3901 # old code, only used to shutdown old VM after update
3903 my ($fh, $timeout) = @_;
3905 my $sel = new IO::Select;
3912 while (scalar (@ready = $sel->can_read($timeout))) {
3914 if ($count = $fh->sysread($buf, 8192)) {
3915 if ($buf =~ /^(.*)\(qemu\) $/s) {
3922 if (!defined($count)) {
3929 die "monitor
read timeout
\n" if !scalar(@ready);
3934 # old code, only used to shutdown old VM after update
3935 sub vm_monitor_command {
3936 my ($vmid, $cmdstr, $nocheck) = @_;
3941 die "VM
$vmid not running
\n" if !check_running($vmid, $nocheck);
3943 my $sname = "${var_run_tmpdir
}/$vmid.mon
";
3945 my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3946 die "unable to
connect to VM
$vmid socket - $!\n";
3950 # hack: migrate sometime blocks the monitor (when migrate_downtime
3952 if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3953 $timeout = 60*60; # 1 hour
3957 my $data = __read_avail($sock, $timeout);
3959 if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3960 die "got unexpected qemu monitor banner
\n";
3963 my $sel = new IO::Select;
3966 if (!scalar(my @ready = $sel->can_write($timeout))) {
3967 die "monitor
write error
- timeout
";
3970 my $fullcmd = "$cmdstr\r";
3972 # syslog('info', "VM
$vmid monitor command
: $cmdstr");
3975 if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3976 die "monitor
write error
- $!";
3979 return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3983 if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3984 $timeout = 60*60; # 1 hour
3985 } elsif ($cmdstr =~ m/^(eject|change)/) {
3986 $timeout = 60; # note: cdrom mount command is slow
3988 if ($res = __read_avail($sock, $timeout)) {
3990 my @lines = split("\r?
\n", $res);
3992 shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3994 $res = join("\n", @lines);
4002 syslog("err
", "VM
$vmid monitor command failed
- $err");
4009 sub qemu_block_resize {
4010 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4012 my $running = check_running($vmid);
4014 return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4016 return if !$running;
4018 vm_mon_cmd($vmid, "block_resize
", device => $deviceid, size => int($size));
4022 sub qemu_volume_snapshot {
4023 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4025 my $running = check_running($vmid);
4027 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4028 vm_mon_cmd($vmid, "snapshot-drive
", device => $deviceid, name => $snap);
4030 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4034 sub qemu_volume_snapshot_delete {
4035 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4037 my $running = check_running($vmid);
4039 return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4041 return if !$running;
4043 vm_mon_cmd($vmid, "delete-drive-snapshot
", device => $deviceid, name => $snap);
4046 sub set_migration_caps {
4052 "auto-converge
" => 1,
4054 "x-rdma-pin-all
" => 0,
4059 my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities
");
4061 for my $supported_capability (@$supported_capabilities) {
4063 capability => $supported_capability->{capability},
4064 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4068 vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities
", capabilities => $cap_ref);
4071 my $fast_plug_option = {
4080 # hotplug changes in [PENDING]
4081 # $selection hash can be used to only apply specified options, for
4082 # example: { cores => 1 } (only apply changed 'cores')
4083 # $errors ref is used to return error messages
4084 sub vmconfig_hotplug_pending {
4085 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4087 my $defaults = load_defaults();
4089 # commit values which do not have any impact on running VM first
4090 # Note: those option cannot raise errors, we we do not care about
4091 # $selection and always apply them.
4093 my $add_error = sub {
4094 my ($opt, $msg) = @_;
4095 $errors->{$opt} = "hotplug problem
- $msg";
4099 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4100 if ($fast_plug_option->{$opt}) {
4101 $conf->{$opt} = $conf->{pending}->{$opt};
4102 delete $conf->{pending}->{$opt};
4108 PVE::QemuConfig->write_config($vmid, $conf);
4109 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4112 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4114 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4115 while (my ($opt, $force) = each %$pending_delete_hash) {
4116 next if $selection && !$selection->{$opt};
4118 if ($opt eq 'hotplug') {
4119 die "skip
\n" if ($conf->{hotplug} =~ /memory/);
4120 } elsif ($opt eq 'tablet') {
4121 die "skip
\n" if !$hotplug_features->{usb};
4122 if ($defaults->{tablet}) {
4123 vm_deviceplug($storecfg, $conf, $vmid, $opt);
4125 vm_deviceunplug($vmid, $conf, $opt);
4127 } elsif ($opt eq 'vcpus') {
4128 die "skip
\n" if !$hotplug_features->{cpu};
4129 qemu_cpu_hotplug($vmid, $conf, undef);
4130 } elsif ($opt eq 'balloon') {
4131 # enable balloon device is not hotpluggable
4132 die "skip
\n" if !defined($conf->{balloon}) || $conf->{balloon};
4133 } elsif ($fast_plug_option->{$opt}) {
4135 } elsif ($opt =~ m/^net(\d+)$/) {
4136 die "skip
\n" if !$hotplug_features->{network};
4137 vm_deviceunplug($vmid, $conf, $opt);
4138 } elsif (is_valid_drivename($opt)) {
4139 die "skip
\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4140 vm_deviceunplug($vmid, $conf, $opt);
4141 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4142 } elsif ($opt =~ m/^memory$/) {
4143 die "skip
\n" if !$hotplug_features->{memory};
4144 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4145 } elsif ($opt eq 'cpuunits') {
4146 cgroups_write("cpu
", $vmid, "cpu
.shares
", $defaults->{cpuunits});
4147 } elsif ($opt eq 'cpulimit') {
4148 cgroups_write("cpu
", $vmid, "cpu
.cfs_quota_us
", -1);
4154 &$add_error($opt, $err) if $err ne "skip
\n";
4156 # save new config if hotplug was successful
4157 delete $conf->{$opt};
4158 vmconfig_undelete_pending_option($conf, $opt);
4159 PVE::QemuConfig->write_config($vmid, $conf);
4160 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4164 foreach my $opt (keys %{$conf->{pending}}) {
4165 next if $selection && !$selection->{$opt};
4166 my $value = $conf->{pending}->{$opt};
4168 if ($opt eq 'hotplug') {
4169 die "skip
\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4170 } elsif ($opt eq 'tablet') {
4171 die "skip
\n" if !$hotplug_features->{usb};
4173 vm_deviceplug($storecfg, $conf, $vmid, $opt);
4174 } elsif ($value == 0) {
4175 vm_deviceunplug($vmid, $conf, $opt);
4177 } elsif ($opt eq 'vcpus') {
4178 die "skip
\n" if !$hotplug_features->{cpu};
4179 qemu_cpu_hotplug($vmid, $conf, $value);
4180 } elsif ($opt eq 'balloon') {
4181 # enable/disable balloning device is not hotpluggable
4182 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4183 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4184 die "skip
\n" if $old_balloon_enabled != $new_balloon_enabled;
4186 # allow manual ballooning if shares is set to zero
4187 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4188 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4189 vm_mon_cmd($vmid, "balloon
", value => $balloon*1024*1024);
4191 } elsif ($opt =~ m/^net(\d+)$/) {
4192 # some changes can be done without hotplug
4193 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4194 $vmid, $opt, $value);
4195 } elsif (is_valid_drivename($opt)) {
4196 # some changes can be done without hotplug
4197 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4198 $vmid, $opt, $value, 1);
4199 } elsif ($opt =~ m/^memory$/) { #dimms
4200 die "skip
\n" if !$hotplug_features->{memory};
4201 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4202 } elsif ($opt eq 'cpuunits') {
4203 cgroups_write("cpu
", $vmid, "cpu
.shares
", $conf->{pending}->{$opt});
4204 } elsif ($opt eq 'cpulimit') {
4205 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4206 cgroups_write("cpu
", $vmid, "cpu
.cfs_quota_us
", $cpulimit);
4208 die "skip
\n"; # skip non-hot-pluggable options
4212 &$add_error($opt, $err) if $err ne "skip
\n";
4214 # save new config if hotplug was successful
4215 $conf->{$opt} = $value;
4216 delete $conf->{pending}->{$opt};
4217 PVE::QemuConfig->write_config($vmid, $conf);
4218 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4223 sub try_deallocate_drive {
4224 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4226 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4227 my $volid = $drive->{file};
4228 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4229 my $sid = PVE::Storage::parse_volume_id($volid);
4230 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4232 # check if the disk is really unused
4233 die "unable to
delete '$volid' - volume
is still
in use (snapshot?
)\n"
4234 if is_volume_in_use($storecfg, $conf, $key, $volid);
4235 PVE::Storage::vdisk_free($storecfg, $volid);
4238 # If vm is not owner of this disk remove from config
4246 sub vmconfig_delete_or_detach_drive {
4247 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4249 my $drive = parse_drive($opt, $conf->{$opt});
4251 my $rpcenv = PVE::RPCEnvironment::get();
4252 my $authuser = $rpcenv->get_user();
4255 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4256 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4258 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4262 sub vmconfig_apply_pending {
4263 my ($vmid, $conf, $storecfg) = @_;
4267 my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4268 while (my ($opt, $force) = each %$pending_delete_hash) {
4269 die "internal error
" if $opt =~ m/^unused/;
4270 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4271 if (!defined($conf->{$opt})) {
4272 vmconfig_undelete_pending_option($conf, $opt);
4273 PVE::QemuConfig->write_config($vmid, $conf);
4274 } elsif (is_valid_drivename($opt)) {
4275 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4276 vmconfig_undelete_pending_option($conf, $opt);
4277 delete $conf->{$opt};
4278 PVE::QemuConfig->write_config($vmid, $conf);
4280 vmconfig_undelete_pending_option($conf, $opt);
4281 delete $conf->{$opt};
4282 PVE::QemuConfig->write_config($vmid, $conf);
4286 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4288 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4289 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4291 if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4292 # skip if nothing changed
4293 } elsif (is_valid_drivename($opt)) {
4294 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4295 if defined($conf->{$opt});
4296 $conf->{$opt} = $conf->{pending}->{$opt};
4298 $conf->{$opt} = $conf->{pending}->{$opt};
4301 delete $conf->{pending}->{$opt};
4302 PVE::QemuConfig->write_config($vmid, $conf);
4306 my $safe_num_ne = sub {
4309 return 0 if !defined($a) && !defined($b);
4310 return 1 if !defined($a);
4311 return 1 if !defined($b);
4316 my $safe_string_ne = sub {
4319 return 0 if !defined($a) && !defined($b);
4320 return 1 if !defined($a);
4321 return 1 if !defined($b);
4326 sub vmconfig_update_net {
4327 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4329 my $newnet = parse_net($value);
4331 if ($conf->{$opt}) {
4332 my $oldnet = parse_net($conf->{$opt});
4334 if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4335 &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4336 &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4337 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4339 # for non online change, we try to hot-unplug
4340 die "skip
\n" if !$hotplug;
4341 vm_deviceunplug($vmid, $conf, $opt);
4344 die "internal error
" if $opt !~ m/net(\d+)/;
4345 my $iface = "tap
${vmid
}i
$1";
4347 if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4348 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4349 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4350 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4351 PVE::Network::tap_unplug($iface);
4352 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4353 } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4354 # Rate can be applied on its own but any change above needs to
4355 # include the rate in tap_plug since OVS resets everything.
4356 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4359 if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4360 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4368 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4374 sub vmconfig_update_disk {
4375 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4377 # fixme: do we need force?
4379 my $drive = parse_drive($opt, $value);
4381 if ($conf->{$opt}) {
4383 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
4385 my $media = $drive->{media} || 'disk';
4386 my $oldmedia = $old_drive->{media} || 'disk';
4387 die "unable to change media type
\n" if $media ne $oldmedia;
4389 if (!drive_is_cdrom($old_drive)) {
4391 if ($drive->{file} ne $old_drive->{file}) {
4393 die "skip
\n" if !$hotplug;
4395 # unplug and register as unused
4396 vm_deviceunplug($vmid, $conf, $opt);
4397 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4400 # update existing disk
4402 # skip non hotpluggable value
4403 if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4404 &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4405 &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4406 &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4411 if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4412 &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4413 &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4414 &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4415 &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4416 &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4417 &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4418 &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4419 &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4420 &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4421 &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4422 &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4424 qemu_block_set_io_throttle($vmid,"drive-
$opt",
4425 ($drive->{mbps} || 0)*1024*1024,
4426 ($drive->{mbps_rd} || 0)*1024*1024,
4427 ($drive->{mbps_wr} || 0)*1024*1024,
4428 $drive->{iops} || 0,
4429 $drive->{iops_rd} || 0,
4430 $drive->{iops_wr} || 0,
4431 ($drive->{mbps_max} || 0)*1024*1024,
4432 ($drive->{mbps_rd_max} || 0)*1024*1024,
4433 ($drive->{mbps_wr_max} || 0)*1024*1024,
4434 $drive->{iops_max} || 0,
4435 $drive->{iops_rd_max} || 0,
4436 $drive->{iops_wr_max} || 0);
4445 if ($drive->{file} eq 'none') {
4446 vm_mon_cmd($vmid, "eject
",force => JSON::true,device => "drive-
$opt");
4448 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4449 vm_mon_cmd($vmid, "eject
", force => JSON::true,device => "drive-
$opt"); # force eject if locked
4450 vm_mon_cmd($vmid, "change
", device => "drive-
$opt",target => "$path") if $path;
4458 die "skip
\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4460 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4461 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4465 my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4466 $forcemachine, $spice_ticket) = @_;
4468 PVE::QemuConfig->lock_config($vmid, sub {
4469 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4471 die "you can
't start a vm if it's a template
\n" if PVE::QemuConfig->is_template($conf);
4473 PVE::QemuConfig->check_lock($conf) if !$skiplock;
4475 die "VM
$vmid already running
\n" if check_running($vmid, undef, $migratedfrom);
4477 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4478 vmconfig_apply_pending($vmid, $conf, $storecfg);
4479 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4482 my $defaults = load_defaults();
4484 # set environment variable useful inside network script
4485 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4487 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4489 my $migrate_port = 0;
4492 if ($statefile eq 'tcp') {
4493 my $localip = "localhost
";
4494 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4495 my $nodename = PVE::INotify::nodename();
4496 if ($datacenterconf->{migration_unsecure}) {
4497 $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4498 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4500 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4501 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4502 $migrate_uri = "tcp
:${localip
}:${migrate_port
}";
4503 push @$cmd, '-incoming', $migrate_uri;
4506 push @$cmd, '-loadstate', $statefile;
4513 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
4514 my $d = parse_hostpci($conf->{"hostpci
$i"});
4516 my $pcidevices = $d->{pciid};
4517 foreach my $pcidevice (@$pcidevices) {
4518 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4520 my $info = pci_device_info("0000:$pciid");
4521 die "IOMMU
not present
\n" if !check_iommu_support();
4522 die "no pci device info
for device
'$pciid'\n" if !$info;
4523 die "can
't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4524 die "can't
reset pci device
'$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4528 PVE::Storage::activate_volumes($storecfg, $vollist);
4530 eval { run_command($cmd, timeout => $statefile ? undef : 30,
4534 # deactivate volumes if start fails
4535 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4536 die "start failed
: $err";
4539 print "migration listens on
$migrate_uri\n" if $migrate_uri;
4541 if ($statefile && $statefile ne 'tcp') {
4542 eval { vm_mon_cmd_nocheck($vmid, "cont
"); };
4546 if ($migratedfrom) {
4549 set_migration_caps($vmid);
4554 print "spice listens on port
$spice_port\n";
4555 if ($spice_ticket) {
4556 vm_mon_cmd_nocheck($vmid, "set_password
", protocol => 'spice', password => $spice_ticket);
4557 vm_mon_cmd_nocheck($vmid, "expire_password
", protocol => 'spice', time => "+30");
4563 if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4564 vm_mon_cmd_nocheck($vmid, "balloon
", value => $conf->{balloon}*1024*1024)
4565 if $conf->{balloon};
4568 foreach my $opt (keys %$conf) {
4569 next if $opt !~ m/^net\d+$/;
4570 my $nicconf = parse_net($conf->{$opt});
4571 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4575 vm_mon_cmd_nocheck($vmid, 'qom-set',
4576 path => "machine
/peripheral/balloon0
",
4577 property => "guest-stats-polling-interval
",
4578 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4584 my ($vmid, $execute, %params) = @_;
4586 my $cmd = { execute => $execute, arguments => \%params };
4587 vm_qmp_command($vmid, $cmd);
4590 sub vm_mon_cmd_nocheck {
4591 my ($vmid, $execute, %params) = @_;
4593 my $cmd = { execute => $execute, arguments => \%params };
4594 vm_qmp_command($vmid, $cmd, 1);
4597 sub vm_qmp_command {
4598 my ($vmid, $cmd, $nocheck) = @_;
4603 if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4604 $timeout = $cmd->{arguments}->{timeout};
4605 delete $cmd->{arguments}->{timeout};
4609 die "VM
$vmid not running
\n" if !check_running($vmid, $nocheck);
4610 my $sname = qmp_socket($vmid);
4611 if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4612 my $qmpclient = PVE::QMPClient->new();
4614 $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4615 } elsif (-e "${var_run_tmpdir
}/$vmid.mon
") {
4616 die "can
't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4617 if scalar(%{$cmd->{arguments}});
4618 vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4620 die "unable to open monitor socket\n";
4624 syslog("err", "VM $vmid qmp command failed - $err");
4631 sub vm_human_monitor_command {
4632 my ($vmid, $cmdline) = @_;
4637 execute => 'human-monitor-command
',
4638 arguments => { 'command-line
' => $cmdline},
4641 return vm_qmp_command($vmid, $cmd);
4644 sub vm_commandline {
4645 my ($storecfg, $vmid) = @_;
4647 my $conf = PVE::QemuConfig->load_config($vmid);
4649 my $defaults = load_defaults();
4651 my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4653 return join(' ', @$cmd);
4657 my ($vmid, $skiplock) = @_;
4659 PVE::QemuConfig->lock_config($vmid, sub {
4661 my $conf = PVE::QemuConfig->load_config($vmid);
4663 PVE::QemuConfig->check_lock($conf) if !$skiplock;
4665 vm_mon_cmd($vmid, "system_reset");
4669 sub get_vm_volumes {
4673 foreach_volid($conf, sub {
4674 my ($volid, $is_cdrom) = @_;
4676 return if $volid =~ m|^/|;
4678 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4681 push @$vollist, $volid;
4687 sub vm_stop_cleanup {
4688 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4693 my $vollist = get_vm_volumes($conf);
4694 PVE::Storage::deactivate_volumes($storecfg, $vollist);
4697 foreach my $ext (qw(mon qmp pid vnc qga)) {
4698 unlink "/var/run/qemu-server/${vmid}.$ext";
4701 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
4703 warn $@ if $@; # avoid errors - just warn
4706 # Note: use $nockeck to skip tests if VM configuration file exists.
4707 # We need that when migration VMs to other nodes (files already moved)
4708 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4710 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4712 $force = 1 if !defined($force) && !$shutdown;
4715 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
4716 kill 15, $pid if $pid;
4717 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
4718 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
4722 PVE
::QemuConfig-
>lock_config($vmid, sub {
4724 my $pid = check_running
($vmid, $nocheck);
4729 $conf = PVE
::QemuConfig-
>load_config($vmid);
4730 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
4731 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
4732 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
4733 $timeout = $opts->{down
} if $opts->{down
};
4737 $timeout = 60 if !defined($timeout);
4741 if (defined($conf) && $conf->{agent
}) {
4742 vm_qmp_command
($vmid, { execute
=> "guest-shutdown" }, $nocheck);
4744 vm_qmp_command
($vmid, { execute
=> "system_powerdown" }, $nocheck);
4747 vm_qmp_command
($vmid, { execute
=> "quit" }, $nocheck);
4754 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
4759 if ($count >= $timeout) {
4761 warn "VM still running - terminating now with SIGTERM\n";
4764 die "VM quit/powerdown failed - got timeout\n";
4767 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4772 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4775 die "VM quit/powerdown failed\n";
4783 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
4788 if ($count >= $timeout) {
4789 warn "VM still running - terminating now with SIGKILL\n";
4794 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4799 my ($vmid, $skiplock) = @_;
4801 PVE
::QemuConfig-
>lock_config($vmid, sub {
4803 my $conf = PVE
::QemuConfig-
>load_config($vmid);
4805 PVE
::QemuConfig-
>check_lock($conf)
4806 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
4808 vm_mon_cmd
($vmid, "stop");
4813 my ($vmid, $skiplock, $nocheck) = @_;
4815 PVE
::QemuConfig-
>lock_config($vmid, sub {
4819 my $conf = PVE
::QemuConfig-
>load_config($vmid);
4821 PVE
::QemuConfig-
>check_lock($conf)
4822 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
4824 vm_mon_cmd
($vmid, "cont");
4827 vm_mon_cmd_nocheck
($vmid, "cont");
4833 my ($vmid, $skiplock, $key) = @_;
4835 PVE
::QemuConfig-
>lock_config($vmid, sub {
4837 my $conf = PVE
::QemuConfig-
>load_config($vmid);
4839 # there is no qmp command, so we use the human monitor command
4840 vm_human_monitor_command
($vmid, "sendkey $key");
4845 my ($storecfg, $vmid, $skiplock) = @_;
4847 PVE
::QemuConfig-
>lock_config($vmid, sub {
4849 my $conf = PVE
::QemuConfig-
>load_config($vmid);
4851 if (!check_running
($vmid)) {
4852 destroy_vm
($storecfg, $vmid, undef, $skiplock);
4854 die "VM $vmid is running - destroy failed\n";
4862 my ($filename, $buf) = @_;
4864 my $fh = IO
::File-
>new($filename, "w");
4865 return undef if !$fh;
4867 my $res = print $fh $buf;
4874 sub pci_device_info
{
4879 return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4880 my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4882 my $irq = file_read_firstline
("$pcisysfs/devices/$name/irq");
4883 return undef if !defined($irq) || $irq !~ m/^\d+$/;
4885 my $vendor = file_read_firstline
("$pcisysfs/devices/$name/vendor");
4886 return undef if !defined($vendor) || $vendor !~ s/^0x//;
4888 my $product = file_read_firstline
("$pcisysfs/devices/$name/device");
4889 return undef if !defined($product) || $product !~ s/^0x//;
4894 product
=> $product,
4900 has_fl_reset
=> -f
"$pcisysfs/devices/$name/reset" || 0,
4909 my $name = $dev->{name
};
4911 my $fn = "$pcisysfs/devices/$name/reset";
4913 return file_write
($fn, "1");
4916 sub pci_dev_bind_to_vfio
{
4919 my $name = $dev->{name
};
4921 my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4923 if (!-d
$vfio_basedir) {
4924 system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4926 die "Cannot find vfio-pci module!\n" if !-d
$vfio_basedir;
4928 my $testdir = "$vfio_basedir/$name";
4929 return 1 if -d
$testdir;
4931 my $data = "$dev->{vendor} $dev->{product}";
4932 return undef if !file_write
("$vfio_basedir/new_id", $data);
4934 my $fn = "$pcisysfs/devices/$name/driver/unbind";
4935 if (!file_write
($fn, $name)) {
4936 return undef if -f
$fn;
4939 $fn = "$vfio_basedir/bind";
4940 if (! -d
$testdir) {
4941 return undef if !file_write
($fn, $name);
4947 sub pci_dev_group_bind_to_vfio
{
4950 my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4952 if (!-d
$vfio_basedir) {
4953 system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4955 die "Cannot find vfio-pci module!\n" if !-d
$vfio_basedir;
4957 # get IOMMU group devices
4958 opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4959 my @devs = grep /^0000:/, readdir($D);
4962 foreach my $pciid (@devs) {
4963 $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4965 # pci bridges, switches or root ports are not supported
4966 # they have a pci_bus subdirectory so skip them
4967 next if (-e
"$pcisysfs/devices/$pciid/pci_bus");
4969 my $info = pci_device_info
($1);
4970 pci_dev_bind_to_vfio
($info) || die "Cannot bind $pciid to vfio\n";
4976 sub print_pci_addr
{
4977 my ($id, $bridges) = @_;
4981 piix3
=> { bus
=> 0, addr
=> 1 },
4982 #addr2 : first videocard
4983 balloon0
=> { bus
=> 0, addr
=> 3 },
4984 watchdog
=> { bus
=> 0, addr
=> 4 },
4985 scsihw0
=> { bus
=> 0, addr
=> 5 },
4986 'pci.3' => { bus
=> 0, addr
=> 5 }, #can also be used for virtio-scsi-single bridge
4987 scsihw1
=> { bus
=> 0, addr
=> 6 },
4988 ahci0
=> { bus
=> 0, addr
=> 7 },
4989 qga0
=> { bus
=> 0, addr
=> 8 },
4990 spice
=> { bus
=> 0, addr
=> 9 },
4991 virtio0
=> { bus
=> 0, addr
=> 10 },
4992 virtio1
=> { bus
=> 0, addr
=> 11 },
4993 virtio2
=> { bus
=> 0, addr
=> 12 },
4994 virtio3
=> { bus
=> 0, addr
=> 13 },
4995 virtio4
=> { bus
=> 0, addr
=> 14 },
4996 virtio5
=> { bus
=> 0, addr
=> 15 },
4997 hostpci0
=> { bus
=> 0, addr
=> 16 },
4998 hostpci1
=> { bus
=> 0, addr
=> 17 },
4999 net0
=> { bus
=> 0, addr
=> 18 },
5000 net1
=> { bus
=> 0, addr
=> 19 },
5001 net2
=> { bus
=> 0, addr
=> 20 },
5002 net3
=> { bus
=> 0, addr
=> 21 },
5003 net4
=> { bus
=> 0, addr
=> 22 },
5004 net5
=> { bus
=> 0, addr
=> 23 },
5005 vga1
=> { bus
=> 0, addr
=> 24 },
5006 vga2
=> { bus
=> 0, addr
=> 25 },
5007 vga3
=> { bus
=> 0, addr
=> 26 },
5008 hostpci2
=> { bus
=> 0, addr
=> 27 },
5009 hostpci3
=> { bus
=> 0, addr
=> 28 },
5010 #addr29 : usb-host (pve-usb.cfg)
5011 'pci.1' => { bus
=> 0, addr
=> 30 },
5012 'pci.2' => { bus
=> 0, addr
=> 31 },
5013 'net6' => { bus
=> 1, addr
=> 1 },
5014 'net7' => { bus
=> 1, addr
=> 2 },
5015 'net8' => { bus
=> 1, addr
=> 3 },
5016 'net9' => { bus
=> 1, addr
=> 4 },
5017 'net10' => { bus
=> 1, addr
=> 5 },
5018 'net11' => { bus
=> 1, addr
=> 6 },
5019 'net12' => { bus
=> 1, addr
=> 7 },
5020 'net13' => { bus
=> 1, addr
=> 8 },
5021 'net14' => { bus
=> 1, addr
=> 9 },
5022 'net15' => { bus
=> 1, addr
=> 10 },
5023 'net16' => { bus
=> 1, addr
=> 11 },
5024 'net17' => { bus
=> 1, addr
=> 12 },
5025 'net18' => { bus
=> 1, addr
=> 13 },
5026 'net19' => { bus
=> 1, addr
=> 14 },
5027 'net20' => { bus
=> 1, addr
=> 15 },
5028 'net21' => { bus
=> 1, addr
=> 16 },
5029 'net22' => { bus
=> 1, addr
=> 17 },
5030 'net23' => { bus
=> 1, addr
=> 18 },
5031 'net24' => { bus
=> 1, addr
=> 19 },
5032 'net25' => { bus
=> 1, addr
=> 20 },
5033 'net26' => { bus
=> 1, addr
=> 21 },
5034 'net27' => { bus
=> 1, addr
=> 22 },
5035 'net28' => { bus
=> 1, addr
=> 23 },
5036 'net29' => { bus
=> 1, addr
=> 24 },
5037 'net30' => { bus
=> 1, addr
=> 25 },
5038 'net31' => { bus
=> 1, addr
=> 26 },
5039 'xhci' => { bus
=> 1, addr
=> 27 },
5040 'virtio6' => { bus
=> 2, addr
=> 1 },
5041 'virtio7' => { bus
=> 2, addr
=> 2 },
5042 'virtio8' => { bus
=> 2, addr
=> 3 },
5043 'virtio9' => { bus
=> 2, addr
=> 4 },
5044 'virtio10' => { bus
=> 2, addr
=> 5 },
5045 'virtio11' => { bus
=> 2, addr
=> 6 },
5046 'virtio12' => { bus
=> 2, addr
=> 7 },
5047 'virtio13' => { bus
=> 2, addr
=> 8 },
5048 'virtio14' => { bus
=> 2, addr
=> 9 },
5049 'virtio15' => { bus
=> 2, addr
=> 10 },
5050 'virtioscsi0' => { bus
=> 3, addr
=> 1 },
5051 'virtioscsi1' => { bus
=> 3, addr
=> 2 },
5052 'virtioscsi2' => { bus
=> 3, addr
=> 3 },
5053 'virtioscsi3' => { bus
=> 3, addr
=> 4 },
5054 'virtioscsi4' => { bus
=> 3, addr
=> 5 },
5055 'virtioscsi5' => { bus
=> 3, addr
=> 6 },
5056 'virtioscsi6' => { bus
=> 3, addr
=> 7 },
5057 'virtioscsi7' => { bus
=> 3, addr
=> 8 },
5058 'virtioscsi8' => { bus
=> 3, addr
=> 9 },
5059 'virtioscsi9' => { bus
=> 3, addr
=> 10 },
5060 'virtioscsi10' => { bus
=> 3, addr
=> 11 },
5061 'virtioscsi11' => { bus
=> 3, addr
=> 12 },
5062 'virtioscsi12' => { bus
=> 3, addr
=> 13 },
5063 'virtioscsi13' => { bus
=> 3, addr
=> 14 },
5064 'virtioscsi14' => { bus
=> 3, addr
=> 15 },
5065 'virtioscsi15' => { bus
=> 3, addr
=> 16 },
5066 'virtioscsi16' => { bus
=> 3, addr
=> 17 },
5067 'virtioscsi17' => { bus
=> 3, addr
=> 18 },
5068 'virtioscsi18' => { bus
=> 3, addr
=> 19 },
5069 'virtioscsi19' => { bus
=> 3, addr
=> 20 },
5070 'virtioscsi20' => { bus
=> 3, addr
=> 21 },
5071 'virtioscsi21' => { bus
=> 3, addr
=> 22 },
5072 'virtioscsi22' => { bus
=> 3, addr
=> 23 },
5073 'virtioscsi23' => { bus
=> 3, addr
=> 24 },
5074 'virtioscsi24' => { bus
=> 3, addr
=> 25 },
5075 'virtioscsi25' => { bus
=> 3, addr
=> 26 },
5076 'virtioscsi26' => { bus
=> 3, addr
=> 27 },
5077 'virtioscsi27' => { bus
=> 3, addr
=> 28 },
5078 'virtioscsi28' => { bus
=> 3, addr
=> 29 },
5079 'virtioscsi29' => { bus
=> 3, addr
=> 30 },
5080 'virtioscsi30' => { bus
=> 3, addr
=> 31 },
5084 if (defined($devices->{$id}->{bus
}) && defined($devices->{$id}->{addr
})) {
5085 my $addr = sprintf("0x%x", $devices->{$id}->{addr
});
5086 my $bus = $devices->{$id}->{bus
};
5087 $res = ",bus=pci.$bus,addr=$addr";
5088 $bridges->{$bus} = 1 if $bridges;
5094 sub print_pcie_addr
{
5099 hostpci0
=> { bus
=> "ich9-pcie-port-1", addr
=> 0 },
5100 hostpci1
=> { bus
=> "ich9-pcie-port-2", addr
=> 0 },
5101 hostpci2
=> { bus
=> "ich9-pcie-port-3", addr
=> 0 },
5102 hostpci3
=> { bus
=> "ich9-pcie-port-4", addr
=> 0 },
5105 if (defined($devices->{$id}->{bus
}) && defined($devices->{$id}->{addr
})) {
5106 my $addr = sprintf("0x%x", $devices->{$id}->{addr
});
5107 my $bus = $devices->{$id}->{bus
};
5108 $res = ",bus=$bus,addr=$addr";
5114 # vzdump restore implementaion
5116 sub tar_archive_read_firstfile
{
5117 my $archive = shift;
5119 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
5121 # try to detect archive type first
5122 my $pid = open (TMP
, "tar tf '$archive'|") ||
5123 die "unable to open file '$archive'\n";
5124 my $firstfile = <TMP
>;
5128 die "ERROR: archive contaions no data\n" if !$firstfile;
5134 sub tar_restore_cleanup
{
5135 my ($storecfg, $statfile) = @_;
5137 print STDERR
"starting cleanup\n";
5139 if (my $fd = IO
::File-
>new($statfile, "r")) {
5140 while (defined(my $line = <$fd>)) {
5141 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5144 if ($volid =~ m
|^/|) {
5145 unlink $volid || die 'unlink failed\n';
5147 PVE
::Storage
::vdisk_free
($storecfg, $volid);
5149 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5151 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5153 print STDERR
"unable to parse line in statfile - $line";
5160 sub restore_archive
{
5161 my ($archive, $vmid, $user, $opts) = @_;
5163 my $format = $opts->{format
};
5166 if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5167 $format = 'tar' if !$format;
5169 } elsif ($archive =~ m/\.tar$/) {
5170 $format = 'tar' if !$format;
5171 } elsif ($archive =~ m/.tar.lzo$/) {
5172 $format = 'tar' if !$format;
5174 } elsif ($archive =~ m/\.vma$/) {
5175 $format = 'vma' if !$format;
5176 } elsif ($archive =~ m/\.vma\.gz$/) {
5177 $format = 'vma' if !$format;
5179 } elsif ($archive =~ m/\.vma\.lzo$/) {
5180 $format = 'vma' if !$format;
5183 $format = 'vma' if !$format; # default
5186 # try to detect archive format
5187 if ($format eq 'tar') {
5188 return restore_tar_archive
($archive, $vmid, $user, $opts);
5190 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
5194 sub restore_update_config_line
{
5195 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5197 return if $line =~ m/^\#qmdump\#/;
5198 return if $line =~ m/^\#vzdump\#/;
5199 return if $line =~ m/^lock:/;
5200 return if $line =~ m/^unused\d+:/;
5201 return if $line =~ m/^parent:/;
5202 return if $line =~ m/^template:/; # restored VM is never a template
5204 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5205 # try to convert old 1.X settings
5206 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5207 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
5208 my ($model, $macaddr) = split(/\=/, $devconfig);
5209 $macaddr = PVE
::Tools
::random_ether_addr
() if !$macaddr || $unique;
5212 bridge
=> "vmbr$ind",
5213 macaddr
=> $macaddr,
5215 my $netstr = print_net
($net);
5217 print $outfd "net$cookie->{netcount}: $netstr\n";
5218 $cookie->{netcount
}++;
5220 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5221 my ($id, $netstr) = ($1, $2);
5222 my $net = parse_net
($netstr);
5223 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
() if $net->{macaddr
};
5224 $netstr = print_net
($net);
5225 print $outfd "$id: $netstr\n";
5226 } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5229 my $di = parse_drive
($virtdev, $value);
5230 if (defined($di->{backup
}) && !$di->{backup
}) {
5231 print $outfd "#$line";
5232 } elsif ($map->{$virtdev}) {
5233 delete $di->{format
}; # format can change on restore
5234 $di->{file
} = $map->{$virtdev};
5235 $value = print_drive
($vmid, $di);
5236 print $outfd "$virtdev: $value\n";
5246 my ($cfg, $vmid) = @_;
5248 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid);
5250 my $volid_hash = {};
5251 foreach my $storeid (keys %$info) {
5252 foreach my $item (@{$info->{$storeid}}) {
5253 next if !($item->{volid
} && $item->{size
});
5254 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
5255 $volid_hash->{$item->{volid
}} = $item;
5262 sub is_volume_in_use
{
5263 my ($storecfg, $conf, $skip_drive, $volid) = @_;
5265 my $path = PVE
::Storage
::path
($storecfg, $volid);
5267 my $scan_config = sub {
5268 my ($cref, $snapname) = @_;
5270 foreach my $key (keys %$cref) {
5271 my $value = $cref->{$key};
5272 if (is_valid_drivename
($key)) {
5273 next if $skip_drive && $key eq $skip_drive;
5274 my $drive = parse_drive
($key, $value);
5275 next if !$drive || !$drive->{file
} || drive_is_cdrom
($drive);
5276 return 1 if $volid eq $drive->{file
};
5277 if ($drive->{file
} =~ m!^/!) {
5278 return 1 if $drive->{file
} eq $path;
5280 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
}, 1);
5282 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid, 1);
5284 return 1 if $path eq PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
5292 return 1 if &$scan_config($conf);
5296 foreach my $snapname (keys %{$conf->{snapshots
}}) {
5297 return 1 if &$scan_config($conf->{snapshots
}->{$snapname}, $snapname);
5303 sub update_disksize
{
5304 my ($vmid, $conf, $volid_hash) = @_;
5310 # Note: it is allowed to define multiple storages with same path (alias), so
5311 # we need to check both 'volid' and real 'path' (two different volid can point
5312 # to the same path).
5317 foreach my $opt (keys %$conf) {
5318 if (is_valid_drivename
($opt)) {
5319 my $drive = parse_drive
($opt, $conf->{$opt});
5320 my $volid = $drive->{file
};
5323 $used->{$volid} = 1;
5324 if ($volid_hash->{$volid} &&
5325 (my $path = $volid_hash->{$volid}->{path
})) {
5326 $usedpath->{$path} = 1;
5329 next if drive_is_cdrom
($drive);
5330 next if !$volid_hash->{$volid};
5332 $drive->{size
} = $volid_hash->{$volid}->{size
};
5333 my $new = print_drive
($vmid, $drive);
5334 if ($new ne $conf->{$opt}) {
5336 $conf->{$opt} = $new;
5341 # remove 'unusedX' entry if volume is used
5342 foreach my $opt (keys %$conf) {
5343 next if $opt !~ m/^unused\d+$/;
5344 my $volid = $conf->{$opt};
5345 my $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
5346 if ($used->{$volid} || ($path && $usedpath->{$path})) {
5348 delete $conf->{$opt};
5352 foreach my $volid (sort keys %$volid_hash) {
5353 next if $volid =~ m/vm-$vmid-state-/;
5354 next if $used->{$volid};
5355 my $path = $volid_hash->{$volid}->{path
};
5356 next if !$path; # just to be sure
5357 next if $usedpath->{$path};
5359 PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
5360 $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5367 my ($vmid, $nolock) = @_;
5369 my $cfg = PVE
::Storage
::config
();
5371 my $volid_hash = scan_volids
($cfg, $vmid);
5373 my $updatefn = sub {
5376 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5378 PVE
::QemuConfig-
>check_lock($conf);
5381 foreach my $volid (keys %$volid_hash) {
5382 my $info = $volid_hash->{$volid};
5383 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
5386 my $changes = update_disksize
($vmid, $conf, $vm_volids);
5388 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes;
5391 if (defined($vmid)) {
5395 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
5398 my $vmlist = config_list
();
5399 foreach my $vmid (keys %$vmlist) {
5403 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
5409 sub restore_vma_archive
{
5410 my ($archive, $vmid, $user, $opts, $comp) = @_;
5412 my $input = $archive eq '-' ?
"<&STDIN" : undef;
5413 my $readfrom = $archive;
5418 my $qarchive = PVE
::Tools
::shellquote
($archive);
5419 if ($comp eq 'gzip') {
5420 $uncomp = "zcat $qarchive|";
5421 } elsif ($comp eq 'lzop') {
5422 $uncomp = "lzop -d -c $qarchive|";
5424 die "unknown compression method '$comp'\n";
5429 my $tmpdir = "/var/tmp/vzdumptmp$$";
5432 # disable interrupts (always do cleanups)
5433 local $SIG{INT
} = $SIG{TERM
} = $SIG{QUIT
} = $SIG{HUP
} = sub {
5434 warn "got interrupt - ignored\n";
5437 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5438 POSIX
::mkfifo
($mapfifo, 0600);
5441 my $openfifo = sub {
5442 open($fifofh, '>', $mapfifo) || die $!;
5445 my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5452 my $rpcenv = PVE
::RPCEnvironment
::get
();
5454 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
5455 my $tmpfn = "$conffile.$$.tmp";
5457 # Note: $oldconf is undef if VM does not exists
5458 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
5459 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
5461 my $print_devmap = sub {
5462 my $virtdev_hash = {};
5464 my $cfgfn = "$tmpdir/qemu-server.conf";
5466 # we can read the config - that is already extracted
5467 my $fh = IO
::File-
>new($cfgfn, "r") ||
5468 "unable to read qemu-server.conf - $!\n";
5470 my $fwcfgfn = "$tmpdir/qemu-server.fw";
5472 my $pve_firewall_dir = '/etc/pve/firewall';
5473 mkdir $pve_firewall_dir; # make sure the dir exists
5474 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5477 while (defined(my $line = <$fh>)) {
5478 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5479 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5480 die "archive does not contain data for drive '$virtdev'\n"
5481 if !$devinfo->{$devname};
5482 if (defined($opts->{storage
})) {
5483 $storeid = $opts->{storage
} || 'local';
5484 } elsif (!$storeid) {
5487 $format = 'raw' if !$format;
5488 $devinfo->{$devname}->{devname
} = $devname;
5489 $devinfo->{$devname}->{virtdev
} = $virtdev;
5490 $devinfo->{$devname}->{format
} = $format;
5491 $devinfo->{$devname}->{storeid
} = $storeid;
5493 # check permission on storage
5494 my $pool = $opts->{pool
}; # todo: do we need that?
5495 if ($user ne 'root@pam') {
5496 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5499 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5503 foreach my $devname (keys %$devinfo) {
5504 die "found no device mapping information for device '$devname'\n"
5505 if !$devinfo->{$devname}->{virtdev
};
5508 my $cfg = PVE
::Storage
::config
();
5510 # create empty/temp config
5512 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
5513 foreach_drive
($oldconf, sub {
5514 my ($ds, $drive) = @_;
5516 return if drive_is_cdrom
($drive);
5518 my $volid = $drive->{file
};
5520 return if !$volid || $volid =~ m
|^/|;
5522 my ($path, $owner) = PVE
::Storage
::path
($cfg, $volid);
5523 return if !$path || !$owner || ($owner != $vmid);
5525 # Note: only delete disk we want to restore
5526 # other volumes will become unused
5527 if ($virtdev_hash->{$ds}) {
5528 PVE
::Storage
::vdisk_free
($cfg, $volid);
5532 # delete vmstate files
5533 # since after the restore we have no snapshots anymore
5534 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
5535 my $snap = $oldconf->{snapshots
}->{$snapname};
5536 if ($snap->{vmstate
}) {
5537 eval { PVE
::Storage
::vdisk_free
($cfg, $snap->{vmstate
}); };
5546 foreach my $virtdev (sort keys %$virtdev_hash) {
5547 my $d = $virtdev_hash->{$virtdev};
5548 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
5549 my $scfg = PVE
::Storage
::storage_config
($cfg, $d->{storeid
});
5551 # test if requested format is supported
5552 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($cfg, $d->{storeid
});
5553 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
5554 $d->{format
} = $defFormat if !$supported;
5556 my $volid = PVE
::Storage
::vdisk_alloc
($cfg, $d->{storeid
}, $vmid,
5557 $d->{format
}, undef, $alloc_size);
5558 print STDERR
"new volume ID is '$volid'\n";
5559 $d->{volid
} = $volid;
5560 my $path = PVE
::Storage
::path
($cfg, $volid);
5562 PVE
::Storage
::activate_volumes
($cfg,[$volid]);
5564 my $write_zeros = 1;
5565 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
5569 print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5571 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5572 $map->{$virtdev} = $volid;
5575 $fh->seek(0, 0) || die "seek failed - $!\n";
5577 my $outfd = new IO
::File
($tmpfn, "w") ||
5578 die "unable to write config for VM $vmid\n";
5580 my $cookie = { netcount
=> 0 };
5581 while (defined(my $line = <$fh>)) {
5582 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
5591 local $SIG{INT
} = $SIG{TERM
} = $SIG{QUIT
} = $SIG{HUP
} = $SIG{PIPE
} = sub {
5592 die "interrupted by signal\n";
5594 local $SIG{ALRM
} = sub { die "got timeout\n"; };
5596 $oldtimeout = alarm($timeout);
5603 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5604 my ($dev_id, $size, $devname) = ($1, $2, $3);
5605 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
5606 } elsif ($line =~ m/^CTIME: /) {
5607 # we correctly received the vma config, so we can disable
5608 # the timeout now for disk allocation (set to 10 minutes, so
5609 # that we always timeout if something goes wrong)
5612 print $fifofh "done\n";
5613 my $tmp = $oldtimeout || 0;
5614 $oldtimeout = undef;
5620 print "restore vma archive: $cmd\n";
5621 run_command
($cmd, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
5625 alarm($oldtimeout) if $oldtimeout;
5628 foreach my $devname (keys %$devinfo) {
5629 my $volid = $devinfo->{$devname}->{volid
};
5630 push @$vollist, $volid if $volid;
5633 my $cfg = PVE
::Storage
::config
();
5634 PVE
::Storage
::deactivate_volumes
($cfg, $vollist);
5642 foreach my $devname (keys %$devinfo) {
5643 my $volid = $devinfo->{$devname}->{volid
};
5646 if ($volid =~ m
|^/|) {
5647 unlink $volid || die 'unlink failed\n';
5649 PVE
::Storage
::vdisk_free
($cfg, $volid);
5651 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
5653 print STDERR
"unable to cleanup '$volid' - $@" if $@;
5660 rename($tmpfn, $conffile) ||
5661 die "unable to commit configuration file '$conffile'\n";
5663 PVE
::Cluster
::cfs_update
(); # make sure we read new file
5665 eval { rescan
($vmid, 1); };
5669 sub restore_tar_archive
{
5670 my ($archive, $vmid, $user, $opts) = @_;
5672 if ($archive ne '-') {
5673 my $firstfile = tar_archive_read_firstfile
($archive);
5674 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5675 if $firstfile ne 'qemu-server.conf';
5678 my $storecfg = PVE
::Storage
::config
();
5680 # destroy existing data - keep empty config
5681 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
5682 destroy_vm
($storecfg, $vmid, 1) if -f
$vmcfgfn;
5684 my $tocmd = "/usr/lib/qemu-server/qmextract";
5686 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
5687 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
5688 $tocmd .= ' --prealloc' if $opts->{prealloc
};
5689 $tocmd .= ' --info' if $opts->{info
};
5691 # tar option "xf" does not autodetect compression when read from STDIN,
5692 # so we pipe to zcat
5693 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
5694 PVE
::Tools
::shellquote
("--to-command=$tocmd");
5696 my $tmpdir = "/var/tmp/vzdumptmp$$";
5699 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
5700 local $ENV{VZDUMP_VMID
} = $vmid;
5701 local $ENV{VZDUMP_USER
} = $user;
5703 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
5704 my $tmpfn = "$conffile.$$.tmp";
5706 # disable interrupts (always do cleanups)
5707 local $SIG{INT
} = $SIG{TERM
} = $SIG{QUIT
} = $SIG{HUP
} = sub {
5708 print STDERR
"got interrupt - ignored\n";
5713 local $SIG{INT
} = $SIG{TERM
} = $SIG{QUIT
} = $SIG{HUP
} = $SIG{PIPE
} = sub {
5714 die "interrupted by signal\n";
5717 if ($archive eq '-') {
5718 print "extracting archive from STDIN\n";
5719 run_command
($cmd, input
=> "<&STDIN");
5721 print "extracting archive '$archive'\n";
5725 return if $opts->{info
};
5729 my $statfile = "$tmpdir/qmrestore.stat";
5730 if (my $fd = IO
::File-
>new($statfile, "r")) {
5731 while (defined (my $line = <$fd>)) {
5732 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5733 $map->{$1} = $2 if $1;
5735 print STDERR
"unable to parse line in statfile - $line\n";
5741 my $confsrc = "$tmpdir/qemu-server.conf";
5743 my $srcfd = new IO
::File
($confsrc, "r") ||
5744 die "unable to open file '$confsrc'\n";
5746 my $outfd = new IO
::File
($tmpfn, "w") ||
5747 die "unable to write config for VM $vmid\n";
5749 my $cookie = { netcount
=> 0 };
5750 while (defined (my $line = <$srcfd>)) {
5751 restore_update_config_line
($outfd, $cookie, $vmid, $map, $line, $opts->{unique
});
5763 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
5770 rename $tmpfn, $conffile ||
5771 die "unable to commit configuration file '$conffile'\n";
5773 PVE
::Cluster
::cfs_update
(); # make sure we read new file
5775 eval { rescan
($vmid, 1); };
5779 sub foreach_writable_storage
{
5780 my ($conf, $func) = @_;
5784 foreach my $ds (keys %$conf) {
5785 next if !is_valid_drivename
($ds);
5787 my $drive = parse_drive
($ds, $conf->{$ds});
5789 next if drive_is_cdrom
($drive);
5791 my $volid = $drive->{file
};
5793 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5794 $sidhash->{$sid} = $sid if $sid;
5797 foreach my $sid (sort keys %$sidhash) {
5802 sub do_snapshots_with_qemu
{
5803 my ($storecfg, $volid) = @_;
5805 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
5807 if ($qemu_snap_storage->{$storecfg->{ids
}->{$storage_name}->{type
}}
5808 && !$storecfg->{ids
}->{$storage_name}->{krbd
}){
5812 if ($volid =~ m/\.(qcow2|qed)$/){
5819 sub qga_check_running
{
5822 eval { vm_mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
5824 warn "Qemu Guest Agent are not running - $@";
5830 sub template_create
{
5831 my ($vmid, $conf, $disk) = @_;
5833 my $storecfg = PVE
::Storage
::config
();
5835 foreach_drive
($conf, sub {
5836 my ($ds, $drive) = @_;
5838 return if drive_is_cdrom
($drive);
5839 return if $disk && $ds ne $disk;
5841 my $volid = $drive->{file
};
5842 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
5844 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
5845 $drive->{file
} = $voliddst;
5846 $conf->{$ds} = print_drive
($vmid, $drive);
5847 PVE
::QemuConfig-
>write_config($vmid, $conf);
5851 sub qemu_img_convert
{
5852 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5854 my $storecfg = PVE
::Storage
::config
();
5855 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
5856 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
5858 if ($src_storeid && $dst_storeid) {
5860 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
5862 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
5863 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
5865 my $src_format = qemu_img_format
($src_scfg, $src_volname);
5866 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
5868 my $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
5869 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
5872 push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5873 push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5874 push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5875 if ($is_zero_initialized) {
5876 push @$cmd, "zeroinit:$dst_path";
5878 push @$cmd, $dst_path;
5883 if($line =~ m/\((\S+)\/100\
%\)/){
5885 my $transferred = int($size * $percent / 100);
5886 my $remaining = $size - $transferred;
5888 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5893 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
5895 die "copy failed: $err" if $err;
5899 sub qemu_img_format
{
5900 my ($scfg, $volname) = @_;
5902 if ($scfg->{path
} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5909 sub qemu_drive_mirror
{
5910 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5912 my $storecfg = PVE
::Storage
::config
();
5913 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
5915 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
5917 my $format = qemu_img_format
($dst_scfg, $dst_volname);
5919 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
5921 my $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
5923 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
5924 $opts->{format
} = $format if $format;
5926 print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5928 my $finish_job = sub {
5930 my $stats = vm_mon_cmd
($vmid, "query-block-jobs");
5931 my $stat = @$stats[0];
5938 vm_mon_cmd
($vmid, "drive-mirror", %$opts);
5940 my $stats = vm_mon_cmd
($vmid, "query-block-jobs");
5941 my $stat = @$stats[0];
5942 die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5943 die "error job is not mirroring" if $stat->{type
} ne "mirror";
5945 my $busy = $stat->{busy
};
5946 my $ready = $stat->{ready
};
5948 if (my $total = $stat->{len
}) {
5949 my $transferred = $stat->{offset
} || 0;
5950 my $remaining = $total - $transferred;
5951 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5953 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5957 if ($stat->{ready
} eq 'true') {
5959 last if $vmiddst != $vmid;
5961 # try to switch the disk if source and destination are on the same guest
5962 eval { vm_mon_cmd
($vmid, "block-job-complete", device
=> "drive-$drive") };
5967 die $@ if $@ !~ m/cannot be completed/;
5976 my $cancel_job = sub {
5977 vm_mon_cmd
($vmid, "block-job-cancel", device
=> "drive-$drive");
5982 eval { &$cancel_job(); };
5983 die "mirroring error: $err";
5986 if ($vmiddst != $vmid) {
5987 # if we clone a disk for a new target vm, we don't switch the disk
5988 &$cancel_job(); # so we call block-job-cancel
5993 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5994 $newvmid, $storage, $format, $full, $newvollist) = @_;
5999 print "create linked clone of drive $drivename ($drive->{file})\n";
6000 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
6001 push @$newvollist, $newvolid;
6003 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6004 $storeid = $storage if $storage;
6006 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6008 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6009 $format = qemu_img_format
($scfg, $volname);
6012 # test if requested format is supported - else use default
6013 my $supported = grep { $_ eq $format } @$validFormats;
6014 $format = $defFormat if !$supported;
6016 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 3);
6018 print "create full clone of drive $drivename ($drive->{file})\n";
6019 $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6020 push @$newvollist, $newvolid;
6022 PVE
::Storage
::activate_volumes
($storecfg, $newvollist);
6024 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
6025 if (!$running || $snapname) {
6026 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
6028 qemu_drive_mirror
($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
6032 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 3);
6035 $disk->{format
} = undef;
6036 $disk->{file
} = $newvolid;
6037 $disk->{size
} = $size;
6042 # this only works if VM is running
6043 sub get_current_qemu_machine
{
6046 my $cmd = { execute
=> 'query-machines', arguments
=> {} };
6047 my $res = vm_qmp_command
($vmid, $cmd);
6049 my ($current, $default);
6050 foreach my $e (@$res) {
6051 $default = $e->{name
} if $e->{'is-default'};
6052 $current = $e->{name
} if $e->{'is-current'};
6055 # fallback to the default machine if current is not supported by qemu
6056 return $current || $default || 'pc';
6059 sub qemu_machine_feature_enabled
{
6060 my ($machine, $kvmver, $version_major, $version_minor) = @_;
6065 if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6067 $current_major = $3;
6068 $current_minor = $4;
6070 } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6072 $current_major = $1;
6073 $current_minor = $2;
6076 return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6081 sub qemu_machine_pxe
{
6082 my ($vmid, $conf, $machine) = @_;
6084 $machine = PVE
::QemuServer
::get_current_qemu_machine
($vmid) if !$machine;
6086 foreach my $opt (keys %$conf) {
6087 next if $opt !~ m/^net(\d+)$/;
6088 my $net = PVE
::QemuServer
::parse_net
($conf->{$opt});
6090 my $romfile = PVE
::QemuServer
::vm_mon_cmd_nocheck
($vmid, 'qom-get', path
=> $opt, property
=> 'romfile');
6091 return $machine.".pxe" if $romfile =~ m/pxe/;
6098 sub qemu_use_old_bios_files
{
6099 my ($machine_type) = @_;
6101 return if !$machine_type;
6103 my $use_old_bios_files = undef;
6105 if ($machine_type =~ m/^(\S+)\.pxe$/) {
6107 $use_old_bios_files = 1;
6109 my $kvmver = kvm_user_version
();
6110 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6111 # load new efi bios files on migration. So this hack is required to allow
6112 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6113 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6114 $use_old_bios_files = !qemu_machine_feature_enabled
($machine_type, $kvmver, 2, 4);
6117 return ($use_old_bios_files, $machine_type);
6124 dir_glob_foreach
("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6125 my (undef, $id, $function) = @_;
6126 my $res = { id
=> $id, function
=> $function};
6127 push @{$devices->{$id}}, $res;
6133 sub vm_iothreads_list
{
6136 my $res = vm_mon_cmd
($vmid, 'query-iothreads');
6139 foreach my $iothread (@$res) {
6140 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
6147 my ($conf, $drive) = @_;
6151 if ($conf->{scsihw
} && ($conf->{scsihw
} =~ m/^lsi/)) {
6153 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
6159 my $controller = int($drive->{index} / $maxdev);
6160 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single') ?
"virtioscsi" : "scsihw";
6162 return ($maxdev, $controller, $controller_prefix);
6165 # bash completion helper
6167 sub complete_backup_archives
{
6168 my ($cmdname, $pname, $cvalue) = @_;
6170 my $cfg = PVE
::Storage
::config
();
6174 if ($cvalue =~ m/^([^:]+):/) {
6178 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
6181 foreach my $id (keys %$data) {
6182 foreach my $item (@{$data->{$id}}) {
6183 next if $item->{format
} !~ m/^vma\.(gz|lzo)$/;
6184 push @$res, $item->{volid
} if defined($item->{volid
});
6191 my $complete_vmid_full = sub {
6194 my $idlist = vmstatus
();
6198 foreach my $id (keys %$idlist) {
6199 my $d = $idlist->{$id};
6200 if (defined($running)) {
6201 next if $d->{template
};
6202 next if $running && $d->{status
} ne 'running';
6203 next if !$running && $d->{status
} eq 'running';
6212 return &$complete_vmid_full();
6215 sub complete_vmid_stopped
{
6216 return &$complete_vmid_full(0);
6219 sub complete_vmid_running
{
6220 return &$complete_vmid_full(1);
6223 sub complete_storage
{
6225 my $cfg = PVE
::Storage
::config
();
6226 my $ids = $cfg->{ids
};
6229 foreach my $sid (keys %$ids) {
6230 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
6231 next if !$ids->{$sid}->{content
}->{images
};