PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  45
  46 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  47
  48 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  49
  50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  51
  52 # Note about locking: we use flock on the config file protect
  53 # against concurent actions.
  54 # Aditionaly, we have a 'lock' setting in the config file. This
  55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  56 # allowed when such lock is set. But you can ignore this kind of
  57 # lock with the --skiplock flag.
  58
  59 cfs_register_file('/qemu-server/',
  60                   \&parse_vm_config,
  61                   \&write_vm_config);
  62
  63 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  64     description => "Some command save/restore state from this location.",
  65     type => 'string',
  66     maxLength => 128,
  67     optional => 1,
  68 });
  69
  70 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  71     description => "The name of the snapshot.",
  72     type => 'string', format => 'pve-configid',
  73     maxLength => 40,
  74 });
  75
  76 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  77     type => 'string',
  78     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  79     description => "The drive's backing file's data format.",
  80     optional => 1,
  81 });
  82
  83 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  84         description => "Specifies the Qemu machine type.",
  85         type => 'string',
  86         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
  87         maxLength => 40,
  88         optional => 1,
  89 });
  90
  91 #no warnings 'redefine';
  92
  93 sub cgroups_write {
  94    my ($controller, $vmid, $option, $value) = @_;
  95
  96    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  97    PVE::ProcFSTools::write_proc_entry($path, $value);
  98
  99 }
 100
 101 my $nodename = PVE::INotify::nodename();
 102
 103 mkdir "/etc/pve/nodes/$nodename";
 104 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 105 mkdir $confdir;
 106
 107 my $var_run_tmpdir = "/var/run/qemu-server";
 108 mkdir $var_run_tmpdir;
 109
 110 my $lock_dir = "/var/lock/qemu-server";
 111 mkdir $lock_dir;
 112
 113 my $pcisysfs = "/sys/bus/pci";
 114
 115 my $cpu_vendor_list = {
 116     # Intel CPUs
 117     486 => 'GenuineIntel',
 118     pentium => 'GenuineIntel',
 119     pentium2  => 'GenuineIntel',
 120     pentium3  => 'GenuineIntel',
 121     coreduo => 'GenuineIntel',
 122     core2duo => 'GenuineIntel',
 123     Conroe  => 'GenuineIntel',
 124     Penryn  => 'GenuineIntel',
 125     Nehalem  => 'GenuineIntel',
 126     'Nehalem-IBRS'  => 'GenuineIntel',
 127     Westmere => 'GenuineIntel',
 128     'Westmere-IBRS' => 'GenuineIntel',
 129     SandyBridge => 'GenuineIntel',
 130     'SandyBridge-IBRS' => 'GenuineIntel',
 131     IvyBridge => 'GenuineIntel',
 132     'IvyBridge-IBRS' => 'GenuineIntel',
 133     Haswell => 'GenuineIntel',
 134     'Haswell-IBRS' => 'GenuineIntel',
 135     'Haswell-noTSX' => 'GenuineIntel',
 136     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 137     Broadwell => 'GenuineIntel',
 138     'Broadwell-IBRS' => 'GenuineIntel',
 139     'Broadwell-noTSX' => 'GenuineIntel',
 140     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 141     'Skylake-Client' => 'GenuineIntel',
 142     'Skylake-Client-IBRS' => 'GenuineIntel',
 143     'Skylake-Server' => 'GenuineIntel',
 144     'Skylake-Server-IBRS' => 'GenuineIntel',
 145
 146     # AMD CPUs
 147     athlon => 'AuthenticAMD',
 148     phenom  => 'AuthenticAMD',
 149     Opteron_G1  => 'AuthenticAMD',
 150     Opteron_G2  => 'AuthenticAMD',
 151     Opteron_G3  => 'AuthenticAMD',
 152     Opteron_G4  => 'AuthenticAMD',
 153     Opteron_G5  => 'AuthenticAMD',
 154     EPYC => 'AuthenticAMD',
 155     'EPYC-IBPB' => 'AuthenticAMD',
 156
 157     # generic types, use vendor from host node
 158     host => 'default',
 159     kvm32 => 'default',
 160     kvm64 => 'default',
 161     qemu32 => 'default',
 162     qemu64 => 'default',
 163     max => 'default',
 164 };
 165
 166 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 167
 168 my $cpu_fmt = {
 169     cputype => {
 170         description => "Emulated CPU type.",
 171         type => 'string',
 172         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 173         default => 'kvm64',
 174         default_key => 1,
 175     },
 176     hidden => {
 177         description => "Do not identify as a KVM virtual machine.",
 178         type => 'boolean',
 179         optional => 1,
 180         default => 0
 181     },
 182     flags => {
 183         description => "List of additional CPU flags separated by ';'."
 184                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 185                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 186         format_description => '+FLAG[;-FLAG...]',
 187         type => 'string',
 188         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 189         optional => 1,
 190     },
 191 };
 192
 193 my $watchdog_fmt = {
 194     model => {
 195         default_key => 1,
 196         type => 'string',
 197         enum => [qw(i6300esb ib700)],
 198         description => "Watchdog type to emulate.",
 199         default => 'i6300esb',
 200         optional => 1,
 201     },
 202     action => {
 203         type => 'string',
 204         enum => [qw(reset shutdown poweroff pause debug none)],
 205         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 206         optional => 1,
 207     },
 208 };
 209 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 210
 211 my $agent_fmt = {
 212     enabled => {
 213         description => "Enable/disable Qemu GuestAgent.",
 214         type => 'boolean',
 215         default => 0,
 216         default_key => 1,
 217     },
 218     fstrim_cloned_disks => {
 219         description => "Run fstrim after cloning/moving a disk.",
 220         type => 'boolean',
 221         optional => 1,
 222         default => 0
 223     },
 224 };
 225
 226 my $vga_fmt = {
 227     type => {
 228         description => "Select the VGA type.",
 229         type => 'string',
 230         default => 'std',
 231         optional => 1,
 232         default_key => 1,
 233         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 234     },
 235     memory => {
 236         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 237         type => 'integer',
 238         optional => 1,
 239         minimum => 4,
 240         maximum => 512,
 241     },
 242 };
 243
 244 my $confdesc = {
 245     onboot => {
 246         optional => 1,
 247         type => 'boolean',
 248         description => "Specifies whether a VM will be started during system bootup.",
 249         default => 0,
 250     },
 251     autostart => {
 252         optional => 1,
 253         type => 'boolean',
 254         description => "Automatic restart after crash (currently ignored).",
 255         default => 0,
 256     },
 257     hotplug => {
 258         optional => 1,
 259         type => 'string', format => 'pve-hotplug-features',
 260         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 261         default => 'network,disk,usb',
 262     },
 263     reboot => {
 264         optional => 1,
 265         type => 'boolean',
 266         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 267         default => 1,
 268     },
 269     lock => {
 270         optional => 1,
 271         type => 'string',
 272         description => "Lock/unlock the VM.",
 273         enum => [qw(migrate backup snapshot rollback)],
 274     },
 275     cpulimit => {
 276         optional => 1,
 277         type => 'number',
 278         description => "Limit of CPU usage.",
 279         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 280         minimum => 0,
 281         maximum => 128,
 282         default => 0,
 283     },
 284     cpuunits => {
 285         optional => 1,
 286         type => 'integer',
 287         description => "CPU weight for a VM.",
 288         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 289         minimum => 2,
 290         maximum => 262144,
 291         default => 1024,
 292     },
 293     memory => {
 294         optional => 1,
 295         type => 'integer',
 296         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 297         minimum => 16,
 298         default => 512,
 299     },
 300     balloon => {
 301         optional => 1,
 302         type => 'integer',
 303         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 304         minimum => 0,
 305     },
 306     shares => {
 307         optional => 1,
 308         type => 'integer',
 309         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 310         minimum => 0,
 311         maximum => 50000,
 312         default => 1000,
 313     },
 314     keyboard => {
 315         optional => 1,
 316         type => 'string',
 317         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 318                        "It should not be necessary to set it.",
 319         enum => PVE::Tools::kvmkeymaplist(),
 320         default => undef,
 321     },
 322     name => {
 323         optional => 1,
 324         type => 'string', format => 'dns-name',
 325         description => "Set a name for the VM. Only used on the configuration web interface.",
 326     },
 327     scsihw => {
 328         optional => 1,
 329         type => 'string',
 330         description => "SCSI controller model",
 331         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 332         default => 'lsi',
 333     },
 334     description => {
 335         optional => 1,
 336         type => 'string',
 337         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 338     },
 339     ostype => {
 340         optional => 1,
 341         type => 'string',
 342         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 343         description => "Specify guest operating system.",
 344         verbose_description => <<EODESC,
 345 Specify guest operating system. This is used to enable special
 346 optimization/features for specific operating systems:
 347
 348 [horizontal]
 349 other;; unspecified OS
 350 wxp;; Microsoft Windows XP
 351 w2k;; Microsoft Windows 2000
 352 w2k3;; Microsoft Windows 2003
 353 w2k8;; Microsoft Windows 2008
 354 wvista;; Microsoft Windows Vista
 355 win7;; Microsoft Windows 7
 356 win8;; Microsoft Windows 8/2012/2012r2
 357 win10;; Microsoft Windows 10/2016
 358 l24;; Linux 2.4 Kernel
 359 l26;; Linux 2.6/3.X Kernel
 360 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 361 EODESC
 362     },
 363     boot => {
 364         optional => 1,
 365         type => 'string',
 366         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 367         pattern => '[acdn]{1,4}',
 368         default => 'cdn',
 369     },
 370     bootdisk => {
 371         optional => 1,
 372         type => 'string', format => 'pve-qm-bootdisk',
 373         description => "Enable booting from specified disk.",
 374         pattern => '(ide|sata|scsi|virtio)\d+',
 375     },
 376     smp => {
 377         optional => 1,
 378         type => 'integer',
 379         description => "The number of CPUs. Please use option -sockets instead.",
 380         minimum => 1,
 381         default => 1,
 382     },
 383     sockets => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "The number of CPU sockets.",
 387         minimum => 1,
 388         default => 1,
 389     },
 390     cores => {
 391         optional => 1,
 392         type => 'integer',
 393         description => "The number of cores per socket.",
 394         minimum => 1,
 395         default => 1,
 396     },
 397     numa => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Enable/disable NUMA.",
 401         default => 0,
 402     },
 403     hugepages => {
 404         optional => 1,
 405         type => 'string',
 406         description => "Enable/disable hugepages memory.",
 407         enum => [qw(any 2 1024)],
 408     },
 409     vcpus => {
 410         optional => 1,
 411         type => 'integer',
 412         description => "Number of hotplugged vcpus.",
 413         minimum => 1,
 414         default => 0,
 415     },
 416     acpi => {
 417         optional => 1,
 418         type => 'boolean',
 419         description => "Enable/disable ACPI.",
 420         default => 1,
 421     },
 422     agent => {
 423         optional => 1,
 424         description => "Enable/disable Qemu GuestAgent and its properties.",
 425         type => 'string',
 426         format => $agent_fmt,
 427     },
 428     kvm => {
 429         optional => 1,
 430         type => 'boolean',
 431         description => "Enable/disable KVM hardware virtualization.",
 432         default => 1,
 433     },
 434     tdf => {
 435         optional => 1,
 436         type => 'boolean',
 437         description => "Enable/disable time drift fix.",
 438         default => 0,
 439     },
 440     localtime => {
 441         optional => 1,
 442         type => 'boolean',
 443         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 444     },
 445     freeze => {
 446         optional => 1,
 447         type => 'boolean',
 448         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 449     },
 450     vga => {
 451         optional => 1,
 452         type => 'string', format => $vga_fmt,
 453         description => "Configure the VGA hardware.",
 454         verbose_description => "Configure the VGA Hardware. If you want to use ".
 455             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 456             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 457             "is 'std' for all OS types besides some Windows versions (XP and " .
 458             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 459             "display server. For win* OS you can select how many independent " .
 460             "displays you want, Linux guests can add displays them self.\n".
 461             "You can also run without any graphic card, using a serial device as terminal.",
 462     },
 463     watchdog => {
 464         optional => 1,
 465         type => 'string', format => 'pve-qm-watchdog',
 466         description => "Create a virtual hardware watchdog device.",
 467         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 468             " (by a guest action), the watchdog must be periodically polled " .
 469             "by an agent inside the guest or else the watchdog will reset " .
 470             "the guest (or execute the respective action specified)",
 471     },
 472     startdate => {
 473         optional => 1,
 474         type => 'string',
 475         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 476         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 477         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 478         default => 'now',
 479     },
 480     startup =>  get_standard_option('pve-startup-order'),
 481     template => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Enable/disable Template.",
 485         default => 0,
 486     },
 487     args => {
 488         optional => 1,
 489         type => 'string',
 490         description => "Arbitrary arguments passed to kvm.",
 491         verbose_description => <<EODESCR,
 492 Arbitrary arguments passed to kvm, for example:
 493
 494 args: -no-reboot -no-hpet
 495
 496 NOTE: this option is for experts only.
 497 EODESCR
 498     },
 499     tablet => {
 500         optional => 1,
 501         type => 'boolean',
 502         default => 1,
 503         description => "Enable/disable the USB tablet device.",
 504         verbose_description => "Enable/disable the USB tablet device. This device is " .
 505             "usually needed to allow absolute mouse positioning with VNC. " .
 506             "Else the mouse runs out of sync with normal VNC clients. " .
 507             "If you're running lots of console-only guests on one host, " .
 508             "you may consider disabling this to save some context switches. " .
 509             "This is turned off by default if you use spice (-vga=qxl).",
 510     },
 511     migrate_speed => {
 512         optional => 1,
 513         type => 'integer',
 514         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 515         minimum => 0,
 516         default => 0,
 517     },
 518     migrate_downtime => {
 519         optional => 1,
 520         type => 'number',
 521         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 522         minimum => 0,
 523         default => 0.1,
 524     },
 525     cdrom => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-ide',
 528         typetext => '<volume>',
 529         description => "This is an alias for option -ide2",
 530     },
 531     cpu => {
 532         optional => 1,
 533         description => "Emulated CPU type.",
 534         type => 'string',
 535         format => $cpu_fmt,
 536     },
 537     parent => get_standard_option('pve-snapshot-name', {
 538         optional => 1,
 539         description => "Parent snapshot name. This is used internally, and should not be modified.",
 540     }),
 541     snaptime => {
 542         optional => 1,
 543         description => "Timestamp for snapshots.",
 544         type => 'integer',
 545         minimum => 0,
 546     },
 547     vmstate => {
 548         optional => 1,
 549         type => 'string', format => 'pve-volume-id',
 550         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 551     },
 552     vmstatestorage => get_standard_option('pve-storage-id', {
 553         description => "Default storage for VM state volumes/files.",
 554         optional => 1,
 555     }),
 556     runningmachine => get_standard_option('pve-qemu-machine', {
 557         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 558     }),
 559     machine => get_standard_option('pve-qemu-machine'),
 560     smbios1 => {
 561         description => "Specify SMBIOS type 1 fields.",
 562         type => 'string', format => 'pve-qm-smbios1',
 563         maxLength => 256,
 564         optional => 1,
 565     },
 566     protection => {
 567         optional => 1,
 568         type => 'boolean',
 569         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 570         default => 0,
 571     },
 572     bios => {
 573         optional => 1,
 574         type => 'string',
 575         enum => [ qw(seabios ovmf) ],
 576         description => "Select BIOS implementation.",
 577         default => 'seabios',
 578     },
 579     vmgenid => {
 580         type => 'string',
 581         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 582         format_description => 'UUID',
 583         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 584         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 585             " 128-bit integer value identifier to the guest OS. This allows to".
 586             " notify the guest operating system when the virtual machine is".
 587             " executed with a different configuration (e.g. snapshot execution".
 588             " or creation from a template). The guest operating system notices".
 589             " the change, and is then able to react as appropriate by marking".
 590             " its copies of distributed databases as dirty, re-initializing its".
 591             " random number generator, etc.\n".
 592             "Note that auto-creation only works when done throug API/CLI create".
 593             " or update methods, but not when manually editing the config file.",
 594         default => "1 (autogenerated)",
 595         optional => 1,
 596     },
 597 };
 598
 599 my $confdesc_cloudinit = {
 600     citype => {
 601         optional => 1,
 602         type => 'string',
 603         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 604         enum => ['configdrive2', 'nocloud'],
 605     },
 606     ciuser => {
 607         optional => 1,
 608         type => 'string',
 609         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 610     },
 611     cipassword => {
 612         optional => 1,
 613         type => 'string',
 614         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 615     },
 616     searchdomain => {
 617         optional => 1,
 618         type => 'string',
 619         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 620     },
 621     nameserver => {
 622         optional => 1,
 623         type => 'string', format => 'address-list',
 624         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 625     },
 626     sshkeys => {
 627         optional => 1,
 628         type => 'string',
 629         format => 'urlencoded',
 630         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 631     },
 632 };
 633
 634 # what about other qemu settings ?
 635 #cpu => 'string',
 636 #machine => 'string',
 637 #fda => 'file',
 638 #fdb => 'file',
 639 #mtdblock => 'file',
 640 #sd => 'file',
 641 #pflash => 'file',
 642 #snapshot => 'bool',
 643 #bootp => 'file',
 644 ##tftp => 'dir',
 645 ##smb => 'dir',
 646 #kernel => 'file',
 647 #append => 'string',
 648 #initrd => 'file',
 649 ##soundhw => 'string',
 650
 651 while (my ($k, $v) = each %$confdesc) {
 652     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 653 }
 654
 655 my $MAX_IDE_DISKS = 4;
 656 my $MAX_SCSI_DISKS = 14;
 657 my $MAX_VIRTIO_DISKS = 16;
 658 my $MAX_SATA_DISKS = 6;
 659 my $MAX_USB_DEVICES = 5;
 660 my $MAX_NETS = 32;
 661 my $MAX_UNUSED_DISKS = 256;
 662 my $MAX_HOSTPCI_DEVICES = 4;
 663 my $MAX_SERIAL_PORTS = 4;
 664 my $MAX_PARALLEL_PORTS = 3;
 665 my $MAX_NUMA = 8;
 666
 667 my $numa_fmt = {
 668     cpus => {
 669         type => "string",
 670         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 671         description => "CPUs accessing this NUMA node.",
 672         format_description => "id[-id];...",
 673     },
 674     memory => {
 675         type => "number",
 676         description => "Amount of memory this NUMA node provides.",
 677         optional => 1,
 678     },
 679     hostnodes => {
 680         type => "string",
 681         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 682         description => "Host NUMA nodes to use.",
 683         format_description => "id[-id];...",
 684         optional => 1,
 685     },
 686     policy => {
 687         type => 'string',
 688         enum => [qw(preferred bind interleave)],
 689         description => "NUMA allocation policy.",
 690         optional => 1,
 691     },
 692 };
 693 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 694 my $numadesc = {
 695     optional => 1,
 696     type => 'string', format => $numa_fmt,
 697     description => "NUMA topology.",
 698 };
 699 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 700
 701 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 702     $confdesc->{"numa$i"} = $numadesc;
 703 }
 704
 705 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 706                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 707                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 708 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 709
 710 my $net_fmt_bridge_descr = <<__EOD__;
 711 Bridge to attach the network device to. The Proxmox VE standard bridge
 712 is called 'vmbr0'.
 713
 714 If you do not specify a bridge, we create a kvm user (NATed) network
 715 device, which provides DHCP and DNS services. The following addresses
 716 are used:
 717
 718  10.0.2.2   Gateway
 719  10.0.2.3   DNS Server
 720  10.0.2.4   SMB Server
 721
 722 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 723 __EOD__
 724
 725 my $net_fmt = {
 726     macaddr => {
 727         type => 'string',
 728         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 729         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 730         format_description => "XX:XX:XX:XX:XX:XX",
 731         optional => 1,
 732     },
 733     model => {
 734         type => 'string',
 735         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 736         enum => $nic_model_list,
 737         default_key => 1,
 738     },
 739     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 740     bridge => {
 741         type => 'string',
 742         description => $net_fmt_bridge_descr,
 743         format_description => 'bridge',
 744         optional => 1,
 745     },
 746     queues => {
 747         type => 'integer',
 748         minimum => 0, maximum => 16,
 749         description => 'Number of packet queues to be used on the device.',
 750         optional => 1,
 751     },
 752     rate => {
 753         type => 'number',
 754         minimum => 0,
 755         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 756         optional => 1,
 757     },
 758     tag => {
 759         type => 'integer',
 760         minimum => 1, maximum => 4094,
 761         description => 'VLAN tag to apply to packets on this interface.',
 762         optional => 1,
 763     },
 764     trunks => {
 765         type => 'string',
 766         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 767         description => 'VLAN trunks to pass through this interface.',
 768         format_description => 'vlanid[;vlanid...]',
 769         optional => 1,
 770     },
 771     firewall => {
 772         type => 'boolean',
 773         description => 'Whether this interface should be protected by the firewall.',
 774         optional => 1,
 775     },
 776     link_down => {
 777         type => 'boolean',
 778         description => 'Whether this interface should be disconnected (like pulling the plug).',
 779         optional => 1,
 780     },
 781 };
 782
 783 my $netdesc = {
 784     optional => 1,
 785     type => 'string', format => $net_fmt,
 786     description => "Specify network devices.",
 787 };
 788
 789 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 790
 791 my $ipconfig_fmt = {
 792     ip => {
 793         type => 'string',
 794         format => 'pve-ipv4-config',
 795         format_description => 'IPv4Format/CIDR',
 796         description => 'IPv4 address in CIDR format.',
 797         optional => 1,
 798         default => 'dhcp',
 799     },
 800     gw => {
 801         type => 'string',
 802         format => 'ipv4',
 803         format_description => 'GatewayIPv4',
 804         description => 'Default gateway for IPv4 traffic.',
 805         optional => 1,
 806         requires => 'ip',
 807     },
 808     ip6 => {
 809         type => 'string',
 810         format => 'pve-ipv6-config',
 811         format_description => 'IPv6Format/CIDR',
 812         description => 'IPv6 address in CIDR format.',
 813         optional => 1,
 814         default => 'dhcp',
 815     },
 816     gw6 => {
 817         type => 'string',
 818         format => 'ipv6',
 819         format_description => 'GatewayIPv6',
 820         description => 'Default gateway for IPv6 traffic.',
 821         optional => 1,
 822         requires => 'ip6',
 823     },
 824 };
 825 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 826 my $ipconfigdesc = {
 827     optional => 1,
 828     type => 'string', format => 'pve-qm-ipconfig',
 829     description => <<'EODESCR',
 830 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 831
 832 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 833
 834 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 835 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 836
 837 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 838 EODESCR
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 841
 842 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 843     $confdesc->{"net$i"} = $netdesc;
 844     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 845 }
 846
 847 foreach my $key (keys %$confdesc_cloudinit) {
 848     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 849 }
 850
 851 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 852 sub verify_volume_id_or_qm_path {
 853     my ($volid, $noerr) = @_;
 854
 855     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 856         return $volid;
 857     }
 858
 859     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 860     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 861     if ($@) {
 862         return undef if $noerr;
 863         die $@;
 864     }
 865     return $volid;
 866 }
 867
 868 my $drivename_hash;
 869
 870 my %drivedesc_base = (
 871     volume => { alias => 'file' },
 872     file => {
 873         type => 'string',
 874         format => 'pve-volume-id-or-qm-path',
 875         default_key => 1,
 876         format_description => 'volume',
 877         description => "The drive's backing volume.",
 878     },
 879     media => {
 880         type => 'string',
 881         enum => [qw(cdrom disk)],
 882         description => "The drive's media type.",
 883         default => 'disk',
 884         optional => 1
 885     },
 886     cyls => {
 887         type => 'integer',
 888         description => "Force the drive's physical geometry to have a specific cylinder count.",
 889         optional => 1
 890     },
 891     heads => {
 892         type => 'integer',
 893         description => "Force the drive's physical geometry to have a specific head count.",
 894         optional => 1
 895     },
 896     secs => {
 897         type => 'integer',
 898         description => "Force the drive's physical geometry to have a specific sector count.",
 899         optional => 1
 900     },
 901     trans => {
 902         type => 'string',
 903         enum => [qw(none lba auto)],
 904         description => "Force disk geometry bios translation mode.",
 905         optional => 1,
 906     },
 907     snapshot => {
 908         type => 'boolean',
 909         description => "Controls qemu's snapshot mode feature."
 910             . " If activated, changes made to the disk are temporary and will"
 911             . " be discarded when the VM is shutdown.",
 912         optional => 1,
 913     },
 914     cache => {
 915         type => 'string',
 916         enum => [qw(none writethrough writeback unsafe directsync)],
 917         description => "The drive's cache mode",
 918         optional => 1,
 919     },
 920     format => get_standard_option('pve-qm-image-format'),
 921     size => {
 922         type => 'string',
 923         format => 'disk-size',
 924         format_description => 'DiskSize',
 925         description => "Disk size. This is purely informational and has no effect.",
 926         optional => 1,
 927     },
 928     backup => {
 929         type => 'boolean',
 930         description => "Whether the drive should be included when making backups.",
 931         optional => 1,
 932     },
 933     replicate => {
 934         type => 'boolean',
 935         description => 'Whether the drive should considered for replication jobs.',
 936         optional => 1,
 937         default => 1,
 938     },
 939     rerror => {
 940         type => 'string',
 941         enum => [qw(ignore report stop)],
 942         description => 'Read error action.',
 943         optional => 1,
 944     },
 945     werror => {
 946         type => 'string',
 947         enum => [qw(enospc ignore report stop)],
 948         description => 'Write error action.',
 949         optional => 1,
 950     },
 951     aio => {
 952         type => 'string',
 953         enum => [qw(native threads)],
 954         description => 'AIO type to use.',
 955         optional => 1,
 956     },
 957     discard => {
 958         type => 'string',
 959         enum => [qw(ignore on)],
 960         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 961         optional => 1,
 962     },
 963     detect_zeroes => {
 964         type => 'boolean',
 965         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 966         optional => 1,
 967     },
 968     serial => {
 969         type => 'string',
 970         format => 'urlencoded',
 971         format_description => 'serial',
 972         maxLength => 20*3, # *3 since it's %xx url enoded
 973         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 974         optional => 1,
 975     },
 976     shared => {
 977         type => 'boolean',
 978         description => 'Mark this locally-managed volume as available on all nodes',
 979         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 980         optional => 1,
 981         default => 0,
 982     }
 983 );
 984
 985 my %iothread_fmt = ( iothread => {
 986         type => 'boolean',
 987         description => "Whether to use iothreads for this drive",
 988         optional => 1,
 989 });
 990
 991 my %model_fmt = (
 992     model => {
 993         type => 'string',
 994         format => 'urlencoded',
 995         format_description => 'model',
 996         maxLength => 40*3, # *3 since it's %xx url enoded
 997         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 998         optional => 1,
 999     },
1000 );
1001
1002 my %queues_fmt = (
1003     queues => {
1004         type => 'integer',
1005         description => "Number of queues.",
1006         minimum => 2,
1007         optional => 1
1008     }
1009 );
1010
1011 my %scsiblock_fmt = (
1012     scsiblock => {
1013         type => 'boolean',
1014         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1015         optional => 1,
1016         default => 0,
1017     },
1018 );
1019
1020 my %ssd_fmt = (
1021     ssd => {
1022         type => 'boolean',
1023         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1024         optional => 1,
1025     },
1026 );
1027
1028 my $add_throttle_desc = sub {
1029     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1030     my $d = {
1031         type => $type,
1032         format_description => $unit,
1033         description => "Maximum $what in $longunit.",
1034         optional => 1,
1035     };
1036     $d->{minimum} = $minimum if defined($minimum);
1037     $drivedesc_base{$key} = $d;
1038 };
1039 # throughput: (leaky bucket)
1040 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1041 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1042 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1043 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1044 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1045 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1046 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1047 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1048 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1049
1050 # pools: (pool of IO before throttling starts taking effect)
1051 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1052 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1053 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1054 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1055 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1056 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1057
1058 # burst lengths
1059 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1060 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1061 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1062 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1063 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1064 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1065
1066 # legacy support
1067 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1068 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1069 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1070 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1071
1072 my $ide_fmt = {
1073     %drivedesc_base,
1074     %model_fmt,
1075     %ssd_fmt,
1076 };
1077 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1078
1079 my $idedesc = {
1080     optional => 1,
1081     type => 'string', format => $ide_fmt,
1082     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1083 };
1084 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1085
1086 my $scsi_fmt = {
1087     %drivedesc_base,
1088     %iothread_fmt,
1089     %queues_fmt,
1090     %scsiblock_fmt,
1091     %ssd_fmt,
1092 };
1093 my $scsidesc = {
1094     optional => 1,
1095     type => 'string', format => $scsi_fmt,
1096     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1097 };
1098 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1099
1100 my $sata_fmt = {
1101     %drivedesc_base,
1102     %ssd_fmt,
1103 };
1104 my $satadesc = {
1105     optional => 1,
1106     type => 'string', format => $sata_fmt,
1107     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1108 };
1109 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1110
1111 my $virtio_fmt = {
1112     %drivedesc_base,
1113     %iothread_fmt,
1114 };
1115 my $virtiodesc = {
1116     optional => 1,
1117     type => 'string', format => $virtio_fmt,
1118     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1119 };
1120 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1121
1122 my $alldrive_fmt = {
1123     %drivedesc_base,
1124     %iothread_fmt,
1125     %model_fmt,
1126     %queues_fmt,
1127     %scsiblock_fmt,
1128     %ssd_fmt,
1129 };
1130
1131 my $efidisk_fmt = {
1132     volume => { alias => 'file' },
1133     file => {
1134         type => 'string',
1135         format => 'pve-volume-id-or-qm-path',
1136         default_key => 1,
1137         format_description => 'volume',
1138         description => "The drive's backing volume.",
1139     },
1140     format => get_standard_option('pve-qm-image-format'),
1141     size => {
1142         type => 'string',
1143         format => 'disk-size',
1144         format_description => 'DiskSize',
1145         description => "Disk size. This is purely informational and has no effect.",
1146         optional => 1,
1147     },
1148 };
1149
1150 my $efidisk_desc = {
1151     optional => 1,
1152     type => 'string', format => $efidisk_fmt,
1153     description => "Configure a Disk for storing EFI vars",
1154 };
1155
1156 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1157
1158 my $usb_fmt = {
1159     host => {
1160         default_key => 1,
1161         type => 'string', format => 'pve-qm-usb-device',
1162         format_description => 'HOSTUSBDEVICE|spice',
1163         description => <<EODESCR,
1164 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1165
1166  'bus-port(.port)*' (decimal numbers) or
1167  'vendor_id:product_id' (hexadeciaml numbers) or
1168  'spice'
1169
1170 You can use the 'lsusb -t' command to list existing usb devices.
1171
1172 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1173
1174 The value 'spice' can be used to add a usb redirection devices for spice.
1175 EODESCR
1176     },
1177     usb3 => {
1178         optional => 1,
1179         type => 'boolean',
1180         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1181         default => 0,
1182     },
1183 };
1184
1185 my $usbdesc = {
1186     optional => 1,
1187     type => 'string', format => $usb_fmt,
1188     description => "Configure an USB device (n is 0 to 4).",
1189 };
1190 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1191
1192 # NOTE: the match-groups of this regex are used in parse_hostpci
1193 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1194 my $hostpci_fmt = {
1195     host => {
1196         default_key => 1,
1197         type => 'string',
1198         pattern => qr/$PCIRE(;$PCIRE)*/,
1199         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1200         description => <<EODESCR,
1201 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1202 of PCI virtual functions of the host. HOSTPCIID syntax is:
1203
1204 'bus:dev.func' (hexadecimal numbers)
1205
1206 You can us the 'lspci' command to list existing PCI devices.
1207 EODESCR
1208     },
1209     rombar => {
1210         type => 'boolean',
1211         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1212         optional => 1,
1213         default => 1,
1214     },
1215     romfile => {
1216         type => 'string',
1217         pattern => '[^,;]+',
1218         format_description => 'string',
1219         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1220         optional => 1,
1221     },
1222     pcie => {
1223         type => 'boolean',
1224         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1225         optional => 1,
1226         default => 0,
1227     },
1228     'x-vga' => {
1229         type => 'boolean',
1230         description =>  "Enable vfio-vga device support.",
1231         optional => 1,
1232         default => 0,
1233     },
1234 };
1235 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1236
1237 my $hostpcidesc = {
1238         optional => 1,
1239         type => 'string', format => 'pve-qm-hostpci',
1240         description => "Map host PCI devices into guest.",
1241         verbose_description =>  <<EODESCR,
1242 Map host PCI devices into guest.
1243
1244 NOTE: This option allows direct access to host hardware. So it is no longer
1245 possible to migrate such machines - use with special care.
1246
1247 CAUTION: Experimental! User reported problems with this option.
1248 EODESCR
1249 };
1250 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1251
1252 my $serialdesc = {
1253         optional => 1,
1254         type => 'string',
1255         pattern => '(/dev/.+|socket)',
1256         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1257         verbose_description =>  <<EODESCR,
1258 Create a serial device inside the VM (n is 0 to 3), and pass through a
1259 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1260 host side (use 'qm terminal' to open a terminal connection).
1261
1262 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1263
1264 CAUTION: Experimental! User reported problems with this option.
1265 EODESCR
1266 };
1267
1268 my $paralleldesc= {
1269         optional => 1,
1270         type => 'string',
1271         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1272         description =>  "Map host parallel devices (n is 0 to 2).",
1273         verbose_description =>  <<EODESCR,
1274 Map host parallel devices (n is 0 to 2).
1275
1276 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1277
1278 CAUTION: Experimental! User reported problems with this option.
1279 EODESCR
1280 };
1281
1282 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1283     $confdesc->{"parallel$i"} = $paralleldesc;
1284 }
1285
1286 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1287     $confdesc->{"serial$i"} = $serialdesc;
1288 }
1289
1290 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1291     $confdesc->{"hostpci$i"} = $hostpcidesc;
1292 }
1293
1294 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1295     $drivename_hash->{"ide$i"} = 1;
1296     $confdesc->{"ide$i"} = $idedesc;
1297 }
1298
1299 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1300     $drivename_hash->{"sata$i"} = 1;
1301     $confdesc->{"sata$i"} = $satadesc;
1302 }
1303
1304 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1305     $drivename_hash->{"scsi$i"} = 1;
1306     $confdesc->{"scsi$i"} = $scsidesc ;
1307 }
1308
1309 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1310     $drivename_hash->{"virtio$i"} = 1;
1311     $confdesc->{"virtio$i"} = $virtiodesc;
1312 }
1313
1314 $drivename_hash->{efidisk0} = 1;
1315 $confdesc->{efidisk0} = $efidisk_desc;
1316
1317 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1318     $confdesc->{"usb$i"} = $usbdesc;
1319 }
1320
1321 my $unuseddesc = {
1322     optional => 1,
1323     type => 'string', format => 'pve-volume-id',
1324     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1325 };
1326
1327 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1328     $confdesc->{"unused$i"} = $unuseddesc;
1329 }
1330
1331 my $kvm_api_version = 0;
1332
1333 sub kvm_version {
1334     return $kvm_api_version if $kvm_api_version;
1335
1336     open my $fh, '<', '/dev/kvm'
1337         or return undef;
1338
1339     # 0xae00 => KVM_GET_API_VERSION
1340     $kvm_api_version = ioctl($fh, 0xae00, 0);
1341
1342     return $kvm_api_version;
1343 }
1344
1345 my $kvm_user_version;
1346
1347 sub kvm_user_version {
1348
1349     return $kvm_user_version if $kvm_user_version;
1350
1351     $kvm_user_version = 'unknown';
1352
1353     my $code = sub {
1354         my $line = shift;
1355         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1356             $kvm_user_version = $2;
1357         }
1358     };
1359
1360     eval { run_command("kvm -version", outfunc => $code); };
1361     warn $@ if $@;
1362
1363     return $kvm_user_version;
1364
1365 }
1366
1367 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1368
1369 sub valid_drive_names {
1370     # order is important - used to autoselect boot disk
1371     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1372             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1373             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1374             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1375             'efidisk0');
1376 }
1377
1378 sub is_valid_drivename {
1379     my $dev = shift;
1380
1381     return defined($drivename_hash->{$dev});
1382 }
1383
1384 sub option_exists {
1385     my $key = shift;
1386     return defined($confdesc->{$key});
1387 }
1388
1389 sub nic_models {
1390     return $nic_model_list;
1391 }
1392
1393 sub os_list_description {
1394
1395     return {
1396         other => 'Other',
1397         wxp => 'Windows XP',
1398         w2k => 'Windows 2000',
1399         w2k3 =>, 'Windows 2003',
1400         w2k8 => 'Windows 2008',
1401         wvista => 'Windows Vista',
1402         win7 => 'Windows 7',
1403         win8 => 'Windows 8/2012',
1404         win10 => 'Windows 10/2016',
1405         l24 => 'Linux 2.4',
1406         l26 => 'Linux 2.6',
1407     };
1408 }
1409
1410 my $cdrom_path;
1411
1412 sub get_cdrom_path {
1413
1414     return  $cdrom_path if $cdrom_path;
1415
1416     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1417     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1418     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1419 }
1420
1421 sub get_iso_path {
1422     my ($storecfg, $vmid, $cdrom) = @_;
1423
1424     if ($cdrom eq 'cdrom') {
1425         return get_cdrom_path();
1426     } elsif ($cdrom eq 'none') {
1427         return '';
1428     } elsif ($cdrom =~ m|^/|) {
1429         return $cdrom;
1430     } else {
1431         return PVE::Storage::path($storecfg, $cdrom);
1432     }
1433 }
1434
1435 # try to convert old style file names to volume IDs
1436 sub filename_to_volume_id {
1437     my ($vmid, $file, $media) = @_;
1438
1439      if (!($file eq 'none' || $file eq 'cdrom' ||
1440           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1441
1442         return undef if $file =~ m|/|;
1443
1444         if ($media && $media eq 'cdrom') {
1445             $file = "local:iso/$file";
1446         } else {
1447             $file = "local:$vmid/$file";
1448         }
1449     }
1450
1451     return $file;
1452 }
1453
1454 sub verify_media_type {
1455     my ($opt, $vtype, $media) = @_;
1456
1457     return if !$media;
1458
1459     my $etype;
1460     if ($media eq 'disk') {
1461         $etype = 'images';
1462     } elsif ($media eq 'cdrom') {
1463         $etype = 'iso';
1464     } else {
1465         die "internal error";
1466     }
1467
1468     return if ($vtype eq $etype);
1469
1470     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1471 }
1472
1473 sub cleanup_drive_path {
1474     my ($opt, $storecfg, $drive) = @_;
1475
1476     # try to convert filesystem paths to volume IDs
1477
1478     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1479         ($drive->{file} !~ m|^/dev/.+|) &&
1480         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1481         ($drive->{file} !~ m/^\d+$/)) {
1482         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1483         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1484         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1485         verify_media_type($opt, $vtype, $drive->{media});
1486         $drive->{file} = $volid;
1487     }
1488
1489     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1490 }
1491
1492 sub parse_hotplug_features {
1493     my ($data) = @_;
1494
1495     my $res = {};
1496
1497     return $res if $data eq '0';
1498
1499     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1500
1501     foreach my $feature (PVE::Tools::split_list($data)) {
1502         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1503             $res->{$1} = 1;
1504         } else {
1505             die "invalid hotplug feature '$feature'\n";
1506         }
1507     }
1508     return $res;
1509 }
1510
1511 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1512 sub pve_verify_hotplug_features {
1513     my ($value, $noerr) = @_;
1514
1515     return $value if parse_hotplug_features($value);
1516
1517     return undef if $noerr;
1518
1519     die "unable to parse hotplug option\n";
1520 }
1521
1522 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1523 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1524 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1525 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1526 #        [,iothread=on][,serial=serial][,model=model]
1527
1528 sub parse_drive {
1529     my ($key, $data) = @_;
1530
1531     my ($interface, $index);
1532
1533     if ($key =~ m/^([^\d]+)(\d+)$/) {
1534         $interface = $1;
1535         $index = $2;
1536     } else {
1537         return undef;
1538     }
1539
1540     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1541                                      : $confdesc->{$key}->{format};
1542     if (!$desc) {
1543         warn "invalid drive key: $key\n";
1544         return undef;
1545     }
1546     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1547     return undef if !$res;
1548     $res->{interface} = $interface;
1549     $res->{index} = $index;
1550
1551     my $error = 0;
1552     foreach my $opt (qw(bps bps_rd bps_wr)) {
1553         if (my $bps = defined(delete $res->{$opt})) {
1554             if (defined($res->{"m$opt"})) {
1555                 warn "both $opt and m$opt specified\n";
1556                 ++$error;
1557                 next;
1558             }
1559             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1560         }
1561     }
1562
1563     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1564     for my $requirement (
1565         [mbps_max => 'mbps'],
1566         [mbps_rd_max => 'mbps_rd'],
1567         [mbps_wr_max => 'mbps_wr'],
1568         [miops_max => 'miops'],
1569         [miops_rd_max => 'miops_rd'],
1570         [miops_wr_max => 'miops_wr'],
1571         [bps_max_length => 'mbps_max'],
1572         [bps_rd_max_length => 'mbps_rd_max'],
1573         [bps_wr_max_length => 'mbps_wr_max'],
1574         [iops_max_length => 'iops_max'],
1575         [iops_rd_max_length => 'iops_rd_max'],
1576         [iops_wr_max_length => 'iops_wr_max']) {
1577         my ($option, $requires) = @$requirement;
1578         if ($res->{$option} && !$res->{$requires}) {
1579             warn "$option requires $requires\n";
1580             ++$error;
1581         }
1582     }
1583
1584     return undef if $error;
1585
1586     return undef if $res->{mbps_rd} && $res->{mbps};
1587     return undef if $res->{mbps_wr} && $res->{mbps};
1588     return undef if $res->{iops_rd} && $res->{iops};
1589     return undef if $res->{iops_wr} && $res->{iops};
1590
1591     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1592         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1593         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1594         return undef if $res->{interface} eq 'virtio';
1595     }
1596
1597     if (my $size = $res->{size}) {
1598         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1599     }
1600
1601     return $res;
1602 }
1603
1604 sub print_drive {
1605     my ($vmid, $drive) = @_;
1606     my $data = { %$drive };
1607     delete $data->{$_} for qw(index interface);
1608     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1609 }
1610
1611 sub scsi_inquiry {
1612     my($fh, $noerr) = @_;
1613
1614     my $SG_IO = 0x2285;
1615     my $SG_GET_VERSION_NUM = 0x2282;
1616
1617     my $versionbuf = "\x00" x 8;
1618     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1619     if (!$ret) {
1620         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1621         return undef;
1622     }
1623     my $version = unpack("I", $versionbuf);
1624     if ($version < 30000) {
1625         die "scsi generic interface too old\n"  if !$noerr;
1626         return undef;
1627     }
1628
1629     my $buf = "\x00" x 36;
1630     my $sensebuf = "\x00" x 8;
1631     my $cmd = pack("C x3 C x1", 0x12, 36);
1632
1633     # see /usr/include/scsi/sg.h
1634     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1635
1636     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1637                       length($sensebuf), 0, length($buf), $buf,
1638                       $cmd, $sensebuf, 6000);
1639
1640     $ret = ioctl($fh, $SG_IO, $packet);
1641     if (!$ret) {
1642         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1643         return undef;
1644     }
1645
1646     my @res = unpack($sg_io_hdr_t, $packet);
1647     if ($res[17] || $res[18]) {
1648         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1649         return undef;
1650     }
1651
1652     my $res = {};
1653     (my $byte0, my $byte1, $res->{vendor},
1654      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1655
1656     $res->{removable} = $byte1 & 128 ? 1 : 0;
1657     $res->{type} = $byte0 & 31;
1658
1659     return $res;
1660 }
1661
1662 sub path_is_scsi {
1663     my ($path) = @_;
1664
1665     my $fh = IO::File->new("+<$path") || return undef;
1666     my $res = scsi_inquiry($fh, 1);
1667     close($fh);
1668
1669     return $res;
1670 }
1671
1672 sub machine_type_is_q35 {
1673     my ($conf) = @_;
1674
1675     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1676 }
1677
1678 sub print_tabletdevice_full {
1679     my ($conf) = @_;
1680
1681     my $q35 = machine_type_is_q35($conf);
1682
1683     # we use uhci for old VMs because tablet driver was buggy in older qemu
1684     my $usbbus = $q35 ? "ehci" : "uhci";
1685
1686     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1687 }
1688
1689 sub print_drivedevice_full {
1690     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1691
1692     my $device = '';
1693     my $maxdev = 0;
1694
1695     if ($drive->{interface} eq 'virtio') {
1696         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1697         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1698         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1699     } elsif ($drive->{interface} eq 'scsi') {
1700
1701         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1702         my $unit = $drive->{index} % $maxdev;
1703         my $devicetype = 'hd';
1704         my $path = '';
1705         if (drive_is_cdrom($drive)) {
1706             $devicetype = 'cd';
1707         } else {
1708             if ($drive->{file} =~ m|^/|) {
1709                 $path = $drive->{file};
1710                 if (my $info = path_is_scsi($path)) {
1711                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1712                         $devicetype = 'block';
1713                     } elsif ($info->{type} == 1) { # tape
1714                         $devicetype = 'generic';
1715                     }
1716                 }
1717             } else {
1718                  $path = PVE::Storage::path($storecfg, $drive->{file});
1719             }
1720
1721             if($path =~ m/^iscsi\:\/\//){
1722                 $devicetype = 'generic';
1723             }
1724         }
1725
1726         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1727             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1728         } else {
1729             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1730         }
1731
1732         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1733             $device .= ",rotation_rate=1";
1734         }
1735
1736     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1737         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1738         my $controller = int($drive->{index} / $maxdev);
1739         my $unit = $drive->{index} % $maxdev;
1740         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1741
1742         $device = "ide-$devicetype";
1743         if ($drive->{interface} eq 'ide') {
1744             $device .= ",bus=ide.$controller,unit=$unit";
1745         } else {
1746             $device .= ",bus=ahci$controller.$unit";
1747         }
1748         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1749
1750         if ($devicetype eq 'hd') {
1751             if (my $model = $drive->{model}) {
1752                 $model = URI::Escape::uri_unescape($model);
1753                 $device .= ",model=$model";
1754             }
1755             if ($drive->{ssd}) {
1756                 $device .= ",rotation_rate=1";
1757             }
1758         }
1759     } elsif ($drive->{interface} eq 'usb') {
1760         die "implement me";
1761         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1762     } else {
1763         die "unsupported interface type";
1764     }
1765
1766     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1767
1768     if (my $serial = $drive->{serial}) {
1769         $serial = URI::Escape::uri_unescape($serial);
1770         $device .= ",serial=$serial";
1771     }
1772
1773
1774     return $device;
1775 }
1776
1777 sub get_initiator_name {
1778     my $initiator;
1779
1780     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1781     while (defined(my $line = <$fh>)) {
1782         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1783         $initiator = $1;
1784         last;
1785     }
1786     $fh->close();
1787
1788     return $initiator;
1789 }
1790
1791 sub print_drive_full {
1792     my ($storecfg, $vmid, $drive) = @_;
1793
1794     my $path;
1795     my $volid = $drive->{file};
1796     my $format;
1797
1798     if (drive_is_cdrom($drive)) {
1799         $path = get_iso_path($storecfg, $vmid, $volid);
1800     } else {
1801         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1802         if ($storeid) {
1803             $path = PVE::Storage::path($storecfg, $volid);
1804             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1805             $format = qemu_img_format($scfg, $volname);
1806         } else {
1807             $path = $volid;
1808             $format = "raw";
1809         }
1810    }
1811
1812     my $opts = '';
1813     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1814     foreach my $o (@qemu_drive_options) {
1815         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1816     }
1817
1818     # snapshot only accepts on|off
1819     if (defined($drive->{snapshot})) {
1820         my $v = $drive->{snapshot} ? 'on' : 'off';
1821         $opts .= ",snapshot=$v";
1822     }
1823
1824     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1825         my ($dir, $qmpname) = @$type;
1826         if (my $v = $drive->{"mbps$dir"}) {
1827             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1828         }
1829         if (my $v = $drive->{"mbps${dir}_max"}) {
1830             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1831         }
1832         if (my $v = $drive->{"bps${dir}_max_length"}) {
1833             $opts .= ",throttling.bps$qmpname-max-length=$v";
1834         }
1835         if (my $v = $drive->{"iops${dir}"}) {
1836             $opts .= ",throttling.iops$qmpname=$v";
1837         }
1838         if (my $v = $drive->{"iops${dir}_max"}) {
1839             $opts .= ",throttling.iops$qmpname-max=$v";
1840         }
1841         if (my $v = $drive->{"iops${dir}_max_length"}) {
1842             $opts .= ",throttling.iops$qmpname-max-length=$v";
1843         }
1844     }
1845
1846     $opts .= ",format=$format" if $format && !$drive->{format};
1847
1848     my $cache_direct = 0;
1849
1850     if (my $cache = $drive->{cache}) {
1851         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1852     } elsif (!drive_is_cdrom($drive)) {
1853         $opts .= ",cache=none";
1854         $cache_direct = 1;
1855     }
1856
1857     # aio native works only with O_DIRECT
1858     if (!$drive->{aio}) {
1859         if($cache_direct) {
1860             $opts .= ",aio=native";
1861         } else {
1862             $opts .= ",aio=threads";
1863         }
1864     }
1865
1866     if (!drive_is_cdrom($drive)) {
1867         my $detectzeroes;
1868         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1869             $detectzeroes = 'off';
1870         } elsif ($drive->{discard}) {
1871             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1872         } else {
1873             # This used to be our default with discard not being specified:
1874             $detectzeroes = 'on';
1875         }
1876         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1877     }
1878
1879     my $pathinfo = $path ? "file=$path," : '';
1880
1881     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1882 }
1883
1884 sub print_netdevice_full {
1885     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1886
1887     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1888
1889     my $device = $net->{model};
1890     if ($net->{model} eq 'virtio') {
1891          $device = 'virtio-net-pci';
1892      };
1893
1894     my $pciaddr = print_pci_addr("$netid", $bridges);
1895     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1896     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1897         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1898         my $vectors = $net->{queues} * 2 + 2;
1899         $tmpstr .= ",vectors=$vectors,mq=on";
1900     }
1901     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1902
1903     if ($use_old_bios_files) {
1904         my $romfile;
1905         if ($device eq 'virtio-net-pci') {
1906             $romfile = 'pxe-virtio.rom';
1907         } elsif ($device eq 'e1000') {
1908             $romfile = 'pxe-e1000.rom';
1909         } elsif ($device eq 'ne2k') {
1910             $romfile = 'pxe-ne2k_pci.rom';
1911         } elsif ($device eq 'pcnet') {
1912             $romfile = 'pxe-pcnet.rom';
1913         } elsif ($device eq 'rtl8139') {
1914             $romfile = 'pxe-rtl8139.rom';
1915         }
1916         $tmpstr .= ",romfile=$romfile" if $romfile;
1917     }
1918
1919     return $tmpstr;
1920 }
1921
1922 sub print_netdev_full {
1923     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1924
1925     my $i = '';
1926     if ($netid =~ m/^net(\d+)$/) {
1927         $i = int($1);
1928     }
1929
1930     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1931
1932     my $ifname = "tap${vmid}i$i";
1933
1934     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1935     die "interface name '$ifname' is too long (max 15 character)\n"
1936         if length($ifname) >= 16;
1937
1938     my $vhostparam = '';
1939     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1940
1941     my $vmname = $conf->{name} || "vm$vmid";
1942
1943     my $netdev = "";
1944     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1945
1946     if ($net->{bridge}) {
1947         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1948     } else {
1949         $netdev = "type=user,id=$netid,hostname=$vmname";
1950     }
1951
1952     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1953
1954     return $netdev;
1955 }
1956
1957
1958 sub print_cpu_device {
1959     my ($conf, $id) = @_;
1960
1961     my $kvm = $conf->{kvm} // 1;
1962     my $cpu = $kvm ? "kvm64" : "qemu64";
1963     if (my $cputype = $conf->{cpu}) {
1964         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1965             or die "Cannot parse cpu description: $cputype\n";
1966         $cpu = $cpuconf->{cputype};
1967     }
1968
1969     my $cores = $conf->{cores} || 1;
1970
1971     my $current_core = ($id - 1) % $cores;
1972     my $current_socket = int(($id - 1 - $current_core)/$cores);
1973
1974     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1975 }
1976
1977 my $vga_map = {
1978     'cirrus' => 'cirrus-vga',
1979     'std' => 'VGA',
1980     'vmware' => 'vmware-svga',
1981     'virtio' => 'virtio-vga',
1982 };
1983
1984 sub print_vga_device {
1985     my ($conf, $vga, $id, $qxlnum, $bridges) = @_;
1986
1987     my $type = $vga_map->{$vga->{type}};
1988     my $vgamem_mb = $vga->{memory};
1989     if ($qxlnum) {
1990         $type = $id ? 'qxl' : 'qxl-vga';
1991     }
1992     die "no devicetype for $vga->{type}\n" if !$type;
1993
1994     my $memory = "";
1995     if ($vgamem_mb) {
1996         if ($vga->{type} eq 'virtio') {
1997             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1998             $memory = ",max_hostmem=$bytes";
1999         } elsif ($qxlnum) {
2000             # from https://www.spice-space.org/multiple-monitors.html
2001             $memory = ",vgamem_mb=$vga->{memory}";
2002             my $ram = $vgamem_mb * 4;
2003             my $vram = $vgamem_mb * 2;
2004             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2005         } else {
2006             $memory = ",vgamem_mb=$vga->{memory}";
2007         }
2008     } elsif ($qxlnum && $id) {
2009         $memory = ",ram_size=67108864,vram_size=33554432";
2010     }
2011
2012     my $q35 = machine_type_is_q35($conf);
2013     my $vgaid = "vga" . ($id // '');
2014     my $pciaddr;
2015
2016     if ($q35 && $vgaid eq 'vga') {
2017         # the first display uses pcie.0 bus on q35 machines
2018         $pciaddr = print_pcie_addr($vgaid, $bridges);
2019     } else {
2020         $pciaddr = print_pci_addr($vgaid, $bridges);
2021     }
2022
2023     return "$type,id=${vgaid}${memory}${pciaddr}";
2024 }
2025
2026 sub drive_is_cloudinit {
2027     my ($drive) = @_;
2028     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2029 }
2030
2031 sub drive_is_cdrom {
2032     my ($drive, $exclude_cloudinit) = @_;
2033
2034     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2035
2036     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2037
2038 }
2039
2040 sub parse_number_sets {
2041     my ($set) = @_;
2042     my $res = [];
2043     foreach my $part (split(/;/, $set)) {
2044         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2045             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2046             push @$res, [ $1, $2 ];
2047         } else {
2048             die "invalid range: $part\n";
2049         }
2050     }
2051     return $res;
2052 }
2053
2054 sub parse_numa {
2055     my ($data) = @_;
2056
2057     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2058     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2059     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2060     return $res;
2061 }
2062
2063 sub parse_hostpci {
2064     my ($value) = @_;
2065
2066     return undef if !$value;
2067
2068     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2069
2070     my @idlist = split(/;/, $res->{host});
2071     delete $res->{host};
2072     foreach my $id (@idlist) {
2073         if ($id =~ /^$PCIRE$/) {
2074             if (defined($2)) {
2075                 push @{$res->{pciid}}, { id => $1, function => $2 };
2076             } else {
2077                 my $pcidevices = lspci($1);
2078                 $res->{pciid} = $pcidevices->{$1};
2079             }
2080         } else {
2081             # should have been caught by parse_property_string already
2082             die "failed to parse PCI id: $id\n";
2083         }
2084     }
2085     return $res;
2086 }
2087
2088 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2089 sub parse_net {
2090     my ($data) = @_;
2091
2092     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2093     if ($@) {
2094         warn $@;
2095         return undef;
2096     }
2097     if (!defined($res->{macaddr})) {
2098         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2099         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2100     }
2101     return $res;
2102 }
2103
2104 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2105 sub parse_ipconfig {
2106     my ($data) = @_;
2107
2108     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2109     if ($@) {
2110         warn $@;
2111         return undef;
2112     }
2113
2114     if ($res->{gw} && !$res->{ip}) {
2115         warn 'gateway specified without specifying an IP address';
2116         return undef;
2117     }
2118     if ($res->{gw6} && !$res->{ip6}) {
2119         warn 'IPv6 gateway specified without specifying an IPv6 address';
2120         return undef;
2121     }
2122     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2123         warn 'gateway specified together with DHCP';
2124         return undef;
2125     }
2126     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2127         # gw6 + auto/dhcp
2128         warn "IPv6 gateway specified together with $res->{ip6} address";
2129         return undef;
2130     }
2131
2132     if (!$res->{ip} && !$res->{ip6}) {
2133         return { ip => 'dhcp', ip6 => 'dhcp' };
2134     }
2135
2136     return $res;
2137 }
2138
2139 sub print_net {
2140     my $net = shift;
2141
2142     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2143 }
2144
2145 sub add_random_macs {
2146     my ($settings) = @_;
2147
2148     foreach my $opt (keys %$settings) {
2149         next if $opt !~ m/^net(\d+)$/;
2150         my $net = parse_net($settings->{$opt});
2151         next if !$net;
2152         $settings->{$opt} = print_net($net);
2153     }
2154 }
2155
2156 sub vm_is_volid_owner {
2157     my ($storecfg, $vmid, $volid) = @_;
2158
2159     if ($volid !~  m|^/|) {
2160         my ($path, $owner);
2161         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2162         if ($owner && ($owner == $vmid)) {
2163             return 1;
2164         }
2165     }
2166
2167     return undef;
2168 }
2169
2170 sub split_flagged_list {
2171     my $text = shift || '';
2172     $text =~ s/[,;]/ /g;
2173     $text =~ s/^\s+//;
2174     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2175 }
2176
2177 sub join_flagged_list {
2178     my ($how, $lst) = @_;
2179     join $how, map { $lst->{$_} . $_ } keys %$lst;
2180 }
2181
2182 sub vmconfig_delete_pending_option {
2183     my ($conf, $key, $force) = @_;
2184
2185     delete $conf->{pending}->{$key};
2186     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2187     $pending_delete_hash->{$key} = $force ? '!' : '';
2188     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2189 }
2190
2191 sub vmconfig_undelete_pending_option {
2192     my ($conf, $key) = @_;
2193
2194     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2195     delete $pending_delete_hash->{$key};
2196
2197     if (%$pending_delete_hash) {
2198         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2199     } else {
2200         delete $conf->{pending}->{delete};
2201     }
2202 }
2203
2204 sub vmconfig_register_unused_drive {
2205     my ($storecfg, $vmid, $conf, $drive) = @_;
2206
2207     if (drive_is_cloudinit($drive)) {
2208         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2209         warn $@ if $@;
2210     } elsif (!drive_is_cdrom($drive)) {
2211         my $volid = $drive->{file};
2212         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2213             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2214         }
2215     }
2216 }
2217
2218 sub vmconfig_cleanup_pending {
2219     my ($conf) = @_;
2220
2221     # remove pending changes when nothing changed
2222     my $changes;
2223     foreach my $opt (keys %{$conf->{pending}}) {
2224         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2225             $changes = 1;
2226             delete $conf->{pending}->{$opt};
2227         }
2228     }
2229
2230     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2231     my $pending_delete_hash = {};
2232     while (my ($opt, $force) = each %$current_delete_hash) {
2233         if (defined($conf->{$opt})) {
2234             $pending_delete_hash->{$opt} = $force;
2235         } else {
2236             $changes = 1;
2237         }
2238     }
2239
2240     if (%$pending_delete_hash) {
2241         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2242     } else {
2243         delete $conf->{pending}->{delete};
2244     }
2245
2246     return $changes;
2247 }
2248
2249 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2250 my $smbios1_fmt = {
2251     uuid => {
2252         type => 'string',
2253         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2254         format_description => 'UUID',
2255         description => "Set SMBIOS1 UUID.",
2256         optional => 1,
2257     },
2258     version => {
2259         type => 'string',
2260         pattern => '\S+',
2261         format_description => 'string',
2262         description => "Set SMBIOS1 version.",
2263         optional => 1,
2264     },
2265     serial => {
2266         type => 'string',
2267         pattern => '\S+',
2268         format_description => 'string',
2269         description => "Set SMBIOS1 serial number.",
2270         optional => 1,
2271     },
2272     manufacturer => {
2273         type => 'string',
2274         pattern => '\S+',
2275         format_description => 'string',
2276         description => "Set SMBIOS1 manufacturer.",
2277         optional => 1,
2278     },
2279     product => {
2280         type => 'string',
2281         pattern => '\S+',
2282         format_description => 'string',
2283         description => "Set SMBIOS1 product ID.",
2284         optional => 1,
2285     },
2286     sku => {
2287         type => 'string',
2288         pattern => '\S+',
2289         format_description => 'string',
2290         description => "Set SMBIOS1 SKU string.",
2291         optional => 1,
2292     },
2293     family => {
2294         type => 'string',
2295         pattern => '\S+',
2296         format_description => 'string',
2297         description => "Set SMBIOS1 family string.",
2298         optional => 1,
2299     },
2300 };
2301
2302 sub parse_smbios1 {
2303     my ($data) = @_;
2304
2305     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2306     warn $@ if $@;
2307     return $res;
2308 }
2309
2310 sub print_smbios1 {
2311     my ($smbios1) = @_;
2312     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2313 }
2314
2315 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2316
2317 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2318 sub verify_bootdisk {
2319     my ($value, $noerr) = @_;
2320
2321     return $value if is_valid_drivename($value);
2322
2323     return undef if $noerr;
2324
2325     die "invalid boot disk '$value'\n";
2326 }
2327
2328 sub parse_watchdog {
2329     my ($value) = @_;
2330
2331     return undef if !$value;
2332
2333     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2334     warn $@ if $@;
2335     return $res;
2336 }
2337
2338 sub parse_guest_agent {
2339     my ($value) = @_;
2340
2341     return {} if !defined($value->{agent});
2342
2343     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2344     warn $@ if $@;
2345
2346     # if the agent is disabled ignore the other potentially set properties
2347     return {} if !$res->{enabled};
2348     return $res;
2349 }
2350
2351 sub parse_vga {
2352     my ($value) = @_;
2353
2354     return {} if !$value;
2355     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2356     warn $@ if $@;
2357     return $res;
2358 }
2359
2360 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2361 sub verify_usb_device {
2362     my ($value, $noerr) = @_;
2363
2364     return $value if parse_usb_device($value);
2365
2366     return undef if $noerr;
2367
2368     die "unable to parse usb device\n";
2369 }
2370
2371 # add JSON properties for create and set function
2372 sub json_config_properties {
2373     my $prop = shift;
2374
2375     foreach my $opt (keys %$confdesc) {
2376         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2377         $prop->{$opt} = $confdesc->{$opt};
2378     }
2379
2380     return $prop;
2381 }
2382
2383 # return copy of $confdesc_cloudinit to generate documentation
2384 sub cloudinit_config_properties {
2385
2386     return dclone($confdesc_cloudinit);
2387 }
2388
2389 sub check_type {
2390     my ($key, $value) = @_;
2391
2392     die "unknown setting '$key'\n" if !$confdesc->{$key};
2393
2394     my $type = $confdesc->{$key}->{type};
2395
2396     if (!defined($value)) {
2397         die "got undefined value\n";
2398     }
2399
2400     if ($value =~ m/[\n\r]/) {
2401         die "property contains a line feed\n";
2402     }
2403
2404     if ($type eq 'boolean') {
2405         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2406         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2407         die "type check ('boolean') failed - got '$value'\n";
2408     } elsif ($type eq 'integer') {
2409         return int($1) if $value =~ m/^(\d+)$/;
2410         die "type check ('integer') failed - got '$value'\n";
2411     } elsif ($type eq 'number') {
2412         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2413         die "type check ('number') failed - got '$value'\n";
2414     } elsif ($type eq 'string') {
2415         if (my $fmt = $confdesc->{$key}->{format}) {
2416             PVE::JSONSchema::check_format($fmt, $value);
2417             return $value;
2418         }
2419         $value =~ s/^\"(.*)\"$/$1/;
2420         return $value;
2421     } else {
2422         die "internal error"
2423     }
2424 }
2425
2426 sub check_iommu_support{
2427     #fixme : need to check IOMMU support
2428     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2429
2430     my $iommu=1;
2431     return $iommu;
2432
2433 }
2434
2435 sub touch_config {
2436     my ($vmid) = @_;
2437
2438     my $conf = PVE::QemuConfig->config_file($vmid);
2439     utime undef, undef, $conf;
2440 }
2441
2442 sub destroy_vm {
2443     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2444
2445     my $conffile = PVE::QemuConfig->config_file($vmid);
2446
2447     my $conf = PVE::QemuConfig->load_config($vmid);
2448
2449     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2450
2451     if ($conf->{template}) {
2452         # check if any base image is still used by a linked clone
2453         foreach_drive($conf, sub {
2454                 my ($ds, $drive) = @_;
2455
2456                 return if drive_is_cdrom($drive);
2457
2458                 my $volid = $drive->{file};
2459
2460                 return if !$volid || $volid =~ m|^/|;
2461
2462                 die "base volume '$volid' is still in use by linked cloned\n"
2463                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2464
2465         });
2466     }
2467
2468     # only remove disks owned by this VM
2469     foreach_drive($conf, sub {
2470         my ($ds, $drive) = @_;
2471
2472         return if drive_is_cdrom($drive, 1);
2473
2474         my $volid = $drive->{file};
2475
2476         return if !$volid || $volid =~ m|^/|;
2477
2478         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2479         return if !$path || !$owner || ($owner != $vmid);
2480
2481         eval {
2482             PVE::Storage::vdisk_free($storecfg, $volid);
2483         };
2484         warn "Could not remove disk '$volid', check manually: $@" if $@;
2485
2486     });
2487
2488     if ($keep_empty_config) {
2489         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2490     } else {
2491         unlink $conffile;
2492     }
2493
2494     # also remove unused disk
2495     eval {
2496         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2497
2498         eval {
2499             PVE::Storage::foreach_volid($dl, sub {
2500                 my ($volid, $sid, $volname, $d) = @_;
2501                 PVE::Storage::vdisk_free($storecfg, $volid);
2502             });
2503         };
2504         warn $@ if $@;
2505
2506     };
2507     warn $@ if $@;
2508 }
2509
2510 sub parse_vm_config {
2511     my ($filename, $raw) = @_;
2512
2513     return undef if !defined($raw);
2514
2515     my $res = {
2516         digest => Digest::SHA::sha1_hex($raw),
2517         snapshots => {},
2518         pending => {},
2519     };
2520
2521     $filename =~ m|/qemu-server/(\d+)\.conf$|
2522         || die "got strange filename '$filename'";
2523
2524     my $vmid = $1;
2525
2526     my $conf = $res;
2527     my $descr;
2528     my $section = '';
2529
2530     my @lines = split(/\n/, $raw);
2531     foreach my $line (@lines) {
2532         next if $line =~ m/^\s*$/;
2533
2534         if ($line =~ m/^\[PENDING\]\s*$/i) {
2535             $section = 'pending';
2536             if (defined($descr)) {
2537                 $descr =~ s/\s+$//;
2538                 $conf->{description} = $descr;
2539             }
2540             $descr = undef;
2541             $conf = $res->{$section} = {};
2542             next;
2543
2544         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2545             $section = $1;
2546             if (defined($descr)) {
2547                 $descr =~ s/\s+$//;
2548                 $conf->{description} = $descr;
2549             }
2550             $descr = undef;
2551             $conf = $res->{snapshots}->{$section} = {};
2552             next;
2553         }
2554
2555         if ($line =~ m/^\#(.*)\s*$/) {
2556             $descr = '' if !defined($descr);
2557             $descr .= PVE::Tools::decode_text($1) . "\n";
2558             next;
2559         }
2560
2561         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2562             $descr = '' if !defined($descr);
2563             $descr .= PVE::Tools::decode_text($2);
2564         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2565             $conf->{snapstate} = $1;
2566         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2567             my $key = $1;
2568             my $value = $2;
2569             $conf->{$key} = $value;
2570         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2571             my $value = $1;
2572             if ($section eq 'pending') {
2573                 $conf->{delete} = $value; # we parse this later
2574             } else {
2575                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2576             }
2577         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2578             my $key = $1;
2579             my $value = $2;
2580             eval { $value = check_type($key, $value); };
2581             if ($@) {
2582                 warn "vm $vmid - unable to parse value of '$key' - $@";
2583             } else {
2584                 $key = 'ide2' if $key eq 'cdrom';
2585                 my $fmt = $confdesc->{$key}->{format};
2586                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2587                     my $v = parse_drive($key, $value);
2588                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2589                         $v->{file} = $volid;
2590                         $value = print_drive($vmid, $v);
2591                     } else {
2592                         warn "vm $vmid - unable to parse value of '$key'\n";
2593                         next;
2594                     }
2595                 }
2596
2597                 $conf->{$key} = $value;
2598             }
2599         }
2600     }
2601
2602     if (defined($descr)) {
2603         $descr =~ s/\s+$//;
2604         $conf->{description} = $descr;
2605     }
2606     delete $res->{snapstate}; # just to be sure
2607
2608     return $res;
2609 }
2610
2611 sub write_vm_config {
2612     my ($filename, $conf) = @_;
2613
2614     delete $conf->{snapstate}; # just to be sure
2615
2616     if ($conf->{cdrom}) {
2617         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2618         $conf->{ide2} = $conf->{cdrom};
2619         delete $conf->{cdrom};
2620     }
2621
2622     # we do not use 'smp' any longer
2623     if ($conf->{sockets}) {
2624         delete $conf->{smp};
2625     } elsif ($conf->{smp}) {
2626         $conf->{sockets} = $conf->{smp};
2627         delete $conf->{cores};
2628         delete $conf->{smp};
2629     }
2630
2631     my $used_volids = {};
2632
2633     my $cleanup_config = sub {
2634         my ($cref, $pending, $snapname) = @_;
2635
2636         foreach my $key (keys %$cref) {
2637             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2638                 $key eq 'snapstate' || $key eq 'pending';
2639             my $value = $cref->{$key};
2640             if ($key eq 'delete') {
2641                 die "propertry 'delete' is only allowed in [PENDING]\n"
2642                     if !$pending;
2643                 # fixme: check syntax?
2644                 next;
2645             }
2646             eval { $value = check_type($key, $value); };
2647             die "unable to parse value of '$key' - $@" if $@;
2648
2649             $cref->{$key} = $value;
2650
2651             if (!$snapname && is_valid_drivename($key)) {
2652                 my $drive = parse_drive($key, $value);
2653                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2654             }
2655         }
2656     };
2657
2658     &$cleanup_config($conf);
2659
2660     &$cleanup_config($conf->{pending}, 1);
2661
2662     foreach my $snapname (keys %{$conf->{snapshots}}) {
2663         die "internal error" if $snapname eq 'pending';
2664         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2665     }
2666
2667     # remove 'unusedX' settings if we re-add a volume
2668     foreach my $key (keys %$conf) {
2669         my $value = $conf->{$key};
2670         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2671             delete $conf->{$key};
2672         }
2673     }
2674
2675     my $generate_raw_config = sub {
2676         my ($conf, $pending) = @_;
2677
2678         my $raw = '';
2679
2680         # add description as comment to top of file
2681         if (defined(my $descr = $conf->{description})) {
2682             if ($descr) {
2683                 foreach my $cl (split(/\n/, $descr)) {
2684                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2685                 }
2686             } else {
2687                 $raw .= "#\n" if $pending;
2688             }
2689         }
2690
2691         foreach my $key (sort keys %$conf) {
2692             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2693             $raw .= "$key: $conf->{$key}\n";
2694         }
2695         return $raw;
2696     };
2697
2698     my $raw = &$generate_raw_config($conf);
2699
2700     if (scalar(keys %{$conf->{pending}})){
2701         $raw .= "\n[PENDING]\n";
2702         $raw .= &$generate_raw_config($conf->{pending}, 1);
2703     }
2704
2705     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2706         $raw .= "\n[$snapname]\n";
2707         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2708     }
2709
2710     return $raw;
2711 }
2712
2713 sub load_defaults {
2714
2715     my $res = {};
2716
2717     # we use static defaults from our JSON schema configuration
2718     foreach my $key (keys %$confdesc) {
2719         if (defined(my $default = $confdesc->{$key}->{default})) {
2720             $res->{$key} = $default;
2721         }
2722     }
2723
2724     return $res;
2725 }
2726
2727 sub config_list {
2728     my $vmlist = PVE::Cluster::get_vmlist();
2729     my $res = {};
2730     return $res if !$vmlist || !$vmlist->{ids};
2731     my $ids = $vmlist->{ids};
2732
2733     foreach my $vmid (keys %$ids) {
2734         my $d = $ids->{$vmid};
2735         next if !$d->{node} || $d->{node} ne $nodename;
2736         next if !$d->{type} || $d->{type} ne 'qemu';
2737         $res->{$vmid}->{exists} = 1;
2738     }
2739     return $res;
2740 }
2741
2742 # test if VM uses local resources (to prevent migration)
2743 sub check_local_resources {
2744     my ($conf, $noerr) = @_;
2745
2746     my $loc_res = 0;
2747
2748     $loc_res = 1 if $conf->{hostusb}; # old syntax
2749     $loc_res = 1 if $conf->{hostpci}; # old syntax
2750
2751     foreach my $k (keys %$conf) {
2752         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2753         # sockets are safe: they will recreated be on the target side post-migrate
2754         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2755         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2756     }
2757
2758     die "VM uses local resources\n" if $loc_res && !$noerr;
2759
2760     return $loc_res;
2761 }
2762
2763 # check if used storages are available on all nodes (use by migrate)
2764 sub check_storage_availability {
2765     my ($storecfg, $conf, $node) = @_;
2766
2767     foreach_drive($conf, sub {
2768         my ($ds, $drive) = @_;
2769
2770         my $volid = $drive->{file};
2771         return if !$volid;
2772
2773         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2774         return if !$sid;
2775
2776         # check if storage is available on both nodes
2777         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2778         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2779    });
2780 }
2781
2782 # list nodes where all VM images are available (used by has_feature API)
2783 sub shared_nodes {
2784     my ($conf, $storecfg) = @_;
2785
2786     my $nodelist = PVE::Cluster::get_nodelist();
2787     my $nodehash = { map { $_ => 1 } @$nodelist };
2788     my $nodename = PVE::INotify::nodename();
2789
2790     foreach_drive($conf, sub {
2791         my ($ds, $drive) = @_;
2792
2793         my $volid = $drive->{file};
2794         return if !$volid;
2795
2796         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2797         if ($storeid) {
2798             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2799             if ($scfg->{disable}) {
2800                 $nodehash = {};
2801             } elsif (my $avail = $scfg->{nodes}) {
2802                 foreach my $node (keys %$nodehash) {
2803                     delete $nodehash->{$node} if !$avail->{$node};
2804                 }
2805             } elsif (!$scfg->{shared}) {
2806                 foreach my $node (keys %$nodehash) {
2807                     delete $nodehash->{$node} if $node ne $nodename
2808                 }
2809             }
2810         }
2811     });
2812
2813     return $nodehash
2814 }
2815
2816 sub check_cmdline {
2817     my ($pidfile, $pid) = @_;
2818
2819     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2820     if (defined($fh)) {
2821         my $line = <$fh>;
2822         $fh->close;
2823         return undef if !$line;
2824         my @param = split(/\0/, $line);
2825
2826         my $cmd = $param[0];
2827         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2828
2829         for (my $i = 0; $i < scalar (@param); $i++) {
2830             my $p = $param[$i];
2831             next if !$p;
2832             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2833                 my $p = $param[$i+1];
2834                 return 1 if $p && ($p eq $pidfile);
2835                 return undef;
2836             }
2837         }
2838     }
2839     return undef;
2840 }
2841
2842 sub check_running {
2843     my ($vmid, $nocheck, $node) = @_;
2844
2845     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2846
2847     die "unable to find configuration file for VM $vmid - no such machine\n"
2848         if !$nocheck && ! -f $filename;
2849
2850     my $pidfile = pidfile_name($vmid);
2851
2852     if (my $fd = IO::File->new("<$pidfile")) {
2853         my $st = stat($fd);
2854         my $line = <$fd>;
2855         close($fd);
2856
2857         my $mtime = $st->mtime;
2858         if ($mtime > time()) {
2859             warn "file '$filename' modified in future\n";
2860         }
2861
2862         if ($line =~ m/^(\d+)$/) {
2863             my $pid = $1;
2864             if (check_cmdline($pidfile, $pid)) {
2865                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2866                     return $pid;
2867                 }
2868             }
2869         }
2870     }
2871
2872     return undef;
2873 }
2874
2875 sub vzlist {
2876
2877     my $vzlist = config_list();
2878
2879     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2880
2881     while (defined(my $de = $fd->read)) {
2882         next if $de !~ m/^(\d+)\.pid$/;
2883         my $vmid = $1;
2884         next if !defined($vzlist->{$vmid});
2885         if (my $pid = check_running($vmid)) {
2886             $vzlist->{$vmid}->{pid} = $pid;
2887         }
2888     }
2889
2890     return $vzlist;
2891 }
2892
2893 sub disksize {
2894     my ($storecfg, $conf) = @_;
2895
2896     my $bootdisk = $conf->{bootdisk};
2897     return undef if !$bootdisk;
2898     return undef if !is_valid_drivename($bootdisk);
2899
2900     return undef if !$conf->{$bootdisk};
2901
2902     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2903     return undef if !defined($drive);
2904
2905     return undef if drive_is_cdrom($drive);
2906
2907     my $volid = $drive->{file};
2908     return undef if !$volid;
2909
2910     return $drive->{size};
2911 }
2912
2913 our $vmstatus_return_properties = {
2914     vmid => get_standard_option('pve-vmid'),
2915     status => {
2916         description => "Qemu process status.",
2917         type => 'string',
2918         enum => ['stopped', 'running'],
2919     },
2920     maxmem => {
2921         description => "Maximum memory in bytes.",
2922         type => 'integer',
2923         optional => 1,
2924         renderer => 'bytes',
2925     },
2926     maxdisk => {
2927         description => "Root disk size in bytes.",
2928         type => 'integer',
2929         optional => 1,
2930         renderer => 'bytes',
2931     },
2932     name => {
2933         description => "VM name.",
2934         type => 'string',
2935         optional => 1,
2936     },
2937     qmpstatus => {
2938         description => "Qemu QMP agent status.",
2939         type => 'string',
2940         optional => 1,
2941     },
2942     pid => {
2943         description => "PID of running qemu process.",
2944         type => 'integer',
2945         optional => 1,
2946     },
2947     uptime => {
2948         description => "Uptime.",
2949         type => 'integer',
2950         optional => 1,
2951         renderer => 'duration',
2952     },
2953     cpus => {
2954         description => "Maximum usable CPUs.",
2955         type => 'number',
2956         optional => 1,
2957     },
2958 };
2959
2960 my $last_proc_pid_stat;
2961
2962 # get VM status information
2963 # This must be fast and should not block ($full == false)
2964 # We only query KVM using QMP if $full == true (this can be slow)
2965 sub vmstatus {
2966     my ($opt_vmid, $full) = @_;
2967
2968     my $res = {};
2969
2970     my $storecfg = PVE::Storage::config();
2971
2972     my $list = vzlist();
2973     my $defaults = load_defaults();
2974
2975     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2976
2977     my $cpucount = $cpuinfo->{cpus} || 1;
2978
2979     foreach my $vmid (keys %$list) {
2980         next if $opt_vmid && ($vmid ne $opt_vmid);
2981
2982         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2983         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2984
2985         my $d = { vmid => $vmid };
2986         $d->{pid} = $list->{$vmid}->{pid};
2987
2988         # fixme: better status?
2989         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2990
2991         my $size = disksize($storecfg, $conf);
2992         if (defined($size)) {
2993             $d->{disk} = 0; # no info available
2994             $d->{maxdisk} = $size;
2995         } else {
2996             $d->{disk} = 0;
2997             $d->{maxdisk} = 0;
2998         }
2999
3000         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3001             * ($conf->{cores} || $defaults->{cores});
3002         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3003         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3004
3005         $d->{name} = $conf->{name} || "VM $vmid";
3006         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3007             : $defaults->{memory}*(1024*1024);
3008
3009         if ($conf->{balloon}) {
3010             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3011             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3012                 : $defaults->{shares};
3013         }
3014
3015         $d->{uptime} = 0;
3016         $d->{cpu} = 0;
3017         $d->{mem} = 0;
3018
3019         $d->{netout} = 0;
3020         $d->{netin} = 0;
3021
3022         $d->{diskread} = 0;
3023         $d->{diskwrite} = 0;
3024
3025         $d->{template} = PVE::QemuConfig->is_template($conf);
3026
3027         $d->{serial} = 1 if conf_has_serial($conf);
3028
3029         $res->{$vmid} = $d;
3030     }
3031
3032     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3033     foreach my $dev (keys %$netdev) {
3034         next if $dev !~ m/^tap([1-9]\d*)i/;
3035         my $vmid = $1;
3036         my $d = $res->{$vmid};
3037         next if !$d;
3038
3039         $d->{netout} += $netdev->{$dev}->{receive};
3040         $d->{netin} += $netdev->{$dev}->{transmit};
3041
3042         if ($full) {
3043             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3044             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3045         }
3046
3047     }
3048
3049     my $ctime = gettimeofday;
3050
3051     foreach my $vmid (keys %$list) {
3052
3053         my $d = $res->{$vmid};
3054         my $pid = $d->{pid};
3055         next if !$pid;
3056
3057         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3058         next if !$pstat; # not running
3059
3060         my $used = $pstat->{utime} + $pstat->{stime};
3061
3062         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3063
3064         if ($pstat->{vsize}) {
3065             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3066         }
3067
3068         my $old = $last_proc_pid_stat->{$pid};
3069         if (!$old) {
3070             $last_proc_pid_stat->{$pid} = {
3071                 time => $ctime,
3072                 used => $used,
3073                 cpu => 0,
3074             };
3075             next;
3076         }
3077
3078         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3079
3080         if ($dtime > 1000) {
3081             my $dutime = $used -  $old->{used};
3082
3083             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3084             $last_proc_pid_stat->{$pid} = {
3085                 time => $ctime,
3086                 used => $used,
3087                 cpu => $d->{cpu},
3088             };
3089         } else {
3090             $d->{cpu} = $old->{cpu};
3091         }
3092     }
3093
3094     return $res if !$full;
3095
3096     my $qmpclient = PVE::QMPClient->new();
3097
3098     my $ballooncb = sub {
3099         my ($vmid, $resp) = @_;
3100
3101         my $info = $resp->{'return'};
3102         return if !$info->{max_mem};
3103
3104         my $d = $res->{$vmid};
3105
3106         # use memory assigned to VM
3107         $d->{maxmem} = $info->{max_mem};
3108         $d->{balloon} = $info->{actual};
3109
3110         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3111             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3112             $d->{freemem} = $info->{free_mem};
3113         }
3114
3115         $d->{ballooninfo} = $info;
3116     };
3117
3118     my $blockstatscb = sub {
3119         my ($vmid, $resp) = @_;
3120         my $data = $resp->{'return'} || [];
3121         my $totalrdbytes = 0;
3122         my $totalwrbytes = 0;
3123
3124         for my $blockstat (@$data) {
3125             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3126             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3127
3128             $blockstat->{device} =~ s/drive-//;
3129             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3130         }
3131         $res->{$vmid}->{diskread} = $totalrdbytes;
3132         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3133     };
3134
3135     my $statuscb = sub {
3136         my ($vmid, $resp) = @_;
3137
3138         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3139         # this fails if ballon driver is not loaded, so this must be
3140         # the last commnand (following command are aborted if this fails).
3141         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3142
3143         my $status = 'unknown';
3144         if (!defined($status = $resp->{'return'}->{status})) {
3145             warn "unable to get VM status\n";
3146             return;
3147         }
3148
3149         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3150     };
3151
3152     foreach my $vmid (keys %$list) {
3153         next if $opt_vmid && ($vmid ne $opt_vmid);
3154         next if !$res->{$vmid}->{pid}; # not running
3155         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3156     }
3157
3158     $qmpclient->queue_execute(undef, 2);
3159
3160     foreach my $vmid (keys %$list) {
3161         next if $opt_vmid && ($vmid ne $opt_vmid);
3162         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3163     }
3164
3165     return $res;
3166 }
3167
3168 sub foreach_drive {
3169     my ($conf, $func, @param) = @_;
3170
3171     foreach my $ds (valid_drive_names()) {
3172         next if !defined($conf->{$ds});
3173
3174         my $drive = parse_drive($ds, $conf->{$ds});
3175         next if !$drive;
3176
3177         &$func($ds, $drive, @param);
3178     }
3179 }
3180
3181 sub foreach_volid {
3182     my ($conf, $func, @param) = @_;
3183
3184     my $volhash = {};
3185
3186     my $test_volid = sub {
3187         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3188
3189         return if !$volid;
3190
3191         $volhash->{$volid}->{cdrom} //= 1;
3192         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3193
3194         $volhash->{$volid}->{replicate} //= 0;
3195         $volhash->{$volid}->{replicate} = 1 if $replicate;
3196
3197         $volhash->{$volid}->{shared} //= 0;
3198         $volhash->{$volid}->{shared} = 1 if $shared;
3199
3200         $volhash->{$volid}->{referenced_in_config} //= 0;
3201         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3202
3203         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3204             if defined($snapname);
3205     };
3206
3207     foreach_drive($conf, sub {
3208         my ($ds, $drive) = @_;
3209         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3210     });
3211
3212     foreach my $snapname (keys %{$conf->{snapshots}}) {
3213         my $snap = $conf->{snapshots}->{$snapname};
3214         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3215         foreach_drive($snap, sub {
3216             my ($ds, $drive) = @_;
3217             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3218         });
3219     }
3220
3221     foreach my $volid (keys %$volhash) {
3222         &$func($volid, $volhash->{$volid}, @param);
3223     }
3224 }
3225
3226 sub conf_has_serial {
3227     my ($conf) = @_;
3228
3229     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3230         if ($conf->{"serial$i"}) {
3231             return 1;
3232         }
3233     }
3234
3235     return 0;
3236 }
3237
3238 sub vga_conf_has_spice {
3239     my ($vga) = @_;
3240
3241     my $vgaconf = parse_vga($vga);
3242     my $vgatype = $vgaconf->{type};
3243     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3244
3245     return $1 || 1;
3246 }
3247
3248 sub config_to_command {
3249     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3250
3251     my $cmd = [];
3252     my $globalFlags = [];
3253     my $machineFlags = [];
3254     my $rtcFlags = [];
3255     my $cpuFlags = [];
3256     my $devices = [];
3257     my $pciaddr = '';
3258     my $bridges = {};
3259     my $kvmver = kvm_user_version();
3260     my $vernum = 0; # unknown
3261     my $ostype = $conf->{ostype};
3262     my $winversion = windows_version($ostype);
3263     my $kvm = $conf->{kvm} // 1;
3264
3265     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3266
3267     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3268         $vernum = $1*1000000+$2*1000;
3269     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3270         $vernum = $1*1000000+$2*1000+$3;
3271     }
3272
3273     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3274
3275     my $have_ovz = -f '/proc/vz/vestat';
3276
3277     my $q35 = machine_type_is_q35($conf);
3278     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3279     my $machine_type = $forcemachine || $conf->{machine};
3280     my $use_old_bios_files = undef;
3281     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3282
3283     my $cpuunits = defined($conf->{cpuunits}) ?
3284             $conf->{cpuunits} : $defaults->{cpuunits};
3285
3286     push @$cmd, '/usr/bin/kvm';
3287
3288     push @$cmd, '-id', $vmid;
3289
3290     my $vmname = $conf->{name} || "vm$vmid";
3291
3292     push @$cmd, '-name', $vmname;
3293
3294     my $use_virtio = 0;
3295
3296     my $qmpsocket = qmp_socket($vmid);
3297     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3298     push @$cmd, '-mon', "chardev=qmp,mode=control";
3299
3300     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3301         my $eventsocket = qmp_socket($vmid, 0, 'event');
3302         push @$cmd, '-chardev', "socket,id=qmp-event,path=$eventsocket,server,nowait";
3303         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3304     }
3305
3306     push @$cmd, '-pidfile' , pidfile_name($vmid);
3307
3308     push @$cmd, '-daemonize';
3309
3310     if ($conf->{smbios1}) {
3311         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3312     }
3313
3314     if ($conf->{vmgenid}) {
3315         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3316     }
3317
3318     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3319         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3320
3321         my $path;
3322         my $format;
3323         if (my $efidisk = $conf->{efidisk0}) {
3324             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3325             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3326             $format = $d->{format};
3327             if ($storeid) {
3328                 $path = PVE::Storage::path($storecfg, $d->{file});
3329                 if (!defined($format)) {
3330                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3331                     $format = qemu_img_format($scfg, $volname);
3332                 }
3333             } else {
3334                 $path = $d->{file};
3335                 die "efidisk format must be specified\n"
3336                     if !defined($format);
3337             }
3338         } else {
3339             warn "no efidisk configured! Using temporary efivars disk.\n";
3340             $path = "/tmp/$vmid-ovmf.fd";
3341             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3342             $format = 'raw';
3343         }
3344
3345         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3346         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3347     }
3348
3349
3350     # add usb controllers
3351     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3352     push @$devices, @usbcontrollers if @usbcontrollers;
3353     my $vga = parse_vga($conf->{vga});
3354
3355     my $qxlnum = vga_conf_has_spice($conf->{vga});
3356     $vga->{type} = 'qxl' if $qxlnum;
3357
3358     if (!$vga->{type}) {
3359         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3360             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3361         } else {
3362             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3363         }
3364     }
3365
3366     # enable absolute mouse coordinates (needed by vnc)
3367     my $tablet;
3368     if (defined($conf->{tablet})) {
3369         $tablet = $conf->{tablet};
3370     } else {
3371         $tablet = $defaults->{tablet};
3372         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3373         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3374     }
3375
3376     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3377
3378     my $kvm_off = 0;
3379     my $gpu_passthrough;
3380
3381     # host pci devices
3382     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3383         my $d = parse_hostpci($conf->{"hostpci$i"});
3384         next if !$d;
3385
3386         my $pcie = $d->{pcie};
3387         if($pcie){
3388             die "q35 machine model is not enabled" if !$q35;
3389             $pciaddr = print_pcie_addr("hostpci$i");
3390         }else{
3391             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3392         }
3393
3394         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3395         my $romfile = $d->{romfile};
3396
3397         my $xvga = '';
3398         if ($d->{'x-vga'}) {
3399             $xvga = ',x-vga=on';
3400             $kvm_off = 1;
3401             $vga->{type} = 'none';
3402             $gpu_passthrough = 1;
3403
3404             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3405                 $xvga = "";
3406             }
3407         }
3408         my $pcidevices = $d->{pciid};
3409         my $multifunction = 1 if @$pcidevices > 1;
3410
3411         my $j=0;
3412         foreach my $pcidevice (@$pcidevices) {
3413
3414             my $id = "hostpci$i";
3415             $id .= ".$j" if $multifunction;
3416             my $addr = $pciaddr;
3417             $addr .= ".$j" if $multifunction;
3418             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3419
3420             if($j == 0){
3421                 $devicestr .= "$rombar$xvga";
3422                 $devicestr .= ",multifunction=on" if $multifunction;
3423                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3424             }
3425
3426             push @$devices, '-device', $devicestr;
3427             $j++;
3428         }
3429     }
3430
3431     # usb devices
3432     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3433     push @$devices, @usbdevices if @usbdevices;
3434     # serial devices
3435     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3436         if (my $path = $conf->{"serial$i"}) {
3437             if ($path eq 'socket') {
3438                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3439                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3440                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3441             } else {
3442                 die "no such serial device\n" if ! -c $path;
3443                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3444                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3445             }
3446         }
3447     }
3448
3449     # parallel devices
3450     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3451         if (my $path = $conf->{"parallel$i"}) {
3452             die "no such parallel device\n" if ! -c $path;
3453             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3454             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3455             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3456         }
3457     }
3458
3459
3460     my $sockets = 1;
3461     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3462     $sockets = $conf->{sockets} if  $conf->{sockets};
3463
3464     my $cores = $conf->{cores} || 1;
3465
3466     my $maxcpus = $sockets * $cores;
3467
3468     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3469
3470     my $allowed_vcpus = $cpuinfo->{cpus};
3471
3472     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3473         if ($allowed_vcpus < $maxcpus);
3474
3475     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3476
3477         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3478         for (my $i = 2; $i <= $vcpus; $i++)  {
3479             my $cpustr = print_cpu_device($conf,$i);
3480             push @$cmd, '-device', $cpustr;
3481         }
3482
3483     } else {
3484
3485         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3486     }
3487     push @$cmd, '-nodefaults';
3488
3489     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3490
3491     my $bootindex_hash = {};
3492     my $i = 1;
3493     foreach my $o (split(//, $bootorder)) {
3494         $bootindex_hash->{$o} = $i*100;
3495         $i++;
3496     }
3497
3498     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3499
3500     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3501
3502     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3503
3504     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3505         push @$devices, '-device', print_vga_device($conf, $vga, undef, $qxlnum, $bridges);
3506         my $socket = vnc_socket($vmid);
3507         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3508     } else {
3509         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3510         push @$cmd, '-nographic';
3511     }
3512
3513     # time drift fix
3514     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3515
3516     my $useLocaltime = $conf->{localtime};
3517
3518     if ($winversion >= 5) { # windows
3519         $useLocaltime = 1 if !defined($conf->{localtime});
3520
3521         # use time drift fix when acpi is enabled
3522         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3523             $tdf = 1 if !defined($conf->{tdf});
3524         }
3525     }
3526
3527     if ($winversion >= 6) {
3528         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3529         push @$cmd, '-no-hpet';
3530     }
3531
3532     push @$rtcFlags, 'driftfix=slew' if $tdf;
3533
3534     if (!$kvm) {
3535         push @$machineFlags, 'accel=tcg';
3536     }
3537
3538     if ($machine_type) {
3539         push @$machineFlags, "type=${machine_type}";
3540     }
3541
3542     if ($conf->{startdate}) {
3543         push @$rtcFlags, "base=$conf->{startdate}";
3544     } elsif ($useLocaltime) {
3545         push @$rtcFlags, 'base=localtime';
3546     }
3547
3548     my $cpu = $kvm ? "kvm64" : "qemu64";
3549     if (my $cputype = $conf->{cpu}) {
3550         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3551             or die "Cannot parse cpu description: $cputype\n";
3552         $cpu = $cpuconf->{cputype};
3553         $kvm_off = 1 if $cpuconf->{hidden};
3554
3555         if (defined(my $flags = $cpuconf->{flags})) {
3556             push @$cpuFlags, split(";", $flags);
3557         }
3558     }
3559
3560     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3561
3562     push @$cpuFlags , '-x2apic'
3563         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3564
3565     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3566
3567     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3568
3569     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3570
3571         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3572         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3573     }
3574
3575     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3576
3577     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3578
3579     push @$cpuFlags, 'kvm=off' if $kvm_off;
3580
3581     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3582         die "internal error"; # should not happen
3583
3584     push @$cpuFlags, "vendor=${cpu_vendor}"
3585         if $cpu_vendor ne 'default';
3586
3587     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3588
3589     push @$cmd, '-cpu', $cpu;
3590
3591     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3592
3593     push @$cmd, '-S' if $conf->{freeze};
3594
3595     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3596
3597     # enable sound
3598     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3599     #push @$cmd, '-soundhw', 'es1370';
3600     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3601
3602     if (parse_guest_agent($conf)->{enabled}) {
3603         my $qgasocket = qmp_socket($vmid, 1);
3604         my $pciaddr = print_pci_addr("qga0", $bridges);
3605         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3606         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3607         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3608     }
3609
3610     my $spice_port;
3611
3612     if ($qxlnum) {
3613         if ($qxlnum > 1) {
3614             if ($winversion){
3615                 for(my $i = 1; $i < $qxlnum; $i++){
3616                     push @$devices, '-device', print_vga_device($conf, $vga, $i, $qxlnum, $bridges);
3617                 }
3618             } else {
3619                 # assume other OS works like Linux
3620                 my ($ram, $vram) = ("134217728", "67108864");
3621                 if ($vga->{memory}) {
3622                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3623                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3624                 }
3625                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3626                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3627             }
3628         }
3629
3630         my $pciaddr = print_pci_addr("spice", $bridges);
3631
3632         my $nodename = PVE::INotify::nodename();
3633         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3634         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3635         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3636         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3637         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3638
3639         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3640
3641         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3642         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3643         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3644     }
3645
3646     # enable balloon by default, unless explicitly disabled
3647     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3648         $pciaddr = print_pci_addr("balloon0", $bridges);
3649         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3650     }
3651
3652     if ($conf->{watchdog}) {
3653         my $wdopts = parse_watchdog($conf->{watchdog});
3654         $pciaddr = print_pci_addr("watchdog", $bridges);
3655         my $watchdog = $wdopts->{model} || 'i6300esb';
3656         push @$devices, '-device', "$watchdog$pciaddr";
3657         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3658     }
3659
3660     my $vollist = [];
3661     my $scsicontroller = {};
3662     my $ahcicontroller = {};
3663     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3664
3665     # Add iscsi initiator name if available
3666     if (my $initiator = get_initiator_name()) {
3667         push @$devices, '-iscsi', "initiator-name=$initiator";
3668     }
3669
3670     foreach_drive($conf, sub {
3671         my ($ds, $drive) = @_;
3672
3673         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3674             push @$vollist, $drive->{file};
3675         }
3676
3677         # ignore efidisk here, already added in bios/fw handling code above
3678         return if $drive->{interface} eq 'efidisk';
3679
3680         $use_virtio = 1 if $ds =~ m/^virtio/;
3681
3682         if (drive_is_cdrom ($drive)) {
3683             if ($bootindex_hash->{d}) {
3684                 $drive->{bootindex} = $bootindex_hash->{d};
3685                 $bootindex_hash->{d} += 1;
3686             }
3687         } else {
3688             if ($bootindex_hash->{c}) {
3689                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3690                 $bootindex_hash->{c} += 1;
3691             }
3692         }
3693
3694         if($drive->{interface} eq 'virtio'){
3695            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3696         }
3697
3698         if ($drive->{interface} eq 'scsi') {
3699
3700             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3701
3702             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3703             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3704
3705             my $iothread = '';
3706             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3707                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3708                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3709             } elsif ($drive->{iothread}) {
3710                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3711             }
3712
3713             my $queues = '';
3714             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3715                 $queues = ",num_queues=$drive->{queues}";
3716             }
3717
3718             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3719             $scsicontroller->{$controller}=1;
3720         }
3721
3722         if ($drive->{interface} eq 'sata') {
3723            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3724            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3725            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3726            $ahcicontroller->{$controller}=1;
3727         }
3728
3729         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3730         push @$devices, '-drive',$drive_cmd;
3731         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3732     });
3733
3734     for (my $i = 0; $i < $MAX_NETS; $i++) {
3735          next if !$conf->{"net$i"};
3736          my $d = parse_net($conf->{"net$i"});
3737          next if !$d;
3738
3739          $use_virtio = 1 if $d->{model} eq 'virtio';
3740
3741          if ($bootindex_hash->{n}) {
3742             $d->{bootindex} = $bootindex_hash->{n};
3743             $bootindex_hash->{n} += 1;
3744          }
3745
3746          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3747          push @$devices, '-netdev', $netdevfull;
3748
3749          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3750          push @$devices, '-device', $netdevicefull;
3751     }
3752
3753     if (!$q35) {
3754         # add pci bridges
3755         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3756            $bridges->{1} = 1;
3757            $bridges->{2} = 1;
3758         }
3759
3760         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3761
3762         while (my ($k, $v) = each %$bridges) {
3763             $pciaddr = print_pci_addr("pci.$k");
3764             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3765         }
3766     }
3767
3768     # add custom args
3769     if ($conf->{args}) {
3770         my $aa = PVE::Tools::split_args($conf->{args});
3771         push @$cmd, @$aa;
3772     }
3773
3774     push @$cmd, @$devices;
3775     push @$cmd, '-rtc', join(',', @$rtcFlags)
3776         if scalar(@$rtcFlags);
3777     push @$cmd, '-machine', join(',', @$machineFlags)
3778         if scalar(@$machineFlags);
3779     push @$cmd, '-global', join(',', @$globalFlags)
3780         if scalar(@$globalFlags);
3781
3782     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3783 }
3784
3785 sub vnc_socket {
3786     my ($vmid) = @_;
3787     return "${var_run_tmpdir}/$vmid.vnc";
3788 }
3789
3790 sub spice_port {
3791     my ($vmid) = @_;
3792
3793     my $res = vm_mon_cmd($vmid, 'query-spice');
3794
3795     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3796 }
3797
3798 sub qmp_socket {
3799     my ($vmid, $qga, $name) = @_;
3800     my $sockettype = $qga ? 'qga' : 'qmp';
3801     my $ext = $name ? '-'.$name : '';
3802     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3803 }
3804
3805 sub pidfile_name {
3806     my ($vmid) = @_;
3807     return "${var_run_tmpdir}/$vmid.pid";
3808 }
3809
3810 sub vm_devices_list {
3811     my ($vmid) = @_;
3812
3813     my $res = vm_mon_cmd($vmid, 'query-pci');
3814     my $devices_to_check = [];
3815     my $devices = {};
3816     foreach my $pcibus (@$res) {
3817         push @$devices_to_check, @{$pcibus->{devices}},
3818     }
3819
3820     while (@$devices_to_check) {
3821         my $to_check = [];
3822         for my $d (@$devices_to_check) {
3823             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3824             next if !$d->{'pci_bridge'};
3825
3826             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3827             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3828         }
3829         $devices_to_check = $to_check;
3830     }
3831
3832     my $resblock = vm_mon_cmd($vmid, 'query-block');
3833     foreach my $block (@$resblock) {
3834         if($block->{device} =~ m/^drive-(\S+)/){
3835                 $devices->{$1} = 1;
3836         }
3837     }
3838
3839     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3840     foreach my $mice (@$resmice) {
3841         if ($mice->{name} eq 'QEMU HID Tablet') {
3842             $devices->{tablet} = 1;
3843             last;
3844         }
3845     }
3846
3847     # for usb devices there is no query-usb
3848     # but we can iterate over the entries in
3849     # qom-list path=/machine/peripheral
3850     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3851     foreach my $per (@$resperipheral) {
3852         if ($per->{name} =~ m/^usb\d+$/) {
3853             $devices->{$per->{name}} = 1;
3854         }
3855     }
3856
3857     return $devices;
3858 }
3859
3860 sub vm_deviceplug {
3861     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3862
3863     my $q35 = machine_type_is_q35($conf);
3864
3865     my $devices_list = vm_devices_list($vmid);
3866     return 1 if defined($devices_list->{$deviceid});
3867
3868     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3869
3870     if ($deviceid eq 'tablet') {
3871
3872         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3873
3874     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3875
3876         die "usb hotplug currently not reliable\n";
3877         # since we can't reliably hot unplug all added usb devices
3878         # and usb passthrough disables live migration
3879         # we disable usb hotplugging for now
3880         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3881
3882     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3883
3884         qemu_iothread_add($vmid, $deviceid, $device);
3885
3886         qemu_driveadd($storecfg, $vmid, $device);
3887         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3888
3889         qemu_deviceadd($vmid, $devicefull);
3890         eval { qemu_deviceaddverify($vmid, $deviceid); };
3891         if (my $err = $@) {
3892             eval { qemu_drivedel($vmid, $deviceid); };
3893             warn $@ if $@;
3894             die $err;
3895         }
3896
3897     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3898
3899
3900         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3901         my $pciaddr = print_pci_addr($deviceid);
3902         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3903
3904         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3905
3906         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3907             qemu_iothread_add($vmid, $deviceid, $device);
3908             $devicefull .= ",iothread=iothread-$deviceid";
3909         }
3910
3911         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3912             $devicefull .= ",num_queues=$device->{queues}";
3913         }
3914
3915         qemu_deviceadd($vmid, $devicefull);
3916         qemu_deviceaddverify($vmid, $deviceid);
3917
3918     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3919
3920         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3921         qemu_driveadd($storecfg, $vmid, $device);
3922
3923         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3924         eval { qemu_deviceadd($vmid, $devicefull); };
3925         if (my $err = $@) {
3926             eval { qemu_drivedel($vmid, $deviceid); };
3927             warn $@ if $@;
3928             die $err;
3929         }
3930
3931     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3932
3933         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3934
3935         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3936         my $use_old_bios_files = undef;
3937         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3938
3939         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3940         qemu_deviceadd($vmid, $netdevicefull);
3941         eval { qemu_deviceaddverify($vmid, $deviceid); };
3942         if (my $err = $@) {
3943             eval { qemu_netdevdel($vmid, $deviceid); };
3944             warn $@ if $@;
3945             die $err;
3946         }
3947
3948     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3949
3950         my $bridgeid = $2;
3951         my $pciaddr = print_pci_addr($deviceid);
3952         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3953
3954         qemu_deviceadd($vmid, $devicefull);
3955         qemu_deviceaddverify($vmid, $deviceid);
3956
3957     } else {
3958         die "can't hotplug device '$deviceid'\n";
3959     }
3960
3961     return 1;
3962 }
3963
3964 # fixme: this should raise exceptions on error!
3965 sub vm_deviceunplug {
3966     my ($vmid, $conf, $deviceid) = @_;
3967
3968     my $devices_list = vm_devices_list($vmid);
3969     return 1 if !defined($devices_list->{$deviceid});
3970
3971     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3972
3973     if ($deviceid eq 'tablet') {
3974
3975         qemu_devicedel($vmid, $deviceid);
3976
3977     } elsif ($deviceid =~ m/^usb\d+$/) {
3978
3979         die "usb hotplug currently not reliable\n";
3980         # when unplugging usb devices this way,
3981         # there may be remaining usb controllers/hubs
3982         # so we disable it for now
3983         qemu_devicedel($vmid, $deviceid);
3984         qemu_devicedelverify($vmid, $deviceid);
3985
3986     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3987
3988         qemu_devicedel($vmid, $deviceid);
3989         qemu_devicedelverify($vmid, $deviceid);
3990         qemu_drivedel($vmid, $deviceid);
3991         qemu_iothread_del($conf, $vmid, $deviceid);
3992
3993     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3994
3995         qemu_devicedel($vmid, $deviceid);
3996         qemu_devicedelverify($vmid, $deviceid);
3997         qemu_iothread_del($conf, $vmid, $deviceid);
3998
3999     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4000
4001         qemu_devicedel($vmid, $deviceid);
4002         qemu_drivedel($vmid, $deviceid);
4003         qemu_deletescsihw($conf, $vmid, $deviceid);
4004
4005     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4006
4007         qemu_devicedel($vmid, $deviceid);
4008         qemu_devicedelverify($vmid, $deviceid);
4009         qemu_netdevdel($vmid, $deviceid);
4010
4011     } else {
4012         die "can't unplug device '$deviceid'\n";
4013     }
4014
4015     return 1;
4016 }
4017
4018 sub qemu_deviceadd {
4019     my ($vmid, $devicefull) = @_;
4020
4021     $devicefull = "driver=".$devicefull;
4022     my %options =  split(/[=,]/, $devicefull);
4023
4024     vm_mon_cmd($vmid, "device_add" , %options);
4025 }
4026
4027 sub qemu_devicedel {
4028     my ($vmid, $deviceid) = @_;
4029
4030     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4031 }
4032
4033 sub qemu_iothread_add {
4034     my($vmid, $deviceid, $device) = @_;
4035
4036     if ($device->{iothread}) {
4037         my $iothreads = vm_iothreads_list($vmid);
4038         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4039     }
4040 }
4041
4042 sub qemu_iothread_del {
4043     my($conf, $vmid, $deviceid) = @_;
4044
4045     my $device = parse_drive($deviceid, $conf->{$deviceid});
4046     if ($device->{iothread}) {
4047         my $iothreads = vm_iothreads_list($vmid);
4048         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4049     }
4050 }
4051
4052 sub qemu_objectadd {
4053     my($vmid, $objectid, $qomtype) = @_;
4054
4055     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4056
4057     return 1;
4058 }
4059
4060 sub qemu_objectdel {
4061     my($vmid, $objectid) = @_;
4062
4063     vm_mon_cmd($vmid, "object-del", id => $objectid);
4064
4065     return 1;
4066 }
4067
4068 sub qemu_driveadd {
4069     my ($storecfg, $vmid, $device) = @_;
4070
4071     my $drive = print_drive_full($storecfg, $vmid, $device);
4072     $drive =~ s/\\/\\\\/g;
4073     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4074
4075     # If the command succeeds qemu prints: "OK"
4076     return 1 if $ret =~ m/OK/s;
4077
4078     die "adding drive failed: $ret\n";
4079 }
4080
4081 sub qemu_drivedel {
4082     my($vmid, $deviceid) = @_;
4083
4084     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4085     $ret =~ s/^\s+//;
4086
4087     return 1 if $ret eq "";
4088
4089     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4090     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4091
4092     die "deleting drive $deviceid failed : $ret\n";
4093 }
4094
4095 sub qemu_deviceaddverify {
4096     my ($vmid, $deviceid) = @_;
4097
4098     for (my $i = 0; $i <= 5; $i++) {
4099          my $devices_list = vm_devices_list($vmid);
4100          return 1 if defined($devices_list->{$deviceid});
4101          sleep 1;
4102     }
4103
4104     die "error on hotplug device '$deviceid'\n";
4105 }
4106
4107
4108 sub qemu_devicedelverify {
4109     my ($vmid, $deviceid) = @_;
4110
4111     # need to verify that the device is correctly removed as device_del
4112     # is async and empty return is not reliable
4113
4114     for (my $i = 0; $i <= 5; $i++) {
4115          my $devices_list = vm_devices_list($vmid);
4116          return 1 if !defined($devices_list->{$deviceid});
4117          sleep 1;
4118     }
4119
4120     die "error on hot-unplugging device '$deviceid'\n";
4121 }
4122
4123 sub qemu_findorcreatescsihw {
4124     my ($storecfg, $conf, $vmid, $device) = @_;
4125
4126     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4127
4128     my $scsihwid="$controller_prefix$controller";
4129     my $devices_list = vm_devices_list($vmid);
4130
4131     if(!defined($devices_list->{$scsihwid})) {
4132         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
4133     }
4134
4135     return 1;
4136 }
4137
4138 sub qemu_deletescsihw {
4139     my ($conf, $vmid, $opt) = @_;
4140
4141     my $device = parse_drive($opt, $conf->{$opt});
4142
4143     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4144         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4145         return 1;
4146     }
4147
4148     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4149
4150     my $devices_list = vm_devices_list($vmid);
4151     foreach my $opt (keys %{$devices_list}) {
4152         if (PVE::QemuServer::is_valid_drivename($opt)) {
4153             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4154             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4155                 return 1;
4156             }
4157         }
4158     }
4159
4160     my $scsihwid="scsihw$controller";
4161
4162     vm_deviceunplug($vmid, $conf, $scsihwid);
4163
4164     return 1;
4165 }
4166
4167 sub qemu_add_pci_bridge {
4168     my ($storecfg, $conf, $vmid, $device) = @_;
4169
4170     my $bridges = {};
4171
4172     my $bridgeid;
4173
4174     print_pci_addr($device, $bridges);
4175
4176     while (my ($k, $v) = each %$bridges) {
4177         $bridgeid = $k;
4178     }
4179     return 1 if !defined($bridgeid) || $bridgeid < 1;
4180
4181     my $bridge = "pci.$bridgeid";
4182     my $devices_list = vm_devices_list($vmid);
4183
4184     if (!defined($devices_list->{$bridge})) {
4185         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4186     }
4187
4188     return 1;
4189 }
4190
4191 sub qemu_set_link_status {
4192     my ($vmid, $device, $up) = @_;
4193
4194     vm_mon_cmd($vmid, "set_link", name => $device,
4195                up => $up ? JSON::true : JSON::false);
4196 }
4197
4198 sub qemu_netdevadd {
4199     my ($vmid, $conf, $device, $deviceid) = @_;
4200
4201     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4202     my %options =  split(/[=,]/, $netdev);
4203
4204     vm_mon_cmd($vmid, "netdev_add",  %options);
4205     return 1;
4206 }
4207
4208 sub qemu_netdevdel {
4209     my ($vmid, $deviceid) = @_;
4210
4211     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4212 }
4213
4214 sub qemu_usb_hotplug {
4215     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4216
4217     return if !$device;
4218
4219     # remove the old one first
4220     vm_deviceunplug($vmid, $conf, $deviceid);
4221
4222     # check if xhci controller is necessary and available
4223     if ($device->{usb3}) {
4224
4225         my $devicelist = vm_devices_list($vmid);
4226
4227         if (!$devicelist->{xhci}) {
4228             my $pciaddr = print_pci_addr("xhci");
4229             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4230         }
4231     }
4232     my $d = parse_usb_device($device->{host});
4233     $d->{usb3} = $device->{usb3};
4234
4235     # add the new one
4236     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4237 }
4238
4239 sub qemu_cpu_hotplug {
4240     my ($vmid, $conf, $vcpus) = @_;
4241
4242     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4243
4244     my $sockets = 1;
4245     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4246     $sockets = $conf->{sockets} if  $conf->{sockets};
4247     my $cores = $conf->{cores} || 1;
4248     my $maxcpus = $sockets * $cores;
4249
4250     $vcpus = $maxcpus if !$vcpus;
4251
4252     die "you can't add more vcpus than maxcpus\n"
4253         if $vcpus > $maxcpus;
4254
4255     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4256
4257     if ($vcpus < $currentvcpus) {
4258
4259         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4260
4261             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4262                 qemu_devicedel($vmid, "cpu$i");
4263                 my $retry = 0;
4264                 my $currentrunningvcpus = undef;
4265                 while (1) {
4266                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4267                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4268                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4269                     $retry++;
4270                     sleep 1;
4271                 }
4272                 #update conf after each succesfull cpu unplug
4273                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4274                 PVE::QemuConfig->write_config($vmid, $conf);
4275             }
4276         } else {
4277             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4278         }
4279
4280         return;
4281     }
4282
4283     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4284     die "vcpus in running vm does not match its configuration\n"
4285         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4286
4287     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4288
4289         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4290             my $cpustr = print_cpu_device($conf, $i);
4291             qemu_deviceadd($vmid, $cpustr);
4292
4293             my $retry = 0;
4294             my $currentrunningvcpus = undef;
4295             while (1) {
4296                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4297                 last if scalar(@{$currentrunningvcpus}) == $i;
4298                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4299                 sleep 1;
4300                 $retry++;
4301             }
4302             #update conf after each succesfull cpu hotplug
4303             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4304             PVE::QemuConfig->write_config($vmid, $conf);
4305         }
4306     } else {
4307
4308         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4309             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4310         }
4311     }
4312 }
4313
4314 sub qemu_block_set_io_throttle {
4315     my ($vmid, $deviceid,
4316         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4317         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4318         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4319         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4320
4321     return if !check_running($vmid) ;
4322
4323     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4324         bps => int($bps),
4325         bps_rd => int($bps_rd),
4326         bps_wr => int($bps_wr),
4327         iops => int($iops),
4328         iops_rd => int($iops_rd),
4329         iops_wr => int($iops_wr),
4330         bps_max => int($bps_max),
4331         bps_rd_max => int($bps_rd_max),
4332         bps_wr_max => int($bps_wr_max),
4333         iops_max => int($iops_max),
4334         iops_rd_max => int($iops_rd_max),
4335         iops_wr_max => int($iops_wr_max),
4336         bps_max_length => int($bps_max_length),
4337         bps_rd_max_length => int($bps_rd_max_length),
4338         bps_wr_max_length => int($bps_wr_max_length),
4339         iops_max_length => int($iops_max_length),
4340         iops_rd_max_length => int($iops_rd_max_length),
4341         iops_wr_max_length => int($iops_wr_max_length),
4342     );
4343
4344 }
4345
4346 # old code, only used to shutdown old VM after update
4347 sub __read_avail {
4348     my ($fh, $timeout) = @_;
4349
4350     my $sel = new IO::Select;
4351     $sel->add($fh);
4352
4353     my $res = '';
4354     my $buf;
4355
4356     my @ready;
4357     while (scalar (@ready = $sel->can_read($timeout))) {
4358         my $count;
4359         if ($count = $fh->sysread($buf, 8192)) {
4360             if ($buf =~ /^(.*)\(qemu\) $/s) {
4361                 $res .= $1;
4362                 last;
4363             } else {
4364                 $res .= $buf;
4365             }
4366         } else {
4367             if (!defined($count)) {
4368                 die "$!\n";
4369             }
4370             last;
4371         }
4372     }
4373
4374     die "monitor read timeout\n" if !scalar(@ready);
4375
4376     return $res;
4377 }
4378
4379 sub qemu_block_resize {
4380     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4381
4382     my $running = check_running($vmid);
4383
4384     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4385
4386     return if !$running;
4387
4388     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4389
4390 }
4391
4392 sub qemu_volume_snapshot {
4393     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4394
4395     my $running = check_running($vmid);
4396
4397     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4398         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4399     } else {
4400         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4401     }
4402 }
4403
4404 sub qemu_volume_snapshot_delete {
4405     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4406
4407     my $running = check_running($vmid);
4408
4409     if($running) {
4410
4411         $running = undef;
4412         my $conf = PVE::QemuConfig->load_config($vmid);
4413         foreach_drive($conf, sub {
4414             my ($ds, $drive) = @_;
4415             $running = 1 if $drive->{file} eq $volid;
4416         });
4417     }
4418
4419     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4420         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4421     } else {
4422         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4423     }
4424 }
4425
4426 sub set_migration_caps {
4427     my ($vmid) = @_;
4428
4429     my $cap_ref = [];
4430
4431     my $enabled_cap = {
4432         "auto-converge" => 1,
4433         "xbzrle" => 1,
4434         "x-rdma-pin-all" => 0,
4435         "zero-blocks" => 0,
4436         "compress" => 0
4437     };
4438
4439     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4440
4441     for my $supported_capability (@$supported_capabilities) {
4442         push @$cap_ref, {
4443             capability => $supported_capability->{capability},
4444             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4445         };
4446     }
4447
4448     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4449 }
4450
4451 my $fast_plug_option = {
4452     'lock' => 1,
4453     'name' => 1,
4454     'onboot' => 1,
4455     'shares' => 1,
4456     'startup' => 1,
4457     'description' => 1,
4458     'protection' => 1,
4459     'vmstatestorage' => 1,
4460 };
4461
4462 # hotplug changes in [PENDING]
4463 # $selection hash can be used to only apply specified options, for
4464 # example: { cores => 1 } (only apply changed 'cores')
4465 # $errors ref is used to return error messages
4466 sub vmconfig_hotplug_pending {
4467     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4468
4469     my $defaults = load_defaults();
4470
4471     # commit values which do not have any impact on running VM first
4472     # Note: those option cannot raise errors, we we do not care about
4473     # $selection and always apply them.
4474
4475     my $add_error = sub {
4476         my ($opt, $msg) = @_;
4477         $errors->{$opt} = "hotplug problem - $msg";
4478     };
4479
4480     my $changes = 0;
4481     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4482         if ($fast_plug_option->{$opt}) {
4483             $conf->{$opt} = $conf->{pending}->{$opt};
4484             delete $conf->{pending}->{$opt};
4485             $changes = 1;
4486         }
4487     }
4488
4489     if ($changes) {
4490         PVE::QemuConfig->write_config($vmid, $conf);
4491         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4492     }
4493
4494     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4495
4496     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4497     while (my ($opt, $force) = each %$pending_delete_hash) {
4498         next if $selection && !$selection->{$opt};
4499         eval {
4500             if ($opt eq 'hotplug') {
4501                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4502             } elsif ($opt eq 'tablet') {
4503                 die "skip\n" if !$hotplug_features->{usb};
4504                 if ($defaults->{tablet}) {
4505                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4506                 } else {
4507                     vm_deviceunplug($vmid, $conf, $opt);
4508                 }
4509             } elsif ($opt =~ m/^usb\d+/) {
4510                 die "skip\n";
4511                 # since we cannot reliably hot unplug usb devices
4512                 # we are disabling it
4513                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4514                 vm_deviceunplug($vmid, $conf, $opt);
4515             } elsif ($opt eq 'vcpus') {
4516                 die "skip\n" if !$hotplug_features->{cpu};
4517                 qemu_cpu_hotplug($vmid, $conf, undef);
4518             } elsif ($opt eq 'balloon') {
4519                 # enable balloon device is not hotpluggable
4520                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4521                 # here we reset the ballooning value to memory
4522                 my $balloon = $conf->{memory} || $defaults->{memory};
4523                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4524             } elsif ($fast_plug_option->{$opt}) {
4525                 # do nothing
4526             } elsif ($opt =~ m/^net(\d+)$/) {
4527                 die "skip\n" if !$hotplug_features->{network};
4528                 vm_deviceunplug($vmid, $conf, $opt);
4529             } elsif (is_valid_drivename($opt)) {
4530                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4531                 vm_deviceunplug($vmid, $conf, $opt);
4532                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4533             } elsif ($opt =~ m/^memory$/) {
4534                 die "skip\n" if !$hotplug_features->{memory};
4535                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4536             } elsif ($opt eq 'cpuunits') {
4537                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4538             } elsif ($opt eq 'cpulimit') {
4539                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4540             } else {
4541                 die "skip\n";
4542             }
4543         };
4544         if (my $err = $@) {
4545             &$add_error($opt, $err) if $err ne "skip\n";
4546         } else {
4547             # save new config if hotplug was successful
4548             delete $conf->{$opt};
4549             vmconfig_undelete_pending_option($conf, $opt);
4550             PVE::QemuConfig->write_config($vmid, $conf);
4551             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4552         }
4553     }
4554
4555     my $apply_pending_cloudinit;
4556     $apply_pending_cloudinit = sub {
4557         my ($key, $value) = @_;
4558         $apply_pending_cloudinit = sub {}; # once is enough
4559
4560         my @cloudinit_opts = keys %$confdesc_cloudinit;
4561         foreach my $opt (keys %{$conf->{pending}}) {
4562             next if !grep { $_ eq $opt } @cloudinit_opts;
4563             $conf->{$opt} = delete $conf->{pending}->{$opt};
4564         }
4565
4566         my $new_conf = { %$conf };
4567         $new_conf->{$key} = $value;
4568         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4569     };
4570
4571     foreach my $opt (keys %{$conf->{pending}}) {
4572         next if $selection && !$selection->{$opt};
4573         my $value = $conf->{pending}->{$opt};
4574         eval {
4575             if ($opt eq 'hotplug') {
4576                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4577             } elsif ($opt eq 'tablet') {
4578                 die "skip\n" if !$hotplug_features->{usb};
4579                 if ($value == 1) {
4580                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4581                 } elsif ($value == 0) {
4582                     vm_deviceunplug($vmid, $conf, $opt);
4583                 }
4584             } elsif ($opt =~ m/^usb\d+$/) {
4585                 die "skip\n";
4586                 # since we cannot reliably hot unplug usb devices
4587                 # we are disabling it
4588                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4589                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4590                 die "skip\n" if !$d;
4591                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4592             } elsif ($opt eq 'vcpus') {
4593                 die "skip\n" if !$hotplug_features->{cpu};
4594                 qemu_cpu_hotplug($vmid, $conf, $value);
4595             } elsif ($opt eq 'balloon') {
4596                 # enable/disable balloning device is not hotpluggable
4597                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4598                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4599                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4600
4601                 # allow manual ballooning if shares is set to zero
4602                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4603                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4604                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4605                 }
4606             } elsif ($opt =~ m/^net(\d+)$/) {
4607                 # some changes can be done without hotplug
4608                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4609                                     $vmid, $opt, $value);
4610             } elsif (is_valid_drivename($opt)) {
4611                 # some changes can be done without hotplug
4612                 my $drive = parse_drive($opt, $value);
4613                 if (drive_is_cloudinit($drive)) {
4614                     &$apply_pending_cloudinit($opt, $value);
4615                 }
4616                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4617                                      $vmid, $opt, $value, 1);
4618             } elsif ($opt =~ m/^memory$/) { #dimms
4619                 die "skip\n" if !$hotplug_features->{memory};
4620                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4621             } elsif ($opt eq 'cpuunits') {
4622                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4623             } elsif ($opt eq 'cpulimit') {
4624                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4625                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4626             } else {
4627                 die "skip\n";  # skip non-hot-pluggable options
4628             }
4629         };
4630         if (my $err = $@) {
4631             &$add_error($opt, $err) if $err ne "skip\n";
4632         } else {
4633             # save new config if hotplug was successful
4634             $conf->{$opt} = $value;
4635             delete $conf->{pending}->{$opt};
4636             PVE::QemuConfig->write_config($vmid, $conf);
4637             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4638         }
4639     }
4640 }
4641
4642 sub try_deallocate_drive {
4643     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4644
4645     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4646         my $volid = $drive->{file};
4647         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4648             my $sid = PVE::Storage::parse_volume_id($volid);
4649             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4650
4651             # check if the disk is really unused
4652             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4653                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4654             PVE::Storage::vdisk_free($storecfg, $volid);
4655             return 1;
4656         } else {
4657             # If vm is not owner of this disk remove from config
4658             return 1;
4659         }
4660     }
4661
4662     return undef;
4663 }
4664
4665 sub vmconfig_delete_or_detach_drive {
4666     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4667
4668     my $drive = parse_drive($opt, $conf->{$opt});
4669
4670     my $rpcenv = PVE::RPCEnvironment::get();
4671     my $authuser = $rpcenv->get_user();
4672
4673     if ($force) {
4674         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4675         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4676     } else {
4677         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4678     }
4679 }
4680
4681 sub vmconfig_apply_pending {
4682     my ($vmid, $conf, $storecfg) = @_;
4683
4684     # cold plug
4685
4686     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4687     while (my ($opt, $force) = each %$pending_delete_hash) {
4688         die "internal error" if $opt =~ m/^unused/;
4689         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4690         if (!defined($conf->{$opt})) {
4691             vmconfig_undelete_pending_option($conf, $opt);
4692             PVE::QemuConfig->write_config($vmid, $conf);
4693         } elsif (is_valid_drivename($opt)) {
4694             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4695             vmconfig_undelete_pending_option($conf, $opt);
4696             delete $conf->{$opt};
4697             PVE::QemuConfig->write_config($vmid, $conf);
4698         } else {
4699             vmconfig_undelete_pending_option($conf, $opt);
4700             delete $conf->{$opt};
4701             PVE::QemuConfig->write_config($vmid, $conf);
4702         }
4703     }
4704
4705     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4706
4707     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4708         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4709
4710         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4711             # skip if nothing changed
4712         } elsif (is_valid_drivename($opt)) {
4713             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4714                 if defined($conf->{$opt});
4715             $conf->{$opt} = $conf->{pending}->{$opt};
4716         } else {
4717             $conf->{$opt} = $conf->{pending}->{$opt};
4718         }
4719
4720         delete $conf->{pending}->{$opt};
4721         PVE::QemuConfig->write_config($vmid, $conf);
4722     }
4723 }
4724
4725 my $safe_num_ne = sub {
4726     my ($a, $b) = @_;
4727
4728     return 0 if !defined($a) && !defined($b);
4729     return 1 if !defined($a);
4730     return 1 if !defined($b);
4731
4732     return $a != $b;
4733 };
4734
4735 my $safe_string_ne = sub {
4736     my ($a, $b) = @_;
4737
4738     return 0 if !defined($a) && !defined($b);
4739     return 1 if !defined($a);
4740     return 1 if !defined($b);
4741
4742     return $a ne $b;
4743 };
4744
4745 sub vmconfig_update_net {
4746     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4747
4748     my $newnet = parse_net($value);
4749
4750     if ($conf->{$opt}) {
4751         my $oldnet = parse_net($conf->{$opt});
4752
4753         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4754             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4755             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4756             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4757
4758             # for non online change, we try to hot-unplug
4759             die "skip\n" if !$hotplug;
4760             vm_deviceunplug($vmid, $conf, $opt);
4761         } else {
4762
4763             die "internal error" if $opt !~ m/net(\d+)/;
4764             my $iface = "tap${vmid}i$1";
4765
4766             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4767                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4768                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4769                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4770                 PVE::Network::tap_unplug($iface);
4771                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4772             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4773                 # Rate can be applied on its own but any change above needs to
4774                 # include the rate in tap_plug since OVS resets everything.
4775                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4776             }
4777
4778             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4779                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4780             }
4781
4782             return 1;
4783         }
4784     }
4785
4786     if ($hotplug) {
4787         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4788     } else {
4789         die "skip\n";
4790     }
4791 }
4792
4793 sub vmconfig_update_disk {
4794     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4795
4796     # fixme: do we need force?
4797
4798     my $drive = parse_drive($opt, $value);
4799
4800     if ($conf->{$opt}) {
4801
4802         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4803
4804             my $media = $drive->{media} || 'disk';
4805             my $oldmedia = $old_drive->{media} || 'disk';
4806             die "unable to change media type\n" if $media ne $oldmedia;
4807
4808             if (!drive_is_cdrom($old_drive)) {
4809
4810                 if ($drive->{file} ne $old_drive->{file}) {
4811
4812                     die "skip\n" if !$hotplug;
4813
4814                     # unplug and register as unused
4815                     vm_deviceunplug($vmid, $conf, $opt);
4816                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4817
4818                 } else {
4819                     # update existing disk
4820
4821                     # skip non hotpluggable value
4822                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4823                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4824                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4825                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4826                         die "skip\n";
4827                     }
4828
4829                     # apply throttle
4830                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4831                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4832                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4833                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4834                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4835                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4836                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4837                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4838                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4839                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4840                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4841                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4842                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4843                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4844                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4845                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4846                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4847                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4848
4849                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4850                                                    ($drive->{mbps} || 0)*1024*1024,
4851                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4852                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4853                                                    $drive->{iops} || 0,
4854                                                    $drive->{iops_rd} || 0,
4855                                                    $drive->{iops_wr} || 0,
4856                                                    ($drive->{mbps_max} || 0)*1024*1024,
4857                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4858                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4859                                                    $drive->{iops_max} || 0,
4860                                                    $drive->{iops_rd_max} || 0,
4861                                                    $drive->{iops_wr_max} || 0,
4862                                                    $drive->{bps_max_length} || 1,
4863                                                    $drive->{bps_rd_max_length} || 1,
4864                                                    $drive->{bps_wr_max_length} || 1,
4865                                                    $drive->{iops_max_length} || 1,
4866                                                    $drive->{iops_rd_max_length} || 1,
4867                                                    $drive->{iops_wr_max_length} || 1);
4868
4869                     }
4870
4871                     return 1;
4872                 }
4873
4874             } else { # cdrom
4875
4876                 if ($drive->{file} eq 'none') {
4877                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4878                     if (drive_is_cloudinit($old_drive)) {
4879                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4880                     }
4881                 } else {
4882                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4883                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4884                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4885                 }
4886
4887                 return 1;
4888             }
4889         }
4890     }
4891
4892     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4893     # hotplug new disks
4894     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4895     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4896 }
4897
4898 sub vm_start {
4899     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4900         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4901
4902     PVE::QemuConfig->lock_config($vmid, sub {
4903         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4904
4905         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4906
4907         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4908
4909         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4910
4911         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4912             vmconfig_apply_pending($vmid, $conf, $storecfg);
4913             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4914         }
4915
4916         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4917
4918         my $defaults = load_defaults();
4919
4920         # set environment variable useful inside network script
4921         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4922
4923         my $local_volumes = {};
4924
4925         if ($targetstorage) {
4926             foreach_drive($conf, sub {
4927                 my ($ds, $drive) = @_;
4928
4929                 return if drive_is_cdrom($drive);
4930
4931                 my $volid = $drive->{file};
4932
4933                 return if !$volid;
4934
4935                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4936
4937                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4938                 return if $scfg->{shared};
4939                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4940             });
4941
4942             my $format = undef;
4943
4944             foreach my $opt (sort keys %$local_volumes) {
4945
4946                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4947                 my $drive = parse_drive($opt, $conf->{$opt});
4948
4949                 #if remote storage is specified, use default format
4950                 if ($targetstorage && $targetstorage ne "1") {
4951                     $storeid = $targetstorage;
4952                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4953                     $format = $defFormat;
4954                 } else {
4955                     #else we use same format than original
4956                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4957                     $format = qemu_img_format($scfg, $volid);
4958                 }
4959
4960                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4961                 my $newdrive = $drive;
4962                 $newdrive->{format} = $format;
4963                 $newdrive->{file} = $newvolid;
4964                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4965                 $local_volumes->{$opt} = $drivestr;
4966                 #pass drive to conf for command line
4967                 $conf->{$opt} = $drivestr;
4968             }
4969         }
4970
4971         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4972
4973         my $migrate_port = 0;
4974         my $migrate_uri;
4975         if ($statefile) {
4976             if ($statefile eq 'tcp') {
4977                 my $localip = "localhost";
4978                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4979                 my $nodename = PVE::INotify::nodename();
4980
4981                 if (!defined($migration_type)) {
4982                     if (defined($datacenterconf->{migration}->{type})) {
4983                         $migration_type = $datacenterconf->{migration}->{type};
4984                     } else {
4985                         $migration_type = 'secure';
4986                     }
4987                 }
4988
4989                 if ($migration_type eq 'insecure') {
4990                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4991                     if ($migrate_network_addr) {
4992                         $localip = $migrate_network_addr;
4993                     } else {
4994                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4995                     }
4996
4997                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4998                 }
4999
5000                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5001                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5002                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5003                 push @$cmd, '-incoming', $migrate_uri;
5004                 push @$cmd, '-S';
5005
5006             } elsif ($statefile eq 'unix') {
5007                 # should be default for secure migrations as a ssh TCP forward
5008                 # tunnel is not deterministic reliable ready and fails regurarly
5009                 # to set up in time, so use UNIX socket forwards
5010                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5011                 unlink $socket_addr;
5012
5013                 $migrate_uri = "unix:$socket_addr";
5014
5015                 push @$cmd, '-incoming', $migrate_uri;
5016                 push @$cmd, '-S';
5017
5018             } else {
5019                 push @$cmd, '-loadstate', $statefile;
5020             }
5021         } elsif ($paused) {
5022             push @$cmd, '-S';
5023         }
5024
5025         # host pci devices
5026         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5027           my $d = parse_hostpci($conf->{"hostpci$i"});
5028           next if !$d;
5029           my $pcidevices = $d->{pciid};
5030           foreach my $pcidevice (@$pcidevices) {
5031                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
5032
5033                 my $info = pci_device_info("0000:$pciid");
5034                 die "IOMMU not present\n" if !check_iommu_support();
5035                 die "no pci device info for device '$pciid'\n" if !$info;
5036                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
5037                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
5038           }
5039         }
5040
5041         PVE::Storage::activate_volumes($storecfg, $vollist);
5042
5043         if (!check_running($vmid, 1)) {
5044             eval {
5045                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5046                     outfunc => sub {}, errfunc => sub {});
5047             };
5048         }
5049
5050         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5051                                                   : $defaults->{cpuunits};
5052
5053         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5054         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5055
5056         my %properties = (
5057             Slice => 'qemu.slice',
5058             KillMode => 'none',
5059             CPUShares => $cpuunits
5060         );
5061
5062         if (my $cpulimit = $conf->{cpulimit}) {
5063             $properties{CPUQuota} = int($cpulimit * 100);
5064         }
5065         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5066
5067         my $run_qemu = sub {
5068             PVE::Tools::run_fork sub {
5069                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5070                 run_command($cmd, %run_params);
5071             };
5072         };
5073
5074         if ($conf->{hugepages}) {
5075
5076             my $code = sub {
5077                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5078                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5079
5080                 PVE::QemuServer::Memory::hugepages_mount();
5081                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5082
5083                 eval { $run_qemu->() };
5084                 if (my $err = $@) {
5085                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5086                     die $err;
5087                 }
5088
5089                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5090             };
5091             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5092
5093         } else {
5094             eval { $run_qemu->() };
5095         }
5096
5097         if (my $err = $@) {
5098             # deactivate volumes if start fails
5099             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5100             die "start failed: $err";
5101         }
5102
5103         print "migration listens on $migrate_uri\n" if $migrate_uri;
5104
5105         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5106             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5107             warn $@ if $@;
5108         }
5109
5110         #start nbd server for storage migration
5111         if ($targetstorage) {
5112             my $nodename = PVE::INotify::nodename();
5113             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5114             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5115             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5116             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5117
5118             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5119
5120             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5121
5122             foreach my $opt (sort keys %$local_volumes) {
5123                 my $volid = $local_volumes->{$opt};
5124                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5125                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5126                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5127             }
5128         }
5129
5130         if ($migratedfrom) {
5131             eval {
5132                 set_migration_caps($vmid);
5133             };
5134             warn $@ if $@;
5135
5136             if ($spice_port) {
5137                 print "spice listens on port $spice_port\n";
5138                 if ($spice_ticket) {
5139                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5140                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5141                 }
5142             }
5143
5144         } else {
5145             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5146                 if !$statefile && $conf->{balloon};
5147
5148             foreach my $opt (keys %$conf) {
5149                 next if $opt !~  m/^net\d+$/;
5150                 my $nicconf = parse_net($conf->{$opt});
5151                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5152             }
5153         }
5154
5155         vm_mon_cmd_nocheck($vmid, 'qom-set',
5156                     path => "machine/peripheral/balloon0",
5157                     property => "guest-stats-polling-interval",
5158                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5159
5160     });
5161 }
5162
5163 sub vm_mon_cmd {
5164     my ($vmid, $execute, %params) = @_;
5165
5166     my $cmd = { execute => $execute, arguments => \%params };
5167     vm_qmp_command($vmid, $cmd);
5168 }
5169
5170 sub vm_mon_cmd_nocheck {
5171     my ($vmid, $execute, %params) = @_;
5172
5173     my $cmd = { execute => $execute, arguments => \%params };
5174     vm_qmp_command($vmid, $cmd, 1);
5175 }
5176
5177 sub vm_qmp_command {
5178     my ($vmid, $cmd, $nocheck) = @_;
5179
5180     my $res;
5181
5182     my $timeout;
5183     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5184         $timeout = $cmd->{arguments}->{timeout};
5185         delete $cmd->{arguments}->{timeout};
5186     }
5187
5188     eval {
5189         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5190         my $sname = qmp_socket($vmid);
5191         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5192             my $qmpclient = PVE::QMPClient->new();
5193
5194             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5195         } else {
5196             die "unable to open monitor socket\n";
5197         }
5198     };
5199     if (my $err = $@) {
5200         syslog("err", "VM $vmid qmp command failed - $err");
5201         die $err;
5202     }
5203
5204     return $res;
5205 }
5206
5207 sub vm_human_monitor_command {
5208     my ($vmid, $cmdline) = @_;
5209
5210     my $res;
5211
5212     my $cmd = {
5213         execute => 'human-monitor-command',
5214         arguments => { 'command-line' => $cmdline},
5215     };
5216
5217     return vm_qmp_command($vmid, $cmd);
5218 }
5219
5220 sub vm_commandline {
5221     my ($storecfg, $vmid) = @_;
5222
5223     my $conf = PVE::QemuConfig->load_config($vmid);
5224
5225     my $defaults = load_defaults();
5226
5227     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5228
5229     return PVE::Tools::cmd2string($cmd);
5230 }
5231
5232 sub vm_reset {
5233     my ($vmid, $skiplock) = @_;
5234
5235     PVE::QemuConfig->lock_config($vmid, sub {
5236
5237         my $conf = PVE::QemuConfig->load_config($vmid);
5238
5239         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5240
5241         vm_mon_cmd($vmid, "system_reset");
5242     });
5243 }
5244
5245 sub get_vm_volumes {
5246     my ($conf) = @_;
5247
5248     my $vollist = [];
5249     foreach_volid($conf, sub {
5250         my ($volid, $attr) = @_;
5251
5252         return if $volid =~ m|^/|;
5253
5254         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5255         return if !$sid;
5256
5257         push @$vollist, $volid;
5258     });
5259
5260     return $vollist;
5261 }
5262
5263 sub vm_stop_cleanup {
5264     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5265
5266     eval {
5267
5268         if (!$keepActive) {
5269             my $vollist = get_vm_volumes($conf);
5270             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5271         }
5272
5273         foreach my $ext (qw(mon qmp pid vnc qga)) {
5274             unlink "/var/run/qemu-server/${vmid}.$ext";
5275         }
5276
5277         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5278     };
5279     warn $@ if $@; # avoid errors - just warn
5280 }
5281
5282 # Note: use $nockeck to skip tests if VM configuration file exists.
5283 # We need that when migration VMs to other nodes (files already moved)
5284 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5285 sub vm_stop {
5286     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5287
5288     $force = 1 if !defined($force) && !$shutdown;
5289
5290     if ($migratedfrom){
5291         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5292         kill 15, $pid if $pid;
5293         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5294         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5295         return;
5296     }
5297
5298     PVE::QemuConfig->lock_config($vmid, sub {
5299
5300         my $pid = check_running($vmid, $nocheck);
5301         return if !$pid;
5302
5303         my $conf;
5304         if (!$nocheck) {
5305             $conf = PVE::QemuConfig->load_config($vmid);
5306             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5307             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5308                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5309                 $timeout = $opts->{down} if $opts->{down};
5310             }
5311         }
5312
5313         $timeout = 60 if !defined($timeout);
5314
5315         eval {
5316             if ($shutdown) {
5317                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5318                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5319                 } else {
5320                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5321                 }
5322             } else {
5323                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5324             }
5325         };
5326         my $err = $@;
5327
5328         if (!$err) {
5329             my $count = 0;
5330             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5331                 $count++;
5332                 sleep 1;
5333             }
5334
5335             if ($count >= $timeout) {
5336                 if ($force) {
5337                     warn "VM still running - terminating now with SIGTERM\n";
5338                     kill 15, $pid;
5339                 } else {
5340                     die "VM quit/powerdown failed - got timeout\n";
5341                 }
5342             } else {
5343                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5344                 return;
5345             }
5346         } else {
5347             if ($force) {
5348                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5349                 kill 15, $pid;
5350             } else {
5351                 die "VM quit/powerdown failed\n";
5352             }
5353         }
5354
5355         # wait again
5356         $timeout = 10;
5357
5358         my $count = 0;
5359         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5360             $count++;
5361             sleep 1;
5362         }
5363
5364         if ($count >= $timeout) {
5365             warn "VM still running - terminating now with SIGKILL\n";
5366             kill 9, $pid;
5367             sleep 1;
5368         }
5369
5370         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5371    });
5372 }
5373
5374 sub vm_suspend {
5375     my ($vmid, $skiplock) = @_;
5376
5377     PVE::QemuConfig->lock_config($vmid, sub {
5378
5379         my $conf = PVE::QemuConfig->load_config($vmid);
5380
5381         PVE::QemuConfig->check_lock($conf)
5382             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5383
5384         vm_mon_cmd($vmid, "stop");
5385     });
5386 }
5387
5388 sub vm_resume {
5389     my ($vmid, $skiplock, $nocheck) = @_;
5390
5391     PVE::QemuConfig->lock_config($vmid, sub {
5392
5393         my $res = vm_mon_cmd($vmid, 'query-status');
5394         my $resume_cmd = 'cont';
5395
5396         if ($res->{status} && $res->{status} eq 'suspended') {
5397             $resume_cmd = 'system_wakeup';
5398         }
5399
5400         if (!$nocheck) {
5401
5402             my $conf = PVE::QemuConfig->load_config($vmid);
5403
5404             PVE::QemuConfig->check_lock($conf)
5405                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5406
5407             vm_mon_cmd($vmid, $resume_cmd);
5408
5409         } else {
5410             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5411         }
5412     });
5413 }
5414
5415 sub vm_sendkey {
5416     my ($vmid, $skiplock, $key) = @_;
5417
5418     PVE::QemuConfig->lock_config($vmid, sub {
5419
5420         my $conf = PVE::QemuConfig->load_config($vmid);
5421
5422         # there is no qmp command, so we use the human monitor command
5423         vm_human_monitor_command($vmid, "sendkey $key");
5424     });
5425 }
5426
5427 sub vm_destroy {
5428     my ($storecfg, $vmid, $skiplock) = @_;
5429
5430     PVE::QemuConfig->lock_config($vmid, sub {
5431
5432         my $conf = PVE::QemuConfig->load_config($vmid);
5433
5434         if (!check_running($vmid)) {
5435             destroy_vm($storecfg, $vmid, undef, $skiplock);
5436         } else {
5437             die "VM $vmid is running - destroy failed\n";
5438         }
5439     });
5440 }
5441
5442 # pci helpers
5443
5444 sub file_write {
5445     my ($filename, $buf) = @_;
5446
5447     my $fh = IO::File->new($filename, "w");
5448     return undef if !$fh;
5449
5450     my $res = print $fh $buf;
5451
5452     $fh->close();
5453
5454     return $res;
5455 }
5456
5457 sub pci_device_info {
5458     my ($name) = @_;
5459
5460     my $res;
5461
5462     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5463     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5464
5465     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5466     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5467
5468     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5469     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5470
5471     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5472     return undef if !defined($product) || $product !~ s/^0x//;
5473
5474     $res = {
5475         name => $name,
5476         vendor => $vendor,
5477         product => $product,
5478         domain => $domain,
5479         bus => $bus,
5480         slot => $slot,
5481         func => $func,
5482         irq => $irq,
5483         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5484     };
5485
5486     return $res;
5487 }
5488
5489 sub pci_dev_reset {
5490     my ($dev) = @_;
5491
5492     my $name = $dev->{name};
5493
5494     my $fn = "$pcisysfs/devices/$name/reset";
5495
5496     return file_write($fn, "1");
5497 }
5498
5499 sub pci_dev_bind_to_vfio {
5500     my ($dev) = @_;
5501
5502     my $name = $dev->{name};
5503
5504     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5505
5506     if (!-d $vfio_basedir) {
5507         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5508     }
5509     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5510
5511     my $testdir = "$vfio_basedir/$name";
5512     return 1 if -d $testdir;
5513
5514     my $data = "$dev->{vendor} $dev->{product}";
5515     return undef if !file_write("$vfio_basedir/new_id", $data);
5516
5517     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5518     if (!file_write($fn, $name)) {
5519         return undef if -f $fn;
5520     }
5521
5522     $fn = "$vfio_basedir/bind";
5523     if (! -d $testdir) {
5524         return undef if !file_write($fn, $name);
5525     }
5526
5527     return -d $testdir;
5528 }
5529
5530 sub pci_dev_group_bind_to_vfio {
5531     my ($pciid) = @_;
5532
5533     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5534
5535     if (!-d $vfio_basedir) {
5536         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5537     }
5538     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5539
5540     # get IOMMU group devices
5541     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5542       my @devs = grep /^0000:/, readdir($D);
5543     closedir($D);
5544
5545     foreach my $pciid (@devs) {
5546         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5547
5548         # pci bridges, switches or root ports are not supported
5549         # they have a pci_bus subdirectory so skip them
5550         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5551
5552         my $info = pci_device_info($1);
5553         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5554     }
5555
5556     return 1;
5557 }
5558
5559 # vzdump restore implementaion
5560
5561 sub tar_archive_read_firstfile {
5562     my $archive = shift;
5563
5564     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5565
5566     # try to detect archive type first
5567     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5568         die "unable to open file '$archive'\n";
5569     my $firstfile = <$fh>;
5570     kill 15, $pid;
5571     close $fh;
5572
5573     die "ERROR: archive contaions no data\n" if !$firstfile;
5574     chomp $firstfile;
5575
5576     return $firstfile;
5577 }
5578
5579 sub tar_restore_cleanup {
5580     my ($storecfg, $statfile) = @_;
5581
5582     print STDERR "starting cleanup\n";
5583
5584     if (my $fd = IO::File->new($statfile, "r")) {
5585         while (defined(my $line = <$fd>)) {
5586             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5587                 my $volid = $2;
5588                 eval {
5589                     if ($volid =~ m|^/|) {
5590                         unlink $volid || die 'unlink failed\n';
5591                     } else {
5592                         PVE::Storage::vdisk_free($storecfg, $volid);
5593                     }
5594                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5595                 };
5596                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5597             } else {
5598                 print STDERR "unable to parse line in statfile - $line";
5599             }
5600         }
5601         $fd->close();
5602     }
5603 }
5604
5605 sub restore_archive {
5606     my ($archive, $vmid, $user, $opts) = @_;
5607
5608     my $format = $opts->{format};
5609     my $comp;
5610
5611     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5612         $format = 'tar' if !$format;
5613         $comp = 'gzip';
5614     } elsif ($archive =~ m/\.tar$/) {
5615         $format = 'tar' if !$format;
5616     } elsif ($archive =~ m/.tar.lzo$/) {
5617         $format = 'tar' if !$format;
5618         $comp = 'lzop';
5619     } elsif ($archive =~ m/\.vma$/) {
5620         $format = 'vma' if !$format;
5621     } elsif ($archive =~ m/\.vma\.gz$/) {
5622         $format = 'vma' if !$format;
5623         $comp = 'gzip';
5624     } elsif ($archive =~ m/\.vma\.lzo$/) {
5625         $format = 'vma' if !$format;
5626         $comp = 'lzop';
5627     } else {
5628         $format = 'vma' if !$format; # default
5629     }
5630
5631     # try to detect archive format
5632     if ($format eq 'tar') {
5633         return restore_tar_archive($archive, $vmid, $user, $opts);
5634     } else {
5635         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5636     }
5637 }
5638
5639 sub restore_update_config_line {
5640     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5641
5642     return if $line =~ m/^\#qmdump\#/;
5643     return if $line =~ m/^\#vzdump\#/;
5644     return if $line =~ m/^lock:/;
5645     return if $line =~ m/^unused\d+:/;
5646     return if $line =~ m/^parent:/;
5647     return if $line =~ m/^template:/; # restored VM is never a template
5648
5649     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5650     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5651         # try to convert old 1.X settings
5652         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5653         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5654             my ($model, $macaddr) = split(/\=/, $devconfig);
5655             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5656             my $net = {
5657                 model => $model,
5658                 bridge => "vmbr$ind",
5659                 macaddr => $macaddr,
5660             };
5661             my $netstr = print_net($net);
5662
5663             print $outfd "net$cookie->{netcount}: $netstr\n";
5664             $cookie->{netcount}++;
5665         }
5666     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5667         my ($id, $netstr) = ($1, $2);
5668         my $net = parse_net($netstr);
5669         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5670         $netstr = print_net($net);
5671         print $outfd "$id: $netstr\n";
5672     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5673         my $virtdev = $1;
5674         my $value = $3;
5675         my $di = parse_drive($virtdev, $value);
5676         if (defined($di->{backup}) && !$di->{backup}) {
5677             print $outfd "#$line";
5678         } elsif ($map->{$virtdev}) {
5679             delete $di->{format}; # format can change on restore
5680             $di->{file} = $map->{$virtdev};
5681             $value = print_drive($vmid, $di);
5682             print $outfd "$virtdev: $value\n";
5683         } else {
5684             print $outfd $line;
5685         }
5686     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5687         my $vmgenid = $1;
5688         if ($vmgenid ne '0') {
5689             # always generate a new vmgenid if there was a valid one setup
5690             $vmgenid = generate_uuid();
5691         }
5692         print $outfd "vmgenid: $vmgenid\n";
5693     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5694         my ($uuid, $uuid_str);
5695         UUID::generate($uuid);
5696         UUID::unparse($uuid, $uuid_str);
5697         my $smbios1 = parse_smbios1($2);
5698         $smbios1->{uuid} = $uuid_str;
5699         print $outfd $1.print_smbios1($smbios1)."\n";
5700     } else {
5701         print $outfd $line;
5702     }
5703 }
5704
5705 sub scan_volids {
5706     my ($cfg, $vmid) = @_;
5707
5708     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5709
5710     my $volid_hash = {};
5711     foreach my $storeid (keys %$info) {
5712         foreach my $item (@{$info->{$storeid}}) {
5713             next if !($item->{volid} && $item->{size});
5714             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5715             $volid_hash->{$item->{volid}} = $item;
5716         }
5717     }
5718
5719     return $volid_hash;
5720 }
5721
5722 sub is_volume_in_use {
5723     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5724
5725     my $path = PVE::Storage::path($storecfg, $volid);
5726
5727     my $scan_config = sub {
5728         my ($cref, $snapname) = @_;
5729
5730         foreach my $key (keys %$cref) {
5731             my $value = $cref->{$key};
5732             if (is_valid_drivename($key)) {
5733                 next if $skip_drive && $key eq $skip_drive;
5734                 my $drive = parse_drive($key, $value);
5735                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5736                 return 1 if $volid eq $drive->{file};
5737                 if ($drive->{file} =~ m!^/!) {
5738                     return 1 if $drive->{file} eq $path;
5739                 } else {
5740                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5741                     next if !$storeid;
5742                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5743                     next if !$scfg;
5744                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5745                 }
5746             }
5747         }
5748
5749         return 0;
5750     };
5751
5752     return 1 if &$scan_config($conf);
5753
5754     undef $skip_drive;
5755
5756     foreach my $snapname (keys %{$conf->{snapshots}}) {
5757         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5758     }
5759
5760     return 0;
5761 }
5762
5763 sub update_disksize {
5764     my ($vmid, $conf, $volid_hash) = @_;
5765
5766     my $changes;
5767     my $prefix = "VM $vmid:";
5768
5769     # used and unused disks
5770     my $referenced = {};
5771
5772     # Note: it is allowed to define multiple storages with same path (alias), so
5773     # we need to check both 'volid' and real 'path' (two different volid can point
5774     # to the same path).
5775
5776     my $referencedpath = {};
5777
5778     # update size info
5779     foreach my $opt (keys %$conf) {
5780         if (is_valid_drivename($opt)) {
5781             my $drive = parse_drive($opt, $conf->{$opt});
5782             my $volid = $drive->{file};
5783             next if !$volid;
5784
5785             $referenced->{$volid} = 1;
5786             if ($volid_hash->{$volid} &&
5787                 (my $path = $volid_hash->{$volid}->{path})) {
5788                 $referencedpath->{$path} = 1;
5789             }
5790
5791             next if drive_is_cdrom($drive);
5792             next if !$volid_hash->{$volid};
5793
5794             $drive->{size} = $volid_hash->{$volid}->{size};
5795             my $new = print_drive($vmid, $drive);
5796             if ($new ne $conf->{$opt}) {
5797                 $changes = 1;
5798                 $conf->{$opt} = $new;
5799                 print "$prefix update disk '$opt' information.\n";
5800             }
5801         }
5802     }
5803
5804     # remove 'unusedX' entry if volume is used
5805     foreach my $opt (keys %$conf) {
5806         next if $opt !~ m/^unused\d+$/;
5807         my $volid = $conf->{$opt};
5808         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5809         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5810             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5811             $changes = 1;
5812             delete $conf->{$opt};
5813         }
5814
5815         $referenced->{$volid} = 1;
5816         $referencedpath->{$path} = 1 if $path;
5817     }
5818
5819     foreach my $volid (sort keys %$volid_hash) {
5820         next if $volid =~ m/vm-$vmid-state-/;
5821         next if $referenced->{$volid};
5822         my $path = $volid_hash->{$volid}->{path};
5823         next if !$path; # just to be sure
5824         next if $referencedpath->{$path};
5825         $changes = 1;
5826         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5827         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5828         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5829     }
5830
5831     return $changes;
5832 }
5833
5834 sub rescan {
5835     my ($vmid, $nolock, $dryrun) = @_;
5836
5837     my $cfg = PVE::Storage::config();
5838
5839     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5840     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5841     foreach my $stor (keys %{$cfg->{ids}}) {
5842         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5843     }
5844
5845     print "rescan volumes...\n";
5846     my $volid_hash = scan_volids($cfg, $vmid);
5847
5848     my $updatefn =  sub {
5849         my ($vmid) = @_;
5850
5851         my $conf = PVE::QemuConfig->load_config($vmid);
5852
5853         PVE::QemuConfig->check_lock($conf);
5854
5855         my $vm_volids = {};
5856         foreach my $volid (keys %$volid_hash) {
5857             my $info = $volid_hash->{$volid};
5858             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5859         }
5860
5861         my $changes = update_disksize($vmid, $conf, $vm_volids);
5862
5863         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5864     };
5865
5866     if (defined($vmid)) {
5867         if ($nolock) {
5868             &$updatefn($vmid);
5869         } else {
5870             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5871         }
5872     } else {
5873         my $vmlist = config_list();
5874         foreach my $vmid (keys %$vmlist) {
5875             if ($nolock) {
5876                 &$updatefn($vmid);
5877             } else {
5878                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5879             }
5880         }
5881     }
5882 }
5883
5884 sub restore_vma_archive {
5885     my ($archive, $vmid, $user, $opts, $comp) = @_;
5886
5887     my $readfrom = $archive;
5888
5889     my $cfg = PVE::Storage::config();
5890     my $commands = [];
5891     my $bwlimit = $opts->{bwlimit};
5892
5893     my $dbg_cmdstring = '';
5894     my $add_pipe = sub {
5895         my ($cmd) = @_;
5896         push @$commands, $cmd;
5897         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5898         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5899         $readfrom = '-';
5900     };
5901
5902     my $input = undef;
5903     if ($archive eq '-') {
5904         $input = '<&STDIN';
5905     } else {
5906         # If we use a backup from a PVE defined storage we also consider that
5907         # storage's rate limit:
5908         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5909         if (defined($volid)) {
5910             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5911             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5912             if ($readlimit) {
5913                 print STDERR "applying read rate limit: $readlimit\n";
5914                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5915                 $add_pipe->($cstream);
5916             }
5917         }
5918     }
5919
5920     if ($comp) {
5921         my $cmd;
5922         if ($comp eq 'gzip') {
5923             $cmd = ['zcat', $readfrom];
5924         } elsif ($comp eq 'lzop') {
5925             $cmd = ['lzop', '-d', '-c', $readfrom];
5926         } else {
5927             die "unknown compression method '$comp'\n";
5928         }
5929         $add_pipe->($cmd);
5930     }
5931
5932     my $tmpdir = "/var/tmp/vzdumptmp$$";
5933     rmtree $tmpdir;
5934
5935     # disable interrupts (always do cleanups)
5936     local $SIG{INT} =
5937         local $SIG{TERM} =
5938         local $SIG{QUIT} =
5939         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5940
5941     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5942     POSIX::mkfifo($mapfifo, 0600);
5943     my $fifofh;
5944
5945     my $openfifo = sub {
5946         open($fifofh, '>', $mapfifo) || die $!;
5947     };
5948
5949     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5950
5951     my $oldtimeout;
5952     my $timeout = 5;
5953
5954     my $devinfo = {};
5955
5956     my $rpcenv = PVE::RPCEnvironment::get();
5957
5958     my $conffile = PVE::QemuConfig->config_file($vmid);
5959     my $tmpfn = "$conffile.$$.tmp";
5960
5961     # Note: $oldconf is undef if VM does not exists
5962     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5963     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5964
5965     my %storage_limits;
5966
5967     my $print_devmap = sub {
5968         my $virtdev_hash = {};
5969
5970         my $cfgfn = "$tmpdir/qemu-server.conf";
5971
5972         # we can read the config - that is already extracted
5973         my $fh = IO::File->new($cfgfn, "r") ||
5974             "unable to read qemu-server.conf - $!\n";
5975
5976         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5977         if (-f $fwcfgfn) {
5978             my $pve_firewall_dir = '/etc/pve/firewall';
5979             mkdir $pve_firewall_dir; # make sure the dir exists
5980             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5981         }
5982
5983         while (defined(my $line = <$fh>)) {
5984             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5985                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5986                 die "archive does not contain data for drive '$virtdev'\n"
5987                     if !$devinfo->{$devname};
5988                 if (defined($opts->{storage})) {
5989                     $storeid = $opts->{storage} || 'local';
5990                 } elsif (!$storeid) {
5991                     $storeid = 'local';
5992                 }
5993                 $format = 'raw' if !$format;
5994                 $devinfo->{$devname}->{devname} = $devname;
5995                 $devinfo->{$devname}->{virtdev} = $virtdev;
5996                 $devinfo->{$devname}->{format} = $format;
5997                 $devinfo->{$devname}->{storeid} = $storeid;
5998
5999                 # check permission on storage
6000                 my $pool = $opts->{pool}; # todo: do we need that?
6001                 if ($user ne 'root@pam') {
6002                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6003                 }
6004
6005                 $storage_limits{$storeid} = $bwlimit;
6006
6007                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6008             }
6009         }
6010
6011         foreach my $key (keys %storage_limits) {
6012             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6013             next if !$limit;
6014             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6015             $storage_limits{$key} = $limit * 1024;
6016         }
6017
6018         foreach my $devname (keys %$devinfo) {
6019             die "found no device mapping information for device '$devname'\n"
6020                 if !$devinfo->{$devname}->{virtdev};
6021         }
6022
6023         # create empty/temp config
6024         if ($oldconf) {
6025             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6026             foreach_drive($oldconf, sub {
6027                 my ($ds, $drive) = @_;
6028
6029                 return if drive_is_cdrom($drive);
6030
6031                 my $volid = $drive->{file};
6032
6033                 return if !$volid || $volid =~ m|^/|;
6034
6035                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6036                 return if !$path || !$owner || ($owner != $vmid);
6037
6038                 # Note: only delete disk we want to restore
6039                 # other volumes will become unused
6040                 if ($virtdev_hash->{$ds}) {
6041                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6042                     if (my $err = $@) {
6043                         warn $err;
6044                     }
6045                 }
6046             });
6047
6048             # delete vmstate files
6049             # since after the restore we have no snapshots anymore
6050             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6051                 my $snap = $oldconf->{snapshots}->{$snapname};
6052                 if ($snap->{vmstate}) {
6053                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6054                     if (my $err = $@) {
6055                         warn $err;
6056                     }
6057                 }
6058             }
6059         }
6060
6061         my $map = {};
6062         foreach my $virtdev (sort keys %$virtdev_hash) {
6063             my $d = $virtdev_hash->{$virtdev};
6064             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6065             my $storeid = $d->{storeid};
6066             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6067
6068             my $map_opts = '';
6069             if (my $limit = $storage_limits{$storeid}) {
6070                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6071             }
6072
6073             # test if requested format is supported
6074             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6075             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6076             $d->{format} = $defFormat if !$supported;
6077
6078             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6079                                                   $d->{format}, undef, $alloc_size);
6080             print STDERR "new volume ID is '$volid'\n";
6081             $d->{volid} = $volid;
6082             my $path = PVE::Storage::path($cfg, $volid);
6083
6084             PVE::Storage::activate_volumes($cfg,[$volid]);
6085
6086             my $write_zeros = 1;
6087             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6088                 $write_zeros = 0;
6089             }
6090
6091             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6092
6093             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6094             $map->{$virtdev} = $volid;
6095         }
6096
6097         $fh->seek(0, 0) || die "seek failed - $!\n";
6098
6099         my $outfd = new IO::File ($tmpfn, "w") ||
6100             die "unable to write config for VM $vmid\n";
6101
6102         my $cookie = { netcount => 0 };
6103         while (defined(my $line = <$fh>)) {
6104             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6105         }
6106
6107         $fh->close();
6108         $outfd->close();
6109     };
6110
6111     eval {
6112         # enable interrupts
6113         local $SIG{INT} =
6114             local $SIG{TERM} =
6115             local $SIG{QUIT} =
6116             local $SIG{HUP} =
6117             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6118         local $SIG{ALRM} = sub { die "got timeout\n"; };
6119
6120         $oldtimeout = alarm($timeout);
6121
6122         my $parser = sub {
6123             my $line = shift;
6124
6125             print "$line\n";
6126
6127             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6128                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6129                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6130             } elsif ($line =~ m/^CTIME: /) {
6131                 # we correctly received the vma config, so we can disable
6132                 # the timeout now for disk allocation (set to 10 minutes, so
6133                 # that we always timeout if something goes wrong)
6134                 alarm(600);
6135                 &$print_devmap();
6136                 print $fifofh "done\n";
6137                 my $tmp = $oldtimeout || 0;
6138                 $oldtimeout = undef;
6139                 alarm($tmp);
6140                 close($fifofh);
6141             }
6142         };
6143
6144         print "restore vma archive: $dbg_cmdstring\n";
6145         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6146     };
6147     my $err = $@;
6148
6149     alarm($oldtimeout) if $oldtimeout;
6150
6151     my $vollist = [];
6152     foreach my $devname (keys %$devinfo) {
6153         my $volid = $devinfo->{$devname}->{volid};
6154         push @$vollist, $volid if $volid;
6155     }
6156
6157     PVE::Storage::deactivate_volumes($cfg, $vollist);
6158
6159     unlink $mapfifo;
6160
6161     if ($err) {
6162         rmtree $tmpdir;
6163         unlink $tmpfn;
6164
6165         foreach my $devname (keys %$devinfo) {
6166             my $volid = $devinfo->{$devname}->{volid};
6167             next if !$volid;
6168             eval {
6169                 if ($volid =~ m|^/|) {
6170                     unlink $volid || die 'unlink failed\n';
6171                 } else {
6172                     PVE::Storage::vdisk_free($cfg, $volid);
6173                 }
6174                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6175             };
6176             print STDERR "unable to cleanup '$volid' - $@" if $@;
6177         }
6178         die $err;
6179     }
6180
6181     rmtree $tmpdir;
6182
6183     rename($tmpfn, $conffile) ||
6184         die "unable to commit configuration file '$conffile'\n";
6185
6186     PVE::Cluster::cfs_update(); # make sure we read new file
6187
6188     eval { rescan($vmid, 1); };
6189     warn $@ if $@;
6190 }
6191
6192 sub restore_tar_archive {
6193     my ($archive, $vmid, $user, $opts) = @_;
6194
6195     if ($archive ne '-') {
6196         my $firstfile = tar_archive_read_firstfile($archive);
6197         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6198             if $firstfile ne 'qemu-server.conf';
6199     }
6200
6201     my $storecfg = PVE::Storage::config();
6202
6203     # destroy existing data - keep empty config
6204     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6205     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6206
6207     my $tocmd = "/usr/lib/qemu-server/qmextract";
6208
6209     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6210     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6211     $tocmd .= ' --prealloc' if $opts->{prealloc};
6212     $tocmd .= ' --info' if $opts->{info};
6213
6214     # tar option "xf" does not autodetect compression when read from STDIN,
6215     # so we pipe to zcat
6216     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6217         PVE::Tools::shellquote("--to-command=$tocmd");
6218
6219     my $tmpdir = "/var/tmp/vzdumptmp$$";
6220     mkpath $tmpdir;
6221
6222     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6223     local $ENV{VZDUMP_VMID} = $vmid;
6224     local $ENV{VZDUMP_USER} = $user;
6225
6226     my $conffile = PVE::QemuConfig->config_file($vmid);
6227     my $tmpfn = "$conffile.$$.tmp";
6228
6229     # disable interrupts (always do cleanups)
6230     local $SIG{INT} =
6231         local $SIG{TERM} =
6232         local $SIG{QUIT} =
6233         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6234
6235     eval {
6236         # enable interrupts
6237         local $SIG{INT} =
6238             local $SIG{TERM} =
6239             local $SIG{QUIT} =
6240             local $SIG{HUP} =
6241             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6242
6243         if ($archive eq '-') {
6244             print "extracting archive from STDIN\n";
6245             run_command($cmd, input => "<&STDIN");
6246         } else {
6247             print "extracting archive '$archive'\n";
6248             run_command($cmd);
6249         }
6250
6251         return if $opts->{info};
6252
6253         # read new mapping
6254         my $map = {};
6255         my $statfile = "$tmpdir/qmrestore.stat";
6256         if (my $fd = IO::File->new($statfile, "r")) {
6257             while (defined (my $line = <$fd>)) {
6258                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6259                     $map->{$1} = $2 if $1;
6260                 } else {
6261                     print STDERR "unable to parse line in statfile - $line\n";
6262                 }
6263             }
6264             $fd->close();
6265         }
6266
6267         my $confsrc = "$tmpdir/qemu-server.conf";
6268
6269         my $srcfd = new IO::File($confsrc, "r") ||
6270             die "unable to open file '$confsrc'\n";
6271
6272         my $outfd = new IO::File ($tmpfn, "w") ||
6273             die "unable to write config for VM $vmid\n";
6274
6275         my $cookie = { netcount => 0 };
6276         while (defined (my $line = <$srcfd>)) {
6277             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6278         }
6279
6280         $srcfd->close();
6281         $outfd->close();
6282     };
6283     my $err = $@;
6284
6285     if ($err) {
6286
6287         unlink $tmpfn;
6288
6289         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6290
6291         die $err;
6292     }
6293
6294     rmtree $tmpdir;
6295
6296     rename $tmpfn, $conffile ||
6297         die "unable to commit configuration file '$conffile'\n";
6298
6299     PVE::Cluster::cfs_update(); # make sure we read new file
6300
6301     eval { rescan($vmid, 1); };
6302     warn $@ if $@;
6303 };
6304
6305 sub foreach_storage_used_by_vm {
6306     my ($conf, $func) = @_;
6307
6308     my $sidhash = {};
6309
6310     foreach_drive($conf, sub {
6311         my ($ds, $drive) = @_;
6312         return if drive_is_cdrom($drive);
6313
6314         my $volid = $drive->{file};
6315
6316         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6317         $sidhash->{$sid} = $sid if $sid;
6318     });
6319
6320     foreach my $sid (sort keys %$sidhash) {
6321         &$func($sid);
6322     }
6323 }
6324
6325 sub do_snapshots_with_qemu {
6326     my ($storecfg, $volid) = @_;
6327
6328     my $storage_name = PVE::Storage::parse_volume_id($volid);
6329
6330     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6331         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6332         return 1;
6333     }
6334
6335     if ($volid =~ m/\.(qcow2|qed)$/){
6336         return 1;
6337     }
6338
6339     return undef;
6340 }
6341
6342 sub qga_check_running {
6343     my ($vmid, $nowarn) = @_;
6344
6345     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6346     if ($@) {
6347         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6348         return 0;
6349     }
6350     return 1;
6351 }
6352
6353 sub template_create {
6354     my ($vmid, $conf, $disk) = @_;
6355
6356     my $storecfg = PVE::Storage::config();
6357
6358     foreach_drive($conf, sub {
6359         my ($ds, $drive) = @_;
6360
6361         return if drive_is_cdrom($drive);
6362         return if $disk && $ds ne $disk;
6363
6364         my $volid = $drive->{file};
6365         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6366
6367         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6368         $drive->{file} = $voliddst;
6369         $conf->{$ds} = print_drive($vmid, $drive);
6370         PVE::QemuConfig->write_config($vmid, $conf);
6371     });
6372 }
6373
6374 sub qemu_img_convert {
6375     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6376
6377     my $storecfg = PVE::Storage::config();
6378     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6379     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6380
6381     if ($src_storeid && $dst_storeid) {
6382
6383         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6384
6385         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6386         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6387
6388         my $src_format = qemu_img_format($src_scfg, $src_volname);
6389         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6390
6391         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6392         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6393
6394         my $cmd = [];
6395         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6396         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6397         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6398         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6399         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6400         if ($is_zero_initialized) {
6401             push @$cmd, "zeroinit:$dst_path";
6402         } else {
6403             push @$cmd, $dst_path;
6404         }
6405
6406         my $parser = sub {
6407             my $line = shift;
6408             if($line =~ m/\((\S+)\/100\%\)/){
6409                 my $percent = $1;
6410                 my $transferred = int($size * $percent / 100);
6411                 my $remaining = $size - $transferred;
6412
6413                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6414             }
6415
6416         };
6417
6418         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6419         my $err = $@;
6420         die "copy failed: $err" if $err;
6421     }
6422 }
6423
6424 sub qemu_img_format {
6425     my ($scfg, $volname) = @_;
6426
6427     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6428         return $1;
6429     } else {
6430         return "raw";
6431     }
6432 }
6433
6434 sub qemu_drive_mirror {
6435     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6436
6437     $jobs = {} if !$jobs;
6438
6439     my $qemu_target;
6440     my $format;
6441     $jobs->{"drive-$drive"} = {};
6442
6443     if ($dst_volid =~ /^nbd:/) {
6444         $qemu_target = $dst_volid;
6445         $format = "nbd";
6446     } else {
6447         my $storecfg = PVE::Storage::config();
6448         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6449
6450         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6451
6452         $format = qemu_img_format($dst_scfg, $dst_volname);
6453
6454         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6455
6456         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6457     }
6458
6459     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6460     $opts->{format} = $format if $format;
6461
6462     print "drive mirror is starting for drive-$drive\n";
6463
6464     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6465
6466     if (my $err = $@) {
6467         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6468         die "mirroring error: $err";
6469     }
6470
6471     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6472 }
6473
6474 sub qemu_drive_mirror_monitor {
6475     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6476
6477     eval {
6478         my $err_complete = 0;
6479
6480         while (1) {
6481             die "storage migration timed out\n" if $err_complete > 300;
6482
6483             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6484
6485             my $running_mirror_jobs = {};
6486             foreach my $stat (@$stats) {
6487                 next if $stat->{type} ne 'mirror';
6488                 $running_mirror_jobs->{$stat->{device}} = $stat;
6489             }
6490
6491             my $readycounter = 0;
6492
6493             foreach my $job (keys %$jobs) {
6494
6495                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6496                     print "$job : finished\n";
6497                     delete $jobs->{$job};
6498                     next;
6499                 }
6500
6501                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6502
6503                 my $busy = $running_mirror_jobs->{$job}->{busy};
6504                 my $ready = $running_mirror_jobs->{$job}->{ready};
6505                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6506                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6507                     my $remaining = $total - $transferred;
6508                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6509
6510                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6511                 }
6512
6513                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6514             }
6515
6516             last if scalar(keys %$jobs) == 0;
6517
6518             if ($readycounter == scalar(keys %$jobs)) {
6519                 print "all mirroring jobs are ready \n";
6520                 last if $skipcomplete; #do the complete later
6521
6522                 if ($vmiddst && $vmiddst != $vmid) {
6523                     my $agent_running = $qga && qga_check_running($vmid);
6524                     if ($agent_running) {
6525                         print "freeze filesystem\n";
6526                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6527                     } else {
6528                         print "suspend vm\n";
6529                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6530                     }
6531
6532                     # if we clone a disk for a new target vm, we don't switch the disk
6533                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6534
6535                     if ($agent_running) {
6536                         print "unfreeze filesystem\n";
6537                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6538                     } else {
6539                         print "resume vm\n";
6540                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6541                     }
6542
6543                     last;
6544                 } else {
6545
6546                     foreach my $job (keys %$jobs) {
6547                         # try to switch the disk if source and destination are on the same guest
6548                         print "$job: Completing block job...\n";
6549
6550                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6551                         if ($@ =~ m/cannot be completed/) {
6552                             print "$job: Block job cannot be completed, try again.\n";
6553                             $err_complete++;
6554                         }else {
6555                             print "$job: Completed successfully.\n";
6556                             $jobs->{$job}->{complete} = 1;
6557                         }
6558                     }
6559                 }
6560             }
6561             sleep 1;
6562         }
6563     };
6564     my $err = $@;
6565
6566     if ($err) {
6567         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6568         die "mirroring error: $err";
6569     }
6570
6571 }
6572
6573 sub qemu_blockjobs_cancel {
6574     my ($vmid, $jobs) = @_;
6575
6576     foreach my $job (keys %$jobs) {
6577         print "$job: Cancelling block job\n";
6578         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6579         $jobs->{$job}->{cancel} = 1;
6580     }
6581
6582     while (1) {
6583         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6584
6585         my $running_jobs = {};
6586         foreach my $stat (@$stats) {
6587             $running_jobs->{$stat->{device}} = $stat;
6588         }
6589
6590         foreach my $job (keys %$jobs) {
6591
6592             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6593                 print "$job: Done.\n";
6594                 delete $jobs->{$job};
6595             }
6596         }
6597
6598         last if scalar(keys %$jobs) == 0;
6599
6600         sleep 1;
6601     }
6602 }
6603
6604 sub clone_disk {
6605     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6606         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6607
6608     my $newvolid;
6609
6610     if (!$full) {
6611         print "create linked clone of drive $drivename ($drive->{file})\n";
6612         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6613         push @$newvollist, $newvolid;
6614     } else {
6615
6616         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6617         $storeid = $storage if $storage;
6618
6619         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6620         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6621
6622         print "create full clone of drive $drivename ($drive->{file})\n";
6623         my $name = undef;
6624         if (drive_is_cloudinit($drive)) {
6625             $name = "vm-$newvmid-cloudinit";
6626             # cloudinit only supports raw and qcow2 atm:
6627             if ($dst_format eq 'qcow2') {
6628                 $name .= '.qcow2';
6629             } elsif ($dst_format ne 'raw') {
6630                 die "clone: unhandled format for cloudinit image\n";
6631             }
6632         }
6633         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6634         push @$newvollist, $newvolid;
6635
6636         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6637
6638         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6639         if (!$running || $snapname) {
6640             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6641         } else {
6642
6643             my $kvmver = get_running_qemu_version ($vmid);
6644             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6645                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6646                     if $drive->{iothread};
6647             }
6648
6649             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6650         }
6651     }
6652
6653     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6654
6655     my $disk = $drive;
6656     $disk->{format} = undef;
6657     $disk->{file} = $newvolid;
6658     $disk->{size} = $size;
6659
6660     return $disk;
6661 }
6662
6663 # this only works if VM is running
6664 sub get_current_qemu_machine {
6665     my ($vmid) = @_;
6666
6667     my $cmd = { execute => 'query-machines', arguments => {} };
6668     my $res = vm_qmp_command($vmid, $cmd);
6669
6670     my ($current, $default);
6671     foreach my $e (@$res) {
6672         $default = $e->{name} if $e->{'is-default'};
6673         $current = $e->{name} if $e->{'is-current'};
6674     }
6675
6676     # fallback to the default machine if current is not supported by qemu
6677     return $current || $default || 'pc';
6678 }
6679
6680 sub get_running_qemu_version {
6681     my ($vmid) = @_;
6682     my $cmd = { execute => 'query-version', arguments => {} };
6683     my $res = vm_qmp_command($vmid, $cmd);
6684     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6685 }
6686
6687 sub qemu_machine_feature_enabled {
6688     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6689
6690     my $current_major;
6691     my $current_minor;
6692
6693     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6694
6695         $current_major = $3;
6696         $current_minor = $4;
6697
6698     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6699
6700         $current_major = $1;
6701         $current_minor = $2;
6702     }
6703
6704     return 1 if $current_major > $version_major ||
6705                 ($current_major == $version_major &&
6706                  $current_minor >= $version_minor);
6707 }
6708
6709 sub qemu_machine_pxe {
6710     my ($vmid, $conf, $machine) = @_;
6711
6712     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6713
6714     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6715         $machine .= '.pxe';
6716     }
6717
6718     return $machine;
6719 }
6720
6721 sub qemu_use_old_bios_files {
6722     my ($machine_type) = @_;
6723
6724     return if !$machine_type;
6725
6726     my $use_old_bios_files = undef;
6727
6728     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6729         $machine_type = $1;
6730         $use_old_bios_files = 1;
6731     } else {
6732         my $kvmver = kvm_user_version();
6733         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6734         # load new efi bios files on migration. So this hack is required to allow
6735         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6736         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6737         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6738     }
6739
6740     return ($use_old_bios_files, $machine_type);
6741 }
6742
6743 sub create_efidisk {
6744     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6745
6746     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6747
6748     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6749     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6750     PVE::Storage::activate_volumes($storecfg, [$volid]);
6751
6752     my $path = PVE::Storage::path($storecfg, $volid);
6753     eval {
6754         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6755     };
6756     die "Copying EFI vars image failed: $@" if $@;
6757
6758     return ($volid, $vars_size);
6759 }
6760
6761 sub lspci {
6762
6763     my $devices = {};
6764
6765     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6766             my (undef, $id, $function) = @_;
6767             my $res = { id => $id, function => $function};
6768             push @{$devices->{$id}}, $res;
6769     });
6770
6771     # Entries should be sorted by functions.
6772     foreach my $id (keys %$devices) {
6773         my $dev = $devices->{$id};
6774         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6775     }
6776
6777     return $devices;
6778 }
6779
6780 sub vm_iothreads_list {
6781     my ($vmid) = @_;
6782
6783     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6784
6785     my $iothreads = {};
6786     foreach my $iothread (@$res) {
6787         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6788     }
6789
6790     return $iothreads;
6791 }
6792
6793 sub scsihw_infos {
6794     my ($conf, $drive) = @_;
6795
6796     my $maxdev = 0;
6797
6798     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6799         $maxdev = 7;
6800     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6801         $maxdev = 1;
6802     } else {
6803         $maxdev = 256;
6804     }
6805
6806     my $controller = int($drive->{index} / $maxdev);
6807     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6808
6809     return ($maxdev, $controller, $controller_prefix);
6810 }
6811
6812 sub add_hyperv_enlightenments {
6813     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6814
6815     return if $winversion < 6;
6816     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6817
6818     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6819
6820     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6821         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6822         push @$cpuFlags , 'hv_vapic';
6823         push @$cpuFlags , 'hv_time';
6824     } else {
6825         push @$cpuFlags , 'hv_spinlocks=0xffff';
6826     }
6827
6828     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6829         push @$cpuFlags , 'hv_reset';
6830         push @$cpuFlags , 'hv_vpindex';
6831         push @$cpuFlags , 'hv_runtime';
6832     }
6833
6834     if ($winversion >= 7) {
6835         push @$cpuFlags , 'hv_relaxed';
6836
6837         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6838             push @$cpuFlags , 'hv_synic';
6839             push @$cpuFlags , 'hv_stimer';
6840         }
6841     }
6842 }
6843
6844 sub windows_version {
6845     my ($ostype) = @_;
6846
6847     return 0 if !$ostype;
6848
6849     my $winversion = 0;
6850
6851     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6852         $winversion = 5;
6853     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6854         $winversion = 6;
6855     } elsif ($ostype =~ m/^win(\d+)$/) {
6856         $winversion = $1;
6857     }
6858
6859     return $winversion;
6860 }
6861
6862 sub resolve_dst_disk_format {
6863         my ($storecfg, $storeid, $src_volname, $format) = @_;
6864         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6865
6866         if (!$format) {
6867             # if no target format is specified, use the source disk format as hint
6868             if ($src_volname) {
6869                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6870                 $format = qemu_img_format($scfg, $src_volname);
6871             } else {
6872                 return $defFormat;
6873             }
6874         }
6875
6876         # test if requested format is supported - else use default
6877         my $supported = grep { $_ eq $format } @$validFormats;
6878         $format = $defFormat if !$supported;
6879         return $format;
6880 }
6881
6882 sub resolve_first_disk {
6883     my $conf = shift;
6884     my @disks = PVE::QemuServer::valid_drive_names();
6885     my $firstdisk;
6886     foreach my $ds (reverse @disks) {
6887         next if !$conf->{$ds};
6888         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6889         next if PVE::QemuServer::drive_is_cdrom($disk);
6890         $firstdisk = $ds;
6891     }
6892     return $firstdisk;
6893 }
6894
6895 sub generate_uuid {
6896     my ($uuid, $uuid_str);
6897     UUID::generate($uuid);
6898     UUID::unparse($uuid, $uuid_str);
6899     return $uuid_str;
6900 }
6901
6902 sub generate_smbios1_uuid {
6903     return "uuid=".generate_uuid();
6904 }
6905
6906 sub nbd_stop {
6907     my ($vmid) = @_;
6908
6909     vm_mon_cmd($vmid, 'nbd-server-stop');
6910 }
6911
6912 # bash completion helper
6913
6914 sub complete_backup_archives {
6915     my ($cmdname, $pname, $cvalue) = @_;
6916
6917     my $cfg = PVE::Storage::config();
6918
6919     my $storeid;
6920
6921     if ($cvalue =~ m/^([^:]+):/) {
6922         $storeid = $1;
6923     }
6924
6925     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6926
6927     my $res = [];
6928     foreach my $id (keys %$data) {
6929         foreach my $item (@{$data->{$id}}) {
6930             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6931             push @$res, $item->{volid} if defined($item->{volid});
6932         }
6933     }
6934
6935     return $res;
6936 }
6937
6938 my $complete_vmid_full = sub {
6939     my ($running) = @_;
6940
6941     my $idlist = vmstatus();
6942
6943     my $res = [];
6944
6945     foreach my $id (keys %$idlist) {
6946         my $d = $idlist->{$id};
6947         if (defined($running)) {
6948             next if $d->{template};
6949             next if $running && $d->{status} ne 'running';
6950             next if !$running && $d->{status} eq 'running';
6951         }
6952         push @$res, $id;
6953
6954     }
6955     return $res;
6956 };
6957
6958 sub complete_vmid {
6959     return &$complete_vmid_full();
6960 }
6961
6962 sub complete_vmid_stopped {
6963     return &$complete_vmid_full(0);
6964 }
6965
6966 sub complete_vmid_running {
6967     return &$complete_vmid_full(1);
6968 }
6969
6970 sub complete_storage {
6971
6972     my $cfg = PVE::Storage::config();
6973     my $ids = $cfg->{ids};
6974
6975     my $res = [];
6976     foreach my $sid (keys %$ids) {
6977         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6978         next if !$ids->{$sid}->{content}->{images};
6979         push @$res, $sid;
6980     }
6981
6982     return $res;
6983 }
6984
6985 1;