PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120     'Skylake-Client' => 'GenuineIntel',
 121
 122     # AMD CPUs
 123     athlon => 'AuthenticAMD',
 124     phenom  => 'AuthenticAMD',
 125     Opteron_G1  => 'AuthenticAMD',
 126     Opteron_G2  => 'AuthenticAMD',
 127     Opteron_G3  => 'AuthenticAMD',
 128     Opteron_G4  => 'AuthenticAMD',
 129     Opteron_G5  => 'AuthenticAMD',
 130
 131     # generic types, use vendor from host node
 132     host => 'default',
 133     kvm32 => 'default',
 134     kvm64 => 'default',
 135     qemu32 => 'default',
 136     qemu64 => 'default',
 137 };
 138
 139 my $cpu_fmt = {
 140     cputype => {
 141         description => "Emulated CPU type.",
 142         type => 'string',
 143         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 218         minimum => 0,
 219         maximum => 500000,
 220         default => 1024,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 247         enum => PVE::Tools::kvmkeymaplist(),
 248         default => 'en-us',
 249     },
 250     name => {
 251         optional => 1,
 252         type => 'string', format => 'dns-name',
 253         description => "Set a name for the VM. Only used on the configuration web interface.",
 254     },
 255     scsihw => {
 256         optional => 1,
 257         type => 'string',
 258         description => "SCSI controller model",
 259         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 260         default => 'lsi',
 261     },
 262     description => {
 263         optional => 1,
 264         type => 'string',
 265         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 266     },
 267     ostype => {
 268         optional => 1,
 269         type => 'string',
 270         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 271         description => "Specify guest operating system.",
 272         verbose_description => <<EODESC,
 273 Specify guest operating system. This is used to enable special
 274 optimization/features for specific operating systems:
 275
 276 [horizontal]
 277 other;; unspecified OS
 278 wxp;; Microsoft Windows XP
 279 w2k;; Microsoft Windows 2000
 280 w2k3;; Microsoft Windows 2003
 281 w2k8;; Microsoft Windows 2008
 282 wvista;; Microsoft Windows Vista
 283 win7;; Microsoft Windows 7
 284 win8;; Microsoft Windows 8/2012
 285 l24;; Linux 2.4 Kernel
 286 l26;; Linux 2.6/3.X Kernel
 287 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 288 EODESC
 289     },
 290     boot => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 294         pattern => '[acdn]{1,4}',
 295         default => 'cdn',
 296     },
 297     bootdisk => {
 298         optional => 1,
 299         type => 'string', format => 'pve-qm-bootdisk',
 300         description => "Enable booting from specified disk.",
 301         pattern => '(ide|sata|scsi|virtio)\d+',
 302     },
 303     smp => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "The number of CPUs. Please use option -sockets instead.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     sockets => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "The number of CPU sockets.",
 314         minimum => 1,
 315         default => 1,
 316     },
 317     cores => {
 318         optional => 1,
 319         type => 'integer',
 320         description => "The number of cores per socket.",
 321         minimum => 1,
 322         default => 1,
 323     },
 324     numa => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Enable/disable NUMA.",
 328         default => 0,
 329     },
 330     hugepages => {
 331         optional => 1,
 332         type => 'string',
 333         description => "Enable/disable hugepages memory.",
 334         enum => [qw(any 2 1024)],
 335     },
 336     vcpus => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Number of hotplugged vcpus.",
 340         minimum => 1,
 341         default => 0,
 342     },
 343     acpi => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable ACPI.",
 347         default => 1,
 348     },
 349     agent => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable Qemu GuestAgent.",
 353         default => 0,
 354     },
 355     kvm => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable KVM hardware virtualization.",
 359         default => 1,
 360     },
 361     tdf => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable time drift fix.",
 365         default => 0,
 366     },
 367     localtime => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 371     },
 372     freeze => {
 373         optional => 1,
 374         type => 'boolean',
 375         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 376     },
 377     vga => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Select the VGA type.",
 381         verbose_description => "Select the VGA type. If you want to use high resolution" .
 382             " modes (>= 1280x1024x16) then you should use the options " .
 383             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 384             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 385             "display sever. For win* OS you can select how many independent " .
 386             "displays you want, Linux guests can add displays them self. " .
 387             "You can also run without any graphic card, using a serial device" .
 388             " as terminal.",
 389         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 390     },
 391     watchdog => {
 392         optional => 1,
 393         type => 'string', format => 'pve-qm-watchdog',
 394         description => "Create a virtual hardware watchdog device.",
 395         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 396             " (by a guest action), the watchdog must be periodically polled " .
 397             "by an agent inside the guest or else the watchdog will reset " .
 398             "the guest (or execute the respective action specified)",
 399     },
 400     startdate => {
 401         optional => 1,
 402         type => 'string',
 403         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 404         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 405         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 406         default => 'now',
 407     },
 408     startup =>  get_standard_option('pve-startup-order'),
 409     template => {
 410         optional => 1,
 411         type => 'boolean',
 412         description => "Enable/disable Template.",
 413         default => 0,
 414     },
 415     args => {
 416         optional => 1,
 417         type => 'string',
 418         description => "Arbitrary arguments passed to kvm.",
 419         verbose_description => <<EODESCR,
 420 Arbitrary arguments passed to kvm, for example:
 421
 422 args: -no-reboot -no-hpet
 423
 424 NOTE: this option is for experts only.
 425 EODESCR
 426     },
 427     tablet => {
 428         optional => 1,
 429         type => 'boolean',
 430         default => 1,
 431         description => "Enable/disable the USB tablet device.",
 432         verbose_description => "Enable/disable the USB tablet device. This device is " .
 433             "usually needed to allow absolute mouse positioning with VNC. " .
 434             "Else the mouse runs out of sync with normal VNC clients. " .
 435             "If you're running lots of console-only guests on one host, " .
 436             "you may consider disabling this to save some context switches. " .
 437             "This is turned off by default if you use spice (-vga=qxl).",
 438     },
 439     migrate_speed => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 443         minimum => 0,
 444         default => 0,
 445     },
 446     migrate_downtime => {
 447         optional => 1,
 448         type => 'number',
 449         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 450         minimum => 0,
 451         default => 0.1,
 452     },
 453     cdrom => {
 454         optional => 1,
 455         type => 'string', format => 'pve-qm-ide',
 456         typetext => '<volume>',
 457         description => "This is an alias for option -ide2",
 458     },
 459     cpu => {
 460         optional => 1,
 461         description => "Emulated CPU type.",
 462         type => 'string',
 463         format => $cpu_fmt,
 464     },
 465     parent => get_standard_option('pve-snapshot-name', {
 466         optional => 1,
 467         description => "Parent snapshot name. This is used internally, and should not be modified.",
 468     }),
 469     snaptime => {
 470         optional => 1,
 471         description => "Timestamp for snapshots.",
 472         type => 'integer',
 473         minimum => 0,
 474     },
 475     vmstate => {
 476         optional => 1,
 477         type => 'string', format => 'pve-volume-id',
 478         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 479     },
 480     machine => {
 481         description => "Specific the Qemu machine type.",
 482         type => 'string',
 483         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 484         maxLength => 40,
 485         optional => 1,
 486     },
 487     smbios1 => {
 488         description => "Specify SMBIOS type 1 fields.",
 489         type => 'string', format => 'pve-qm-smbios1',
 490         maxLength => 256,
 491         optional => 1,
 492     },
 493     protection => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 497         default => 0,
 498     },
 499     bios => {
 500         optional => 1,
 501         type => 'string',
 502         enum => [ qw(seabios ovmf) ],
 503         description => "Select BIOS implementation.",
 504         default => 'seabios',
 505     },
 506 };
 507
 508 # what about other qemu settings ?
 509 #cpu => 'string',
 510 #machine => 'string',
 511 #fda => 'file',
 512 #fdb => 'file',
 513 #mtdblock => 'file',
 514 #sd => 'file',
 515 #pflash => 'file',
 516 #snapshot => 'bool',
 517 #bootp => 'file',
 518 ##tftp => 'dir',
 519 ##smb => 'dir',
 520 #kernel => 'file',
 521 #append => 'string',
 522 #initrd => 'file',
 523 ##soundhw => 'string',
 524
 525 while (my ($k, $v) = each %$confdesc) {
 526     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 527 }
 528
 529 my $MAX_IDE_DISKS = 4;
 530 my $MAX_SCSI_DISKS = 14;
 531 my $MAX_VIRTIO_DISKS = 16;
 532 my $MAX_SATA_DISKS = 6;
 533 my $MAX_USB_DEVICES = 5;
 534 my $MAX_NETS = 32;
 535 my $MAX_UNUSED_DISKS = 8;
 536 my $MAX_HOSTPCI_DEVICES = 4;
 537 my $MAX_SERIAL_PORTS = 4;
 538 my $MAX_PARALLEL_PORTS = 3;
 539 my $MAX_NUMA = 8;
 540
 541 my $numa_fmt = {
 542     cpus => {
 543         type => "string",
 544         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 545         description => "CPUs accessing this NUMA node.",
 546         format_description => "id[-id];...",
 547     },
 548     memory => {
 549         type => "number",
 550         description => "Amount of memory this NUMA node provides.",
 551         optional => 1,
 552     },
 553     hostnodes => {
 554         type => "string",
 555         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 556         description => "Host NUMA nodes to use.",
 557         format_description => "id[-id];...",
 558         optional => 1,
 559     },
 560     policy => {
 561         type => 'string',
 562         enum => [qw(preferred bind interleave)],
 563         description => "NUMA allocation policy.",
 564         optional => 1,
 565     },
 566 };
 567 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 568 my $numadesc = {
 569     optional => 1,
 570     type => 'string', format => $numa_fmt,
 571     description => "NUMA topology.",
 572 };
 573 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 574
 575 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 576     $confdesc->{"numa$i"} = $numadesc;
 577 }
 578
 579 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 580                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 581                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 582 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 583
 584 my $net_fmt_bridge_descr = <<__EOD__;
 585 Bridge to attach the network device to. The Proxmox VE standard bridge
 586 is called 'vmbr0'.
 587
 588 If you do not specify a bridge, we create a kvm user (NATed) network
 589 device, which provides DHCP and DNS services. The following addresses
 590 are used:
 591
 592  10.0.2.2   Gateway
 593  10.0.2.3   DNS Server
 594  10.0.2.4   SMB Server
 595
 596 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 597 __EOD__
 598
 599 my $net_fmt = {
 600     macaddr => {
 601         type => 'string',
 602         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 603         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 604         format_description => "XX:XX:XX:XX:XX:XX",
 605         optional => 1,
 606     },
 607     model => {
 608         type => 'string',
 609         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 610         enum => $nic_model_list,
 611         default_key => 1,
 612     },
 613     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 614     bridge => {
 615         type => 'string',
 616         description => $net_fmt_bridge_descr,
 617         format_description => 'bridge',
 618         optional => 1,
 619     },
 620     queues => {
 621         type => 'integer',
 622         minimum => 0, maximum => 16,
 623         description => 'Number of packet queues to be used on the device.',
 624         optional => 1,
 625     },
 626     rate => {
 627         type => 'number',
 628         minimum => 0,
 629         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 630         optional => 1,
 631     },
 632     tag => {
 633         type => 'integer',
 634         minimum => 1, maximum => 4094,
 635         description => 'VLAN tag to apply to packets on this interface.',
 636         optional => 1,
 637     },
 638     trunks => {
 639         type => 'string',
 640         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 641         description => 'VLAN trunks to pass through this interface.',
 642         format_description => 'vlanid[;vlanid...]',
 643         optional => 1,
 644     },
 645     firewall => {
 646         type => 'boolean',
 647         description => 'Whether this interface should be protected by the firewall.',
 648         optional => 1,
 649     },
 650     link_down => {
 651         type => 'boolean',
 652         description => 'Whether this interface should be disconnected (like pulling the plug).',
 653         optional => 1,
 654     },
 655 };
 656
 657 my $netdesc = {
 658     optional => 1,
 659     type => 'string', format => $net_fmt,
 660     description => "Specify network devices.",
 661 };
 662
 663 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 664
 665 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 666     $confdesc->{"net$i"} = $netdesc;
 667 }
 668
 669 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 670 sub verify_volume_id_or_qm_path {
 671     my ($volid, $noerr) = @_;
 672
 673     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 674         return $volid;
 675     }
 676
 677     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 678     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 679     if ($@) {
 680         return undef if $noerr;
 681         die $@;
 682     }
 683     return $volid;
 684 }
 685
 686 my $drivename_hash;
 687
 688 my %drivedesc_base = (
 689     volume => { alias => 'file' },
 690     file => {
 691         type => 'string',
 692         format => 'pve-volume-id-or-qm-path',
 693         default_key => 1,
 694         format_description => 'volume',
 695         description => "The drive's backing volume.",
 696     },
 697     media => {
 698         type => 'string',
 699         enum => [qw(cdrom disk)],
 700         description => "The drive's media type.",
 701         default => 'disk',
 702         optional => 1
 703     },
 704     cyls => {
 705         type => 'integer',
 706         description => "Force the drive's physical geometry to have a specific cylinder count.",
 707         optional => 1
 708     },
 709     heads => {
 710         type => 'integer',
 711         description => "Force the drive's physical geometry to have a specific head count.",
 712         optional => 1
 713     },
 714     secs => {
 715         type => 'integer',
 716         description => "Force the drive's physical geometry to have a specific sector count.",
 717         optional => 1
 718     },
 719     trans => {
 720         type => 'string',
 721         enum => [qw(none lba auto)],
 722         description => "Force disk geometry bios translation mode.",
 723         optional => 1,
 724     },
 725     snapshot => {
 726         type => 'boolean',
 727         description => "Whether the drive should be included when making snapshots.",
 728         optional => 1,
 729     },
 730     cache => {
 731         type => 'string',
 732         enum => [qw(none writethrough writeback unsafe directsync)],
 733         description => "The drive's cache mode",
 734         optional => 1,
 735     },
 736     format => get_standard_option('pve-qm-image-format'),
 737     size => {
 738         type => 'string',
 739         format => 'disk-size',
 740         format_description => 'DiskSize',
 741         description => "Disk size. This is purely informational and has no effect.",
 742         optional => 1,
 743     },
 744     backup => {
 745         type => 'boolean',
 746         description => "Whether the drive should be included when making backups.",
 747         optional => 1,
 748     },
 749     replicate => {
 750         type => 'boolean',
 751         description => 'Whether the drive should considered for replication jobs.',
 752         optional => 1,
 753         default => 1,
 754     },
 755     rerror => {
 756         type => 'string',
 757         enum => [qw(ignore report stop)],
 758         description => 'Read error action.',
 759         optional => 1,
 760     },
 761     werror => {
 762         type => 'string',
 763         enum => [qw(enospc ignore report stop)],
 764         description => 'Write error action.',
 765         optional => 1,
 766     },
 767     aio => {
 768         type => 'string',
 769         enum => [qw(native threads)],
 770         description => 'AIO type to use.',
 771         optional => 1,
 772     },
 773     discard => {
 774         type => 'string',
 775         enum => [qw(ignore on)],
 776         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 777         optional => 1,
 778     },
 779     detect_zeroes => {
 780         type => 'boolean',
 781         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 782         optional => 1,
 783     },
 784     serial => {
 785         type => 'string',
 786         format => 'urlencoded',
 787         format_description => 'serial',
 788         maxLength => 20*3, # *3 since it's %xx url enoded
 789         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 790         optional => 1,
 791     }
 792 );
 793
 794 my %iothread_fmt = ( iothread => {
 795         type => 'boolean',
 796         description => "Whether to use iothreads for this drive",
 797         optional => 1,
 798 });
 799
 800 my %model_fmt = (
 801     model => {
 802         type => 'string',
 803         format => 'urlencoded',
 804         format_description => 'model',
 805         maxLength => 40*3, # *3 since it's %xx url enoded
 806         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 807         optional => 1,
 808     },
 809 );
 810
 811 my %queues_fmt = (
 812     queues => {
 813         type => 'integer',
 814         description => "Number of queues.",
 815         minimum => 2,
 816         optional => 1
 817     }
 818 );
 819
 820 my %scsiblock_fmt = (
 821     scsiblock => {
 822         type => 'boolean',
 823         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 824         optional => 1,
 825         default => 0,
 826     },
 827 );
 828
 829 my $add_throttle_desc = sub {
 830     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 831     my $d = {
 832         type => $type,
 833         format_description => $unit,
 834         description => "Maximum $what in $longunit.",
 835         optional => 1,
 836     };
 837     $d->{minimum} = $minimum if defined($minimum);
 838     $drivedesc_base{$key} = $d;
 839 };
 840 # throughput: (leaky bucket)
 841 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 842 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 843 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 844 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 845 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 846 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 847 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 848 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 849 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 850
 851 # pools: (pool of IO before throttling starts taking effect)
 852 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 853 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 854 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 855 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 856 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 857 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 858
 859 # burst lengths
 860 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 861 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 862 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 863 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 864 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 865 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 866
 867 # legacy support
 868 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 869 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 870 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 871 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 872
 873 my $ide_fmt = {
 874     %drivedesc_base,
 875     %model_fmt,
 876 };
 877 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 878
 879 my $idedesc = {
 880     optional => 1,
 881     type => 'string', format => $ide_fmt,
 882     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 883 };
 884 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 885
 886 my $scsi_fmt = {
 887     %drivedesc_base,
 888     %iothread_fmt,
 889     %queues_fmt,
 890     %scsiblock_fmt,
 891 };
 892 my $scsidesc = {
 893     optional => 1,
 894     type => 'string', format => $scsi_fmt,
 895     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 896 };
 897 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 898
 899 my $sata_fmt = {
 900     %drivedesc_base,
 901 };
 902 my $satadesc = {
 903     optional => 1,
 904     type => 'string', format => $sata_fmt,
 905     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 906 };
 907 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 908
 909 my $virtio_fmt = {
 910     %drivedesc_base,
 911     %iothread_fmt,
 912 };
 913 my $virtiodesc = {
 914     optional => 1,
 915     type => 'string', format => $virtio_fmt,
 916     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 917 };
 918 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 919
 920 my $alldrive_fmt = {
 921     %drivedesc_base,
 922     %iothread_fmt,
 923     %model_fmt,
 924     %queues_fmt,
 925     %scsiblock_fmt,
 926 };
 927
 928 my $efidisk_fmt = {
 929     volume => { alias => 'file' },
 930     file => {
 931         type => 'string',
 932         format => 'pve-volume-id-or-qm-path',
 933         default_key => 1,
 934         format_description => 'volume',
 935         description => "The drive's backing volume.",
 936     },
 937     format => get_standard_option('pve-qm-image-format'),
 938     size => {
 939         type => 'string',
 940         format => 'disk-size',
 941         format_description => 'DiskSize',
 942         description => "Disk size. This is purely informational and has no effect.",
 943         optional => 1,
 944     },
 945 };
 946
 947 my $efidisk_desc = {
 948     optional => 1,
 949     type => 'string', format => $efidisk_fmt,
 950     description => "Configure a Disk for storing EFI vars",
 951 };
 952
 953 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 954
 955 my $usb_fmt = {
 956     host => {
 957         default_key => 1,
 958         type => 'string', format => 'pve-qm-usb-device',
 959         format_description => 'HOSTUSBDEVICE|spice',
 960         description => <<EODESCR,
 961 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 962
 963  'bus-port(.port)*' (decimal numbers) or
 964  'vendor_id:product_id' (hexadeciaml numbers) or
 965  'spice'
 966
 967 You can use the 'lsusb -t' command to list existing usb devices.
 968
 969 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 970
 971 The value 'spice' can be used to add a usb redirection devices for spice.
 972 EODESCR
 973     },
 974     usb3 => {
 975         optional => 1,
 976         type => 'boolean',
 977         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 978         default => 0,
 979     },
 980 };
 981
 982 my $usbdesc = {
 983     optional => 1,
 984     type => 'string', format => $usb_fmt,
 985     description => "Configure an USB device (n is 0 to 4).",
 986 };
 987 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 988
 989 # NOTE: the match-groups of this regex are used in parse_hostpci
 990 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 991 my $hostpci_fmt = {
 992     host => {
 993         default_key => 1,
 994         type => 'string',
 995         pattern => qr/$PCIRE(;$PCIRE)*/,
 996         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 997         description => <<EODESCR,
 998 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 999 of PCI virtual functions of the host. HOSTPCIID syntax is:
1000
1001 'bus:dev.func' (hexadecimal numbers)
1002
1003 You can us the 'lspci' command to list existing PCI devices.
1004 EODESCR
1005     },
1006     rombar => {
1007         type => 'boolean',
1008         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1009         optional => 1,
1010         default => 1,
1011     },
1012     romfile => {
1013         type => 'string',
1014         pattern => '[^,;]+',
1015         format_description => 'string',
1016         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1017         optional => 1,
1018     },
1019     pcie => {
1020         type => 'boolean',
1021         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1022         optional => 1,
1023         default => 0,
1024     },
1025     'x-vga' => {
1026         type => 'boolean',
1027         description =>  "Enable vfio-vga device support.",
1028         optional => 1,
1029         default => 0,
1030     },
1031 };
1032 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1033
1034 my $hostpcidesc = {
1035         optional => 1,
1036         type => 'string', format => 'pve-qm-hostpci',
1037         description => "Map host PCI devices into guest.",
1038         verbose_description =>  <<EODESCR,
1039 Map host PCI devices into guest.
1040
1041 NOTE: This option allows direct access to host hardware. So it is no longer
1042 possible to migrate such machines - use with special care.
1043
1044 CAUTION: Experimental! User reported problems with this option.
1045 EODESCR
1046 };
1047 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1048
1049 my $serialdesc = {
1050         optional => 1,
1051         type => 'string',
1052         pattern => '(/dev/.+|socket)',
1053         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1054         verbose_description =>  <<EODESCR,
1055 Create a serial device inside the VM (n is 0 to 3), and pass through a
1056 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1057 host side (use 'qm terminal' to open a terminal connection).
1058
1059 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1060
1061 CAUTION: Experimental! User reported problems with this option.
1062 EODESCR
1063 };
1064
1065 my $paralleldesc= {
1066         optional => 1,
1067         type => 'string',
1068         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1069         description =>  "Map host parallel devices (n is 0 to 2).",
1070         verbose_description =>  <<EODESCR,
1071 Map host parallel devices (n is 0 to 2).
1072
1073 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1074
1075 CAUTION: Experimental! User reported problems with this option.
1076 EODESCR
1077 };
1078
1079 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1080     $confdesc->{"parallel$i"} = $paralleldesc;
1081 }
1082
1083 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1084     $confdesc->{"serial$i"} = $serialdesc;
1085 }
1086
1087 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1088     $confdesc->{"hostpci$i"} = $hostpcidesc;
1089 }
1090
1091 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1092     $drivename_hash->{"ide$i"} = 1;
1093     $confdesc->{"ide$i"} = $idedesc;
1094 }
1095
1096 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1097     $drivename_hash->{"sata$i"} = 1;
1098     $confdesc->{"sata$i"} = $satadesc;
1099 }
1100
1101 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1102     $drivename_hash->{"scsi$i"} = 1;
1103     $confdesc->{"scsi$i"} = $scsidesc ;
1104 }
1105
1106 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1107     $drivename_hash->{"virtio$i"} = 1;
1108     $confdesc->{"virtio$i"} = $virtiodesc;
1109 }
1110
1111 $drivename_hash->{efidisk0} = 1;
1112 $confdesc->{efidisk0} = $efidisk_desc;
1113
1114 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1115     $confdesc->{"usb$i"} = $usbdesc;
1116 }
1117
1118 my $unuseddesc = {
1119     optional => 1,
1120     type => 'string', format => 'pve-volume-id',
1121     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1122 };
1123
1124 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1125     $confdesc->{"unused$i"} = $unuseddesc;
1126 }
1127
1128 my $kvm_api_version = 0;
1129
1130 sub kvm_version {
1131
1132     return $kvm_api_version if $kvm_api_version;
1133
1134     my $fh = IO::File->new("</dev/kvm") ||
1135         return 0;
1136
1137     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1138         $kvm_api_version = $v;
1139     }
1140
1141     $fh->close();
1142
1143     return  $kvm_api_version;
1144 }
1145
1146 my $kvm_user_version;
1147
1148 sub kvm_user_version {
1149
1150     return $kvm_user_version if $kvm_user_version;
1151
1152     $kvm_user_version = 'unknown';
1153
1154     my $code = sub {
1155         my $line = shift;
1156         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1157             $kvm_user_version = $2;
1158         }
1159     };
1160
1161     eval { run_command("kvm -version", outfunc => $code); };
1162     warn $@ if $@;
1163
1164     return $kvm_user_version;
1165
1166 }
1167
1168 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1169
1170 sub valid_drive_names {
1171     # order is important - used to autoselect boot disk
1172     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1173             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1174             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1175             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1176             'efidisk0');
1177 }
1178
1179 sub is_valid_drivename {
1180     my $dev = shift;
1181
1182     return defined($drivename_hash->{$dev});
1183 }
1184
1185 sub option_exists {
1186     my $key = shift;
1187     return defined($confdesc->{$key});
1188 }
1189
1190 sub nic_models {
1191     return $nic_model_list;
1192 }
1193
1194 sub os_list_description {
1195
1196     return {
1197         other => 'Other',
1198         wxp => 'Windows XP',
1199         w2k => 'Windows 2000',
1200         w2k3 =>, 'Windows 2003',
1201         w2k8 => 'Windows 2008',
1202         wvista => 'Windows Vista',
1203         win7 => 'Windows 7',
1204         win8 => 'Windows 8/2012',
1205         win10 => 'Windows 10/2016',
1206         l24 => 'Linux 2.4',
1207         l26 => 'Linux 2.6',
1208     };
1209 }
1210
1211 my $cdrom_path;
1212
1213 sub get_cdrom_path {
1214
1215     return  $cdrom_path if $cdrom_path;
1216
1217     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1218     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1219     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1220 }
1221
1222 sub get_iso_path {
1223     my ($storecfg, $vmid, $cdrom) = @_;
1224
1225     if ($cdrom eq 'cdrom') {
1226         return get_cdrom_path();
1227     } elsif ($cdrom eq 'none') {
1228         return '';
1229     } elsif ($cdrom =~ m|^/|) {
1230         return $cdrom;
1231     } else {
1232         return PVE::Storage::path($storecfg, $cdrom);
1233     }
1234 }
1235
1236 # try to convert old style file names to volume IDs
1237 sub filename_to_volume_id {
1238     my ($vmid, $file, $media) = @_;
1239
1240     if (!($file eq 'none' || $file eq 'cdrom' ||
1241           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1242
1243         return undef if $file =~ m|/|;
1244
1245         if ($media && $media eq 'cdrom') {
1246             $file = "local:iso/$file";
1247         } else {
1248             $file = "local:$vmid/$file";
1249         }
1250     }
1251
1252     return $file;
1253 }
1254
1255 sub verify_media_type {
1256     my ($opt, $vtype, $media) = @_;
1257
1258     return if !$media;
1259
1260     my $etype;
1261     if ($media eq 'disk') {
1262         $etype = 'images';
1263     } elsif ($media eq 'cdrom') {
1264         $etype = 'iso';
1265     } else {
1266         die "internal error";
1267     }
1268
1269     return if ($vtype eq $etype);
1270
1271     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1272 }
1273
1274 sub cleanup_drive_path {
1275     my ($opt, $storecfg, $drive) = @_;
1276
1277     # try to convert filesystem paths to volume IDs
1278
1279     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1280         ($drive->{file} !~ m|^/dev/.+|) &&
1281         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1282         ($drive->{file} !~ m/^\d+$/)) {
1283         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1284         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1285         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1286         verify_media_type($opt, $vtype, $drive->{media});
1287         $drive->{file} = $volid;
1288     }
1289
1290     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1291 }
1292
1293 sub parse_hotplug_features {
1294     my ($data) = @_;
1295
1296     my $res = {};
1297
1298     return $res if $data eq '0';
1299
1300     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1301
1302     foreach my $feature (PVE::Tools::split_list($data)) {
1303         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1304             $res->{$1} = 1;
1305         } else {
1306             die "invalid hotplug feature '$feature'\n";
1307         }
1308     }
1309     return $res;
1310 }
1311
1312 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1313 sub pve_verify_hotplug_features {
1314     my ($value, $noerr) = @_;
1315
1316     return $value if parse_hotplug_features($value);
1317
1318     return undef if $noerr;
1319
1320     die "unable to parse hotplug option\n";
1321 }
1322
1323 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1324 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1325 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1326 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1327 #        [,iothread=on][,serial=serial][,model=model]
1328
1329 sub parse_drive {
1330     my ($key, $data) = @_;
1331
1332     my ($interface, $index);
1333
1334     if ($key =~ m/^([^\d]+)(\d+)$/) {
1335         $interface = $1;
1336         $index = $2;
1337     } else {
1338         return undef;
1339     }
1340
1341     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1342                                      : $confdesc->{$key}->{format};
1343     if (!$desc) {
1344         warn "invalid drive key: $key\n";
1345         return undef;
1346     }
1347     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1348     return undef if !$res;
1349     $res->{interface} = $interface;
1350     $res->{index} = $index;
1351
1352     my $error = 0;
1353     foreach my $opt (qw(bps bps_rd bps_wr)) {
1354         if (my $bps = defined(delete $res->{$opt})) {
1355             if (defined($res->{"m$opt"})) {
1356                 warn "both $opt and m$opt specified\n";
1357                 ++$error;
1358                 next;
1359             }
1360             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1361         }
1362     }
1363
1364     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1365     for my $requirement (
1366         [mbps_max => 'mbps'],
1367         [mbps_rd_max => 'mbps_rd'],
1368         [mbps_wr_max => 'mbps_wr'],
1369         [miops_max => 'miops'],
1370         [miops_rd_max => 'miops_rd'],
1371         [miops_wr_max => 'miops_wr'],
1372         [bps_max_length => 'mbps_max'],
1373         [bps_rd_max_length => 'mbps_rd_max'],
1374         [bps_wr_max_length => 'mbps_wr_max'],
1375         [iops_max_length => 'iops_max'],
1376         [iops_rd_max_length => 'iops_rd_max'],
1377         [iops_wr_max_length => 'iops_wr_max']) {
1378         my ($option, $requires) = @$requirement;
1379         if ($res->{$option} && !$res->{$requires}) {
1380             warn "$option requires $requires\n";
1381             ++$error;
1382         }
1383     }
1384
1385     return undef if $error;
1386
1387     return undef if $res->{mbps_rd} && $res->{mbps};
1388     return undef if $res->{mbps_wr} && $res->{mbps};
1389     return undef if $res->{iops_rd} && $res->{iops};
1390     return undef if $res->{iops_wr} && $res->{iops};
1391
1392     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1393         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1394         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1395         return undef if $res->{interface} eq 'virtio';
1396     }
1397
1398     if (my $size = $res->{size}) {
1399         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1400     }
1401
1402     return $res;
1403 }
1404
1405 sub print_drive {
1406     my ($vmid, $drive) = @_;
1407     my $data = { %$drive };
1408     delete $data->{$_} for qw(index interface);
1409     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1410 }
1411
1412 sub scsi_inquiry {
1413     my($fh, $noerr) = @_;
1414
1415     my $SG_IO = 0x2285;
1416     my $SG_GET_VERSION_NUM = 0x2282;
1417
1418     my $versionbuf = "\x00" x 8;
1419     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1420     if (!$ret) {
1421         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1422         return undef;
1423     }
1424     my $version = unpack("I", $versionbuf);
1425     if ($version < 30000) {
1426         die "scsi generic interface too old\n"  if !$noerr;
1427         return undef;
1428     }
1429
1430     my $buf = "\x00" x 36;
1431     my $sensebuf = "\x00" x 8;
1432     my $cmd = pack("C x3 C x1", 0x12, 36);
1433
1434     # see /usr/include/scsi/sg.h
1435     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1436
1437     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1438                       length($sensebuf), 0, length($buf), $buf,
1439                       $cmd, $sensebuf, 6000);
1440
1441     $ret = ioctl($fh, $SG_IO, $packet);
1442     if (!$ret) {
1443         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1444         return undef;
1445     }
1446
1447     my @res = unpack($sg_io_hdr_t, $packet);
1448     if ($res[17] || $res[18]) {
1449         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1450         return undef;
1451     }
1452
1453     my $res = {};
1454     (my $byte0, my $byte1, $res->{vendor},
1455      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1456
1457     $res->{removable} = $byte1 & 128 ? 1 : 0;
1458     $res->{type} = $byte0 & 31;
1459
1460     return $res;
1461 }
1462
1463 sub path_is_scsi {
1464     my ($path) = @_;
1465
1466     my $fh = IO::File->new("+<$path") || return undef;
1467     my $res = scsi_inquiry($fh, 1);
1468     close($fh);
1469
1470     return $res;
1471 }
1472
1473 sub machine_type_is_q35 {
1474     my ($conf) = @_;
1475
1476     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1477 }
1478
1479 sub print_tabletdevice_full {
1480     my ($conf) = @_;
1481
1482     my $q35 = machine_type_is_q35($conf);
1483
1484     # we use uhci for old VMs because tablet driver was buggy in older qemu
1485     my $usbbus = $q35 ? "ehci" : "uhci";
1486
1487     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1488 }
1489
1490 sub print_drivedevice_full {
1491     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1492
1493     my $device = '';
1494     my $maxdev = 0;
1495
1496     if ($drive->{interface} eq 'virtio') {
1497         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1498         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1499         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1500     } elsif ($drive->{interface} eq 'scsi') {
1501
1502         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1503         my $unit = $drive->{index} % $maxdev;
1504         my $devicetype = 'hd';
1505         my $path = '';
1506         if (drive_is_cdrom($drive)) {
1507             $devicetype = 'cd';
1508         } else {
1509             if ($drive->{file} =~ m|^/|) {
1510                 $path = $drive->{file};
1511                 if (my $info = path_is_scsi($path)) {
1512                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1513                         $devicetype = 'block';
1514                     } elsif ($info->{type} == 1) { # tape
1515                         $devicetype = 'generic';
1516                     }
1517                 }
1518             } else {
1519                  $path = PVE::Storage::path($storecfg, $drive->{file});
1520             }
1521
1522             if($path =~ m/^iscsi\:\/\//){
1523                 $devicetype = 'generic';
1524             }
1525         }
1526
1527         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1528             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1529         } else {
1530             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1531         }
1532
1533     } elsif ($drive->{interface} eq 'ide'){
1534         $maxdev = 2;
1535         my $controller = int($drive->{index} / $maxdev);
1536         my $unit = $drive->{index} % $maxdev;
1537         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1538
1539         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1540         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1541             $model = URI::Escape::uri_unescape($model);
1542             $device .= ",model=$model";
1543         }
1544     } elsif ($drive->{interface} eq 'sata'){
1545         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1546         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1547         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1548     } elsif ($drive->{interface} eq 'usb') {
1549         die "implement me";
1550         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1551     } else {
1552         die "unsupported interface type";
1553     }
1554
1555     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1556
1557     return $device;
1558 }
1559
1560 sub get_initiator_name {
1561     my $initiator;
1562
1563     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1564     while (defined(my $line = <$fh>)) {
1565         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1566         $initiator = $1;
1567         last;
1568     }
1569     $fh->close();
1570
1571     return $initiator;
1572 }
1573
1574 sub print_drive_full {
1575     my ($storecfg, $vmid, $drive) = @_;
1576
1577     my $path;
1578     my $volid = $drive->{file};
1579     my $format;
1580
1581     if (drive_is_cdrom($drive)) {
1582         $path = get_iso_path($storecfg, $vmid, $volid);
1583     } else {
1584         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1585         if ($storeid) {
1586             $path = PVE::Storage::path($storecfg, $volid);
1587             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1588             $format = qemu_img_format($scfg, $volname);
1589         } else {
1590             $path = $volid;
1591             $format = "raw";
1592         }
1593    }
1594
1595     my $opts = '';
1596     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1597     foreach my $o (@qemu_drive_options) {
1598         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1599     }
1600     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1601         my ($dir, $qmpname) = @$type;
1602         if (my $v = $drive->{"mbps$dir"}) {
1603             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1604         }
1605         if (my $v = $drive->{"mbps${dir}_max"}) {
1606             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1607         }
1608         if (my $v = $drive->{"bps${dir}_max_length"}) {
1609             $opts .= ",throttling.bps$qmpname-max-length=$v";
1610         }
1611         if (my $v = $drive->{"iops${dir}"}) {
1612             $opts .= ",throttling.iops$qmpname=$v";
1613         }
1614         if (my $v = $drive->{"iops${dir}_max"}) {
1615             $opts .= ",throttling.iops$qmpname-max=$v";
1616         }
1617         if (my $v = $drive->{"iops${dir}_max_length"}) {
1618             $opts .= ",throttling.iops$qmpname-max-length=$v";
1619         }
1620     }
1621
1622     if (my $serial = $drive->{serial}) {
1623         $serial = URI::Escape::uri_unescape($serial);
1624         $opts .= ",serial=$serial";
1625     }
1626
1627     $opts .= ",format=$format" if $format && !$drive->{format};
1628
1629     my $cache_direct = 0;
1630
1631     if (my $cache = $drive->{cache}) {
1632         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1633     } elsif (!drive_is_cdrom($drive)) {
1634         $opts .= ",cache=none";
1635         $cache_direct = 1;
1636     }
1637
1638     # aio native works only with O_DIRECT
1639     if (!$drive->{aio}) {
1640         if($cache_direct) {
1641             $opts .= ",aio=native";
1642         } else {
1643             $opts .= ",aio=threads";
1644         }
1645     }
1646
1647     if (!drive_is_cdrom($drive)) {
1648         my $detectzeroes;
1649         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1650             $detectzeroes = 'off';
1651         } elsif ($drive->{discard}) {
1652             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1653         } else {
1654             # This used to be our default with discard not being specified:
1655             $detectzeroes = 'on';
1656         }
1657         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1658     }
1659
1660     my $pathinfo = $path ? "file=$path," : '';
1661
1662     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1663 }
1664
1665 sub print_netdevice_full {
1666     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1667
1668     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1669
1670     my $device = $net->{model};
1671     if ($net->{model} eq 'virtio') {
1672          $device = 'virtio-net-pci';
1673      };
1674
1675     my $pciaddr = print_pci_addr("$netid", $bridges);
1676     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1677     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1678         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1679         my $vectors = $net->{queues} * 2 + 2;
1680         $tmpstr .= ",vectors=$vectors,mq=on";
1681     }
1682     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1683
1684     if ($use_old_bios_files) {
1685         my $romfile;
1686         if ($device eq 'virtio-net-pci') {
1687             $romfile = 'pxe-virtio.rom';
1688         } elsif ($device eq 'e1000') {
1689             $romfile = 'pxe-e1000.rom';
1690         } elsif ($device eq 'ne2k') {
1691             $romfile = 'pxe-ne2k_pci.rom';
1692         } elsif ($device eq 'pcnet') {
1693             $romfile = 'pxe-pcnet.rom';
1694         } elsif ($device eq 'rtl8139') {
1695             $romfile = 'pxe-rtl8139.rom';
1696         }
1697         $tmpstr .= ",romfile=$romfile" if $romfile;
1698     }
1699
1700     return $tmpstr;
1701 }
1702
1703 sub print_netdev_full {
1704     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1705
1706     my $i = '';
1707     if ($netid =~ m/^net(\d+)$/) {
1708         $i = int($1);
1709     }
1710
1711     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1712
1713     my $ifname = "tap${vmid}i$i";
1714
1715     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1716     die "interface name '$ifname' is too long (max 15 character)\n"
1717         if length($ifname) >= 16;
1718
1719     my $vhostparam = '';
1720     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1721
1722     my $vmname = $conf->{name} || "vm$vmid";
1723
1724     my $netdev = "";
1725     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1726
1727     if ($net->{bridge}) {
1728         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1729     } else {
1730         $netdev = "type=user,id=$netid,hostname=$vmname";
1731     }
1732
1733     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1734
1735     return $netdev;
1736 }
1737
1738
1739 sub print_cpu_device {
1740     my ($conf, $id) = @_;
1741
1742     my $kvm = $conf->{kvm} // 1;
1743     my $cpu = $kvm ? "kvm64" : "qemu64";
1744     if (my $cputype = $conf->{cpu}) {
1745         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1746             or die "Cannot parse cpu description: $cputype\n";
1747         $cpu = $cpuconf->{cputype};
1748     }
1749
1750     my $cores = $conf->{cores} || 1;
1751
1752     my $current_core = ($id - 1) % $cores;
1753     my $current_socket = int(($id - 1 - $current_core)/$cores);
1754
1755     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1756 }
1757
1758 sub drive_is_cdrom {
1759     my ($drive) = @_;
1760
1761     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1762
1763 }
1764
1765 sub parse_number_sets {
1766     my ($set) = @_;
1767     my $res = [];
1768     foreach my $part (split(/;/, $set)) {
1769         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1770             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1771             push @$res, [ $1, $2 ];
1772         } else {
1773             die "invalid range: $part\n";
1774         }
1775     }
1776     return $res;
1777 }
1778
1779 sub parse_numa {
1780     my ($data) = @_;
1781
1782     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1783     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1784     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1785     return $res;
1786 }
1787
1788 sub parse_hostpci {
1789     my ($value) = @_;
1790
1791     return undef if !$value;
1792
1793     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1794
1795     my @idlist = split(/;/, $res->{host});
1796     delete $res->{host};
1797     foreach my $id (@idlist) {
1798         if ($id =~ /^$PCIRE$/) {
1799             if (defined($2)) {
1800                 push @{$res->{pciid}}, { id => $1, function => $2 };
1801             } else {
1802                 my $pcidevices = lspci($1);
1803                 $res->{pciid} = $pcidevices->{$1};
1804             }
1805         } else {
1806             # should have been caught by parse_property_string already
1807             die "failed to parse PCI id: $id\n";
1808         }
1809     }
1810     return $res;
1811 }
1812
1813 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1814 sub parse_net {
1815     my ($data) = @_;
1816
1817     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1818     if ($@) {
1819         warn $@;
1820         return undef;
1821     }
1822     if (!defined($res->{macaddr})) {
1823         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1824         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1825     }
1826     return $res;
1827 }
1828
1829 sub print_net {
1830     my $net = shift;
1831
1832     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1833 }
1834
1835 sub add_random_macs {
1836     my ($settings) = @_;
1837
1838     foreach my $opt (keys %$settings) {
1839         next if $opt !~ m/^net(\d+)$/;
1840         my $net = parse_net($settings->{$opt});
1841         next if !$net;
1842         $settings->{$opt} = print_net($net);
1843     }
1844 }
1845
1846 sub vm_is_volid_owner {
1847     my ($storecfg, $vmid, $volid) = @_;
1848
1849     if ($volid !~  m|^/|) {
1850         my ($path, $owner);
1851         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1852         if ($owner && ($owner == $vmid)) {
1853             return 1;
1854         }
1855     }
1856
1857     return undef;
1858 }
1859
1860 sub split_flagged_list {
1861     my $text = shift || '';
1862     $text =~ s/[,;]/ /g;
1863     $text =~ s/^\s+//;
1864     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1865 }
1866
1867 sub join_flagged_list {
1868     my ($how, $lst) = @_;
1869     join $how, map { $lst->{$_} . $_ } keys %$lst;
1870 }
1871
1872 sub vmconfig_delete_pending_option {
1873     my ($conf, $key, $force) = @_;
1874
1875     delete $conf->{pending}->{$key};
1876     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1877     $pending_delete_hash->{$key} = $force ? '!' : '';
1878     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1879 }
1880
1881 sub vmconfig_undelete_pending_option {
1882     my ($conf, $key) = @_;
1883
1884     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1885     delete $pending_delete_hash->{$key};
1886
1887     if (%$pending_delete_hash) {
1888         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1889     } else {
1890         delete $conf->{pending}->{delete};
1891     }
1892 }
1893
1894 sub vmconfig_register_unused_drive {
1895     my ($storecfg, $vmid, $conf, $drive) = @_;
1896
1897     if (!drive_is_cdrom($drive)) {
1898         my $volid = $drive->{file};
1899         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1900             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1901         }
1902     }
1903 }
1904
1905 sub vmconfig_cleanup_pending {
1906     my ($conf) = @_;
1907
1908     # remove pending changes when nothing changed
1909     my $changes;
1910     foreach my $opt (keys %{$conf->{pending}}) {
1911         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1912             $changes = 1;
1913             delete $conf->{pending}->{$opt};
1914         }
1915     }
1916
1917     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1918     my $pending_delete_hash = {};
1919     while (my ($opt, $force) = each %$current_delete_hash) {
1920         if (defined($conf->{$opt})) {
1921             $pending_delete_hash->{$opt} = $force;
1922         } else {
1923             $changes = 1;
1924         }
1925     }
1926
1927     if (%$pending_delete_hash) {
1928         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1929     } else {
1930         delete $conf->{pending}->{delete};
1931     }
1932
1933     return $changes;
1934 }
1935
1936 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1937 my $smbios1_fmt = {
1938     uuid => {
1939         type => 'string',
1940         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1941         format_description => 'UUID',
1942         description => "Set SMBIOS1 UUID.",
1943         optional => 1,
1944     },
1945     version => {
1946         type => 'string',
1947         pattern => '\S+',
1948         format_description => 'string',
1949         description => "Set SMBIOS1 version.",
1950         optional => 1,
1951     },
1952     serial => {
1953         type => 'string',
1954         pattern => '\S+',
1955         format_description => 'string',
1956         description => "Set SMBIOS1 serial number.",
1957         optional => 1,
1958     },
1959     manufacturer => {
1960         type => 'string',
1961         pattern => '\S+',
1962         format_description => 'string',
1963         description => "Set SMBIOS1 manufacturer.",
1964         optional => 1,
1965     },
1966     product => {
1967         type => 'string',
1968         pattern => '\S+',
1969         format_description => 'string',
1970         description => "Set SMBIOS1 product ID.",
1971         optional => 1,
1972     },
1973     sku => {
1974         type => 'string',
1975         pattern => '\S+',
1976         format_description => 'string',
1977         description => "Set SMBIOS1 SKU string.",
1978         optional => 1,
1979     },
1980     family => {
1981         type => 'string',
1982         pattern => '\S+',
1983         format_description => 'string',
1984         description => "Set SMBIOS1 family string.",
1985         optional => 1,
1986     },
1987 };
1988
1989 sub parse_smbios1 {
1990     my ($data) = @_;
1991
1992     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1993     warn $@ if $@;
1994     return $res;
1995 }
1996
1997 sub print_smbios1 {
1998     my ($smbios1) = @_;
1999     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2000 }
2001
2002 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2003
2004 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2005 sub verify_bootdisk {
2006     my ($value, $noerr) = @_;
2007
2008     return $value if is_valid_drivename($value);
2009
2010     return undef if $noerr;
2011
2012     die "invalid boot disk '$value'\n";
2013 }
2014
2015 sub parse_watchdog {
2016     my ($value) = @_;
2017
2018     return undef if !$value;
2019
2020     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2021     warn $@ if $@;
2022     return $res;
2023 }
2024
2025 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2026 sub verify_usb_device {
2027     my ($value, $noerr) = @_;
2028
2029     return $value if parse_usb_device($value);
2030
2031     return undef if $noerr;
2032
2033     die "unable to parse usb device\n";
2034 }
2035
2036 # add JSON properties for create and set function
2037 sub json_config_properties {
2038     my $prop = shift;
2039
2040     foreach my $opt (keys %$confdesc) {
2041         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2042         $prop->{$opt} = $confdesc->{$opt};
2043     }
2044
2045     return $prop;
2046 }
2047
2048 sub check_type {
2049     my ($key, $value) = @_;
2050
2051     die "unknown setting '$key'\n" if !$confdesc->{$key};
2052
2053     my $type = $confdesc->{$key}->{type};
2054
2055     if (!defined($value)) {
2056         die "got undefined value\n";
2057     }
2058
2059     if ($value =~ m/[\n\r]/) {
2060         die "property contains a line feed\n";
2061     }
2062
2063     if ($type eq 'boolean') {
2064         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2065         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2066         die "type check ('boolean') failed - got '$value'\n";
2067     } elsif ($type eq 'integer') {
2068         return int($1) if $value =~ m/^(\d+)$/;
2069         die "type check ('integer') failed - got '$value'\n";
2070     } elsif ($type eq 'number') {
2071         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2072         die "type check ('number') failed - got '$value'\n";
2073     } elsif ($type eq 'string') {
2074         if (my $fmt = $confdesc->{$key}->{format}) {
2075             PVE::JSONSchema::check_format($fmt, $value);
2076             return $value;
2077         }
2078         $value =~ s/^\"(.*)\"$/$1/;
2079         return $value;
2080     } else {
2081         die "internal error"
2082     }
2083 }
2084
2085 sub check_iommu_support{
2086     #fixme : need to check IOMMU support
2087     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2088
2089     my $iommu=1;
2090     return $iommu;
2091
2092 }
2093
2094 sub touch_config {
2095     my ($vmid) = @_;
2096
2097     my $conf = PVE::QemuConfig->config_file($vmid);
2098     utime undef, undef, $conf;
2099 }
2100
2101 sub destroy_vm {
2102     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2103
2104     my $conffile = PVE::QemuConfig->config_file($vmid);
2105
2106     my $conf = PVE::QemuConfig->load_config($vmid);
2107
2108     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2109
2110     # only remove disks owned by this VM
2111     foreach_drive($conf, sub {
2112         my ($ds, $drive) = @_;
2113
2114         return if drive_is_cdrom($drive);
2115
2116         my $volid = $drive->{file};
2117
2118         return if !$volid || $volid =~ m|^/|;
2119
2120         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2121         return if !$path || !$owner || ($owner != $vmid);
2122
2123         eval {
2124             PVE::Storage::vdisk_free($storecfg, $volid);
2125         };
2126         warn "Could not remove disk '$volid', check manually: $@" if $@;
2127
2128     });
2129
2130     if ($keep_empty_config) {
2131         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2132     } else {
2133         unlink $conffile;
2134     }
2135
2136     # also remove unused disk
2137     eval {
2138         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2139
2140         eval {
2141             PVE::Storage::foreach_volid($dl, sub {
2142                 my ($volid, $sid, $volname, $d) = @_;
2143                 PVE::Storage::vdisk_free($storecfg, $volid);
2144             });
2145         };
2146         warn $@ if $@;
2147
2148     };
2149     warn $@ if $@;
2150 }
2151
2152 sub parse_vm_config {
2153     my ($filename, $raw) = @_;
2154
2155     return undef if !defined($raw);
2156
2157     my $res = {
2158         digest => Digest::SHA::sha1_hex($raw),
2159         snapshots => {},
2160         pending => {},
2161     };
2162
2163     $filename =~ m|/qemu-server/(\d+)\.conf$|
2164         || die "got strange filename '$filename'";
2165
2166     my $vmid = $1;
2167
2168     my $conf = $res;
2169     my $descr;
2170     my $section = '';
2171
2172     my @lines = split(/\n/, $raw);
2173     foreach my $line (@lines) {
2174         next if $line =~ m/^\s*$/;
2175
2176         if ($line =~ m/^\[PENDING\]\s*$/i) {
2177             $section = 'pending';
2178             if (defined($descr)) {
2179                 $descr =~ s/\s+$//;
2180                 $conf->{description} = $descr;
2181             }
2182             $descr = undef;
2183             $conf = $res->{$section} = {};
2184             next;
2185
2186         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2187             $section = $1;
2188             if (defined($descr)) {
2189                 $descr =~ s/\s+$//;
2190                 $conf->{description} = $descr;
2191             }
2192             $descr = undef;
2193             $conf = $res->{snapshots}->{$section} = {};
2194             next;
2195         }
2196
2197         if ($line =~ m/^\#(.*)\s*$/) {
2198             $descr = '' if !defined($descr);
2199             $descr .= PVE::Tools::decode_text($1) . "\n";
2200             next;
2201         }
2202
2203         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2204             $descr = '' if !defined($descr);
2205             $descr .= PVE::Tools::decode_text($2);
2206         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2207             $conf->{snapstate} = $1;
2208         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2209             my $key = $1;
2210             my $value = $2;
2211             $conf->{$key} = $value;
2212         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2213             my $value = $1;
2214             if ($section eq 'pending') {
2215                 $conf->{delete} = $value; # we parse this later
2216             } else {
2217                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2218             }
2219         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2220             my $key = $1;
2221             my $value = $2;
2222             eval { $value = check_type($key, $value); };
2223             if ($@) {
2224                 warn "vm $vmid - unable to parse value of '$key' - $@";
2225             } else {
2226                 $key = 'ide2' if $key eq 'cdrom';
2227                 my $fmt = $confdesc->{$key}->{format};
2228                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2229                     my $v = parse_drive($key, $value);
2230                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2231                         $v->{file} = $volid;
2232                         $value = print_drive($vmid, $v);
2233                     } else {
2234                         warn "vm $vmid - unable to parse value of '$key'\n";
2235                         next;
2236                     }
2237                 }
2238
2239                 $conf->{$key} = $value;
2240             }
2241         }
2242     }
2243
2244     if (defined($descr)) {
2245         $descr =~ s/\s+$//;
2246         $conf->{description} = $descr;
2247     }
2248     delete $res->{snapstate}; # just to be sure
2249
2250     return $res;
2251 }
2252
2253 sub write_vm_config {
2254     my ($filename, $conf) = @_;
2255
2256     delete $conf->{snapstate}; # just to be sure
2257
2258     if ($conf->{cdrom}) {
2259         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2260         $conf->{ide2} = $conf->{cdrom};
2261         delete $conf->{cdrom};
2262     }
2263
2264     # we do not use 'smp' any longer
2265     if ($conf->{sockets}) {
2266         delete $conf->{smp};
2267     } elsif ($conf->{smp}) {
2268         $conf->{sockets} = $conf->{smp};
2269         delete $conf->{cores};
2270         delete $conf->{smp};
2271     }
2272
2273     my $used_volids = {};
2274
2275     my $cleanup_config = sub {
2276         my ($cref, $pending, $snapname) = @_;
2277
2278         foreach my $key (keys %$cref) {
2279             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2280                 $key eq 'snapstate' || $key eq 'pending';
2281             my $value = $cref->{$key};
2282             if ($key eq 'delete') {
2283                 die "propertry 'delete' is only allowed in [PENDING]\n"
2284                     if !$pending;
2285                 # fixme: check syntax?
2286                 next;
2287             }
2288             eval { $value = check_type($key, $value); };
2289             die "unable to parse value of '$key' - $@" if $@;
2290
2291             $cref->{$key} = $value;
2292
2293             if (!$snapname && is_valid_drivename($key)) {
2294                 my $drive = parse_drive($key, $value);
2295                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2296             }
2297         }
2298     };
2299
2300     &$cleanup_config($conf);
2301
2302     &$cleanup_config($conf->{pending}, 1);
2303
2304     foreach my $snapname (keys %{$conf->{snapshots}}) {
2305         die "internal error" if $snapname eq 'pending';
2306         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2307     }
2308
2309     # remove 'unusedX' settings if we re-add a volume
2310     foreach my $key (keys %$conf) {
2311         my $value = $conf->{$key};
2312         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2313             delete $conf->{$key};
2314         }
2315     }
2316
2317     my $generate_raw_config = sub {
2318         my ($conf, $pending) = @_;
2319
2320         my $raw = '';
2321
2322         # add description as comment to top of file
2323         if (defined(my $descr = $conf->{description})) {
2324             if ($descr) {
2325                 foreach my $cl (split(/\n/, $descr)) {
2326                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2327                 }
2328             } else {
2329                 $raw .= "#\n" if $pending;
2330             }
2331         }
2332
2333         foreach my $key (sort keys %$conf) {
2334             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2335             $raw .= "$key: $conf->{$key}\n";
2336         }
2337         return $raw;
2338     };
2339
2340     my $raw = &$generate_raw_config($conf);
2341
2342     if (scalar(keys %{$conf->{pending}})){
2343         $raw .= "\n[PENDING]\n";
2344         $raw .= &$generate_raw_config($conf->{pending}, 1);
2345     }
2346
2347     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2348         $raw .= "\n[$snapname]\n";
2349         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2350     }
2351
2352     return $raw;
2353 }
2354
2355 sub load_defaults {
2356
2357     my $res = {};
2358
2359     # we use static defaults from our JSON schema configuration
2360     foreach my $key (keys %$confdesc) {
2361         if (defined(my $default = $confdesc->{$key}->{default})) {
2362             $res->{$key} = $default;
2363         }
2364     }
2365
2366     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2367     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2368
2369     return $res;
2370 }
2371
2372 sub config_list {
2373     my $vmlist = PVE::Cluster::get_vmlist();
2374     my $res = {};
2375     return $res if !$vmlist || !$vmlist->{ids};
2376     my $ids = $vmlist->{ids};
2377
2378     foreach my $vmid (keys %$ids) {
2379         my $d = $ids->{$vmid};
2380         next if !$d->{node} || $d->{node} ne $nodename;
2381         next if !$d->{type} || $d->{type} ne 'qemu';
2382         $res->{$vmid}->{exists} = 1;
2383     }
2384     return $res;
2385 }
2386
2387 # test if VM uses local resources (to prevent migration)
2388 sub check_local_resources {
2389     my ($conf, $noerr) = @_;
2390
2391     my $loc_res = 0;
2392
2393     $loc_res = 1 if $conf->{hostusb}; # old syntax
2394     $loc_res = 1 if $conf->{hostpci}; # old syntax
2395
2396     foreach my $k (keys %$conf) {
2397         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2398         # sockets are safe: they will recreated be on the target side post-migrate
2399         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2400         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2401     }
2402
2403     die "VM uses local resources\n" if $loc_res && !$noerr;
2404
2405     return $loc_res;
2406 }
2407
2408 # check if used storages are available on all nodes (use by migrate)
2409 sub check_storage_availability {
2410     my ($storecfg, $conf, $node) = @_;
2411
2412     foreach_drive($conf, sub {
2413         my ($ds, $drive) = @_;
2414
2415         my $volid = $drive->{file};
2416         return if !$volid;
2417
2418         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2419         return if !$sid;
2420
2421         # check if storage is available on both nodes
2422         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2423         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2424    });
2425 }
2426
2427 # list nodes where all VM images are available (used by has_feature API)
2428 sub shared_nodes {
2429     my ($conf, $storecfg) = @_;
2430
2431     my $nodelist = PVE::Cluster::get_nodelist();
2432     my $nodehash = { map { $_ => 1 } @$nodelist };
2433     my $nodename = PVE::INotify::nodename();
2434
2435     foreach_drive($conf, sub {
2436         my ($ds, $drive) = @_;
2437
2438         my $volid = $drive->{file};
2439         return if !$volid;
2440
2441         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2442         if ($storeid) {
2443             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2444             if ($scfg->{disable}) {
2445                 $nodehash = {};
2446             } elsif (my $avail = $scfg->{nodes}) {
2447                 foreach my $node (keys %$nodehash) {
2448                     delete $nodehash->{$node} if !$avail->{$node};
2449                 }
2450             } elsif (!$scfg->{shared}) {
2451                 foreach my $node (keys %$nodehash) {
2452                     delete $nodehash->{$node} if $node ne $nodename
2453                 }
2454             }
2455         }
2456     });
2457
2458     return $nodehash
2459 }
2460
2461 sub check_cmdline {
2462     my ($pidfile, $pid) = @_;
2463
2464     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2465     if (defined($fh)) {
2466         my $line = <$fh>;
2467         $fh->close;
2468         return undef if !$line;
2469         my @param = split(/\0/, $line);
2470
2471         my $cmd = $param[0];
2472         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2473
2474         for (my $i = 0; $i < scalar (@param); $i++) {
2475             my $p = $param[$i];
2476             next if !$p;
2477             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2478                 my $p = $param[$i+1];
2479                 return 1 if $p && ($p eq $pidfile);
2480                 return undef;
2481             }
2482         }
2483     }
2484     return undef;
2485 }
2486
2487 sub check_running {
2488     my ($vmid, $nocheck, $node) = @_;
2489
2490     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2491
2492     die "unable to find configuration file for VM $vmid - no such machine\n"
2493         if !$nocheck && ! -f $filename;
2494
2495     my $pidfile = pidfile_name($vmid);
2496
2497     if (my $fd = IO::File->new("<$pidfile")) {
2498         my $st = stat($fd);
2499         my $line = <$fd>;
2500         close($fd);
2501
2502         my $mtime = $st->mtime;
2503         if ($mtime > time()) {
2504             warn "file '$filename' modified in future\n";
2505         }
2506
2507         if ($line =~ m/^(\d+)$/) {
2508             my $pid = $1;
2509             if (check_cmdline($pidfile, $pid)) {
2510                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2511                     return $pid;
2512                 }
2513             }
2514         }
2515     }
2516
2517     return undef;
2518 }
2519
2520 sub vzlist {
2521
2522     my $vzlist = config_list();
2523
2524     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2525
2526     while (defined(my $de = $fd->read)) {
2527         next if $de !~ m/^(\d+)\.pid$/;
2528         my $vmid = $1;
2529         next if !defined($vzlist->{$vmid});
2530         if (my $pid = check_running($vmid)) {
2531             $vzlist->{$vmid}->{pid} = $pid;
2532         }
2533     }
2534
2535     return $vzlist;
2536 }
2537
2538 sub disksize {
2539     my ($storecfg, $conf) = @_;
2540
2541     my $bootdisk = $conf->{bootdisk};
2542     return undef if !$bootdisk;
2543     return undef if !is_valid_drivename($bootdisk);
2544
2545     return undef if !$conf->{$bootdisk};
2546
2547     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2548     return undef if !defined($drive);
2549
2550     return undef if drive_is_cdrom($drive);
2551
2552     my $volid = $drive->{file};
2553     return undef if !$volid;
2554
2555     return $drive->{size};
2556 }
2557
2558 my $last_proc_pid_stat;
2559
2560 # get VM status information
2561 # This must be fast and should not block ($full == false)
2562 # We only query KVM using QMP if $full == true (this can be slow)
2563 sub vmstatus {
2564     my ($opt_vmid, $full) = @_;
2565
2566     my $res = {};
2567
2568     my $storecfg = PVE::Storage::config();
2569
2570     my $list = vzlist();
2571     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2572
2573     my $cpucount = $cpuinfo->{cpus} || 1;
2574
2575     foreach my $vmid (keys %$list) {
2576         next if $opt_vmid && ($vmid ne $opt_vmid);
2577
2578         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2579         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2580
2581         my $d = {};
2582         $d->{pid} = $list->{$vmid}->{pid};
2583
2584         # fixme: better status?
2585         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2586
2587         my $size = disksize($storecfg, $conf);
2588         if (defined($size)) {
2589             $d->{disk} = 0; # no info available
2590             $d->{maxdisk} = $size;
2591         } else {
2592             $d->{disk} = 0;
2593             $d->{maxdisk} = 0;
2594         }
2595
2596         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2597         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2598         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2599
2600         $d->{name} = $conf->{name} || "VM $vmid";
2601         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2602
2603         if ($conf->{balloon}) {
2604             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2605             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2606         }
2607
2608         $d->{uptime} = 0;
2609         $d->{cpu} = 0;
2610         $d->{mem} = 0;
2611
2612         $d->{netout} = 0;
2613         $d->{netin} = 0;
2614
2615         $d->{diskread} = 0;
2616         $d->{diskwrite} = 0;
2617
2618         $d->{template} = PVE::QemuConfig->is_template($conf);
2619
2620         $res->{$vmid} = $d;
2621     }
2622
2623     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2624     foreach my $dev (keys %$netdev) {
2625         next if $dev !~ m/^tap([1-9]\d*)i/;
2626         my $vmid = $1;
2627         my $d = $res->{$vmid};
2628         next if !$d;
2629
2630         $d->{netout} += $netdev->{$dev}->{receive};
2631         $d->{netin} += $netdev->{$dev}->{transmit};
2632
2633         if ($full) {
2634             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2635             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2636         }
2637
2638     }
2639
2640     my $ctime = gettimeofday;
2641
2642     foreach my $vmid (keys %$list) {
2643
2644         my $d = $res->{$vmid};
2645         my $pid = $d->{pid};
2646         next if !$pid;
2647
2648         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2649         next if !$pstat; # not running
2650
2651         my $used = $pstat->{utime} + $pstat->{stime};
2652
2653         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2654
2655         if ($pstat->{vsize}) {
2656             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2657         }
2658
2659         my $old = $last_proc_pid_stat->{$pid};
2660         if (!$old) {
2661             $last_proc_pid_stat->{$pid} = {
2662                 time => $ctime,
2663                 used => $used,
2664                 cpu => 0,
2665             };
2666             next;
2667         }
2668
2669         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2670
2671         if ($dtime > 1000) {
2672             my $dutime = $used -  $old->{used};
2673
2674             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2675             $last_proc_pid_stat->{$pid} = {
2676                 time => $ctime,
2677                 used => $used,
2678                 cpu => $d->{cpu},
2679             };
2680         } else {
2681             $d->{cpu} = $old->{cpu};
2682         }
2683     }
2684
2685     return $res if !$full;
2686
2687     my $qmpclient = PVE::QMPClient->new();
2688
2689     my $ballooncb = sub {
2690         my ($vmid, $resp) = @_;
2691
2692         my $info = $resp->{'return'};
2693         return if !$info->{max_mem};
2694
2695         my $d = $res->{$vmid};
2696
2697         # use memory assigned to VM
2698         $d->{maxmem} = $info->{max_mem};
2699         $d->{balloon} = $info->{actual};
2700
2701         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2702             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2703             $d->{freemem} = $info->{free_mem};
2704         }
2705
2706         $d->{ballooninfo} = $info;
2707     };
2708
2709     my $blockstatscb = sub {
2710         my ($vmid, $resp) = @_;
2711         my $data = $resp->{'return'} || [];
2712         my $totalrdbytes = 0;
2713         my $totalwrbytes = 0;
2714
2715         for my $blockstat (@$data) {
2716             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2717             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2718
2719             $blockstat->{device} =~ s/drive-//;
2720             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2721         }
2722         $res->{$vmid}->{diskread} = $totalrdbytes;
2723         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2724     };
2725
2726     my $statuscb = sub {
2727         my ($vmid, $resp) = @_;
2728
2729         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2730         # this fails if ballon driver is not loaded, so this must be
2731         # the last commnand (following command are aborted if this fails).
2732         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2733
2734         my $status = 'unknown';
2735         if (!defined($status = $resp->{'return'}->{status})) {
2736             warn "unable to get VM status\n";
2737             return;
2738         }
2739
2740         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2741     };
2742
2743     foreach my $vmid (keys %$list) {
2744         next if $opt_vmid && ($vmid ne $opt_vmid);
2745         next if !$res->{$vmid}->{pid}; # not running
2746         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2747     }
2748
2749     $qmpclient->queue_execute(undef, 2);
2750
2751     foreach my $vmid (keys %$list) {
2752         next if $opt_vmid && ($vmid ne $opt_vmid);
2753         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2754     }
2755
2756     return $res;
2757 }
2758
2759 sub foreach_drive {
2760     my ($conf, $func, @param) = @_;
2761
2762     foreach my $ds (valid_drive_names()) {
2763         next if !defined($conf->{$ds});
2764
2765         my $drive = parse_drive($ds, $conf->{$ds});
2766         next if !$drive;
2767
2768         &$func($ds, $drive, @param);
2769     }
2770 }
2771
2772 sub foreach_volid {
2773     my ($conf, $func, @param) = @_;
2774
2775     my $volhash = {};
2776
2777     my $test_volid = sub {
2778         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2779
2780         return if !$volid;
2781
2782         $volhash->{$volid}->{cdrom} //= 1;
2783         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2784
2785         $volhash->{$volid}->{replicate} //= 0;
2786         $volhash->{$volid}->{replicate} = 1 if $replicate;
2787
2788         $volhash->{$volid}->{referenced_in_config} //= 0;
2789         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2790
2791         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2792             if defined($snapname);
2793     };
2794
2795     foreach_drive($conf, sub {
2796         my ($ds, $drive) = @_;
2797         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2798     });
2799
2800     foreach my $snapname (keys %{$conf->{snapshots}}) {
2801         my $snap = $conf->{snapshots}->{$snapname};
2802         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2803         foreach_drive($snap, sub {
2804             my ($ds, $drive) = @_;
2805             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2806         });
2807     }
2808
2809     foreach my $volid (keys %$volhash) {
2810         &$func($volid, $volhash->{$volid}, @param);
2811     }
2812 }
2813
2814 sub vga_conf_has_spice {
2815     my ($vga) = @_;
2816
2817     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2818
2819     return $1 || 1;
2820 }
2821
2822 sub config_to_command {
2823     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2824
2825     my $cmd = [];
2826     my $globalFlags = [];
2827     my $machineFlags = [];
2828     my $rtcFlags = [];
2829     my $cpuFlags = [];
2830     my $devices = [];
2831     my $pciaddr = '';
2832     my $bridges = {};
2833     my $kvmver = kvm_user_version();
2834     my $vernum = 0; # unknown
2835     my $ostype = $conf->{ostype};
2836     my $winversion = windows_version($ostype);
2837     my $kvm = $conf->{kvm} // 1;
2838
2839     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2840
2841     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2842         $vernum = $1*1000000+$2*1000;
2843     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2844         $vernum = $1*1000000+$2*1000+$3;
2845     }
2846
2847     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2848
2849     my $have_ovz = -f '/proc/vz/vestat';
2850
2851     my $q35 = machine_type_is_q35($conf);
2852     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2853     my $machine_type = $forcemachine || $conf->{machine};
2854     my $use_old_bios_files = undef;
2855     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2856
2857     my $cpuunits = defined($conf->{cpuunits}) ?
2858             $conf->{cpuunits} : $defaults->{cpuunits};
2859
2860     push @$cmd, '/usr/bin/kvm';
2861
2862     push @$cmd, '-id', $vmid;
2863
2864     my $use_virtio = 0;
2865
2866     my $qmpsocket = qmp_socket($vmid);
2867     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2868     push @$cmd, '-mon', "chardev=qmp,mode=control";
2869
2870
2871     push @$cmd, '-pidfile' , pidfile_name($vmid);
2872
2873     push @$cmd, '-daemonize';
2874
2875     if ($conf->{smbios1}) {
2876         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2877     }
2878
2879     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2880         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2881
2882         my $path;
2883         if (my $efidisk = $conf->{efidisk0}) {
2884             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2885             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2886             if ($storeid) {
2887                 $path = PVE::Storage::path($storecfg, $d->{file});
2888             } else {
2889                 $path = $d->{file};
2890             }
2891         } else {
2892             warn "no efidisk configured! Using temporary efivars disk.\n";
2893             $path = "/tmp/$vmid-ovmf.fd";
2894             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2895         }
2896
2897         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2898         push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,file=$path";
2899     }
2900
2901
2902     # add usb controllers
2903     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2904     push @$devices, @usbcontrollers if @usbcontrollers;
2905     my $vga = $conf->{vga};
2906
2907     my $qxlnum = vga_conf_has_spice($vga);
2908     $vga = 'qxl' if $qxlnum;
2909
2910     if (!$vga) {
2911         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2912             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2913         } else {
2914             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2915         }
2916     }
2917
2918     # enable absolute mouse coordinates (needed by vnc)
2919     my $tablet;
2920     if (defined($conf->{tablet})) {
2921         $tablet = $conf->{tablet};
2922     } else {
2923         $tablet = $defaults->{tablet};
2924         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2925         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2926     }
2927
2928     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2929
2930     my $kvm_off = 0;
2931     my $gpu_passthrough;
2932
2933     # host pci devices
2934     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2935         my $d = parse_hostpci($conf->{"hostpci$i"});
2936         next if !$d;
2937
2938         my $pcie = $d->{pcie};
2939         if($pcie){
2940             die "q35 machine model is not enabled" if !$q35;
2941             $pciaddr = print_pcie_addr("hostpci$i");
2942         }else{
2943             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2944         }
2945
2946         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2947         my $romfile = $d->{romfile};
2948
2949         my $xvga = '';
2950         if ($d->{'x-vga'}) {
2951             $xvga = ',x-vga=on';
2952             $kvm_off = 1;
2953             $vga = 'none';
2954             $gpu_passthrough = 1;
2955
2956             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2957                 $xvga = "";
2958             }
2959         }
2960         my $pcidevices = $d->{pciid};
2961         my $multifunction = 1 if @$pcidevices > 1;
2962
2963         my $j=0;
2964         foreach my $pcidevice (@$pcidevices) {
2965
2966             my $id = "hostpci$i";
2967             $id .= ".$j" if $multifunction;
2968             my $addr = $pciaddr;
2969             $addr .= ".$j" if $multifunction;
2970             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2971
2972             if($j == 0){
2973                 $devicestr .= "$rombar$xvga";
2974                 $devicestr .= ",multifunction=on" if $multifunction;
2975                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2976             }
2977
2978             push @$devices, '-device', $devicestr;
2979             $j++;
2980         }
2981     }
2982
2983     # usb devices
2984     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2985     push @$devices, @usbdevices if @usbdevices;
2986     # serial devices
2987     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2988         if (my $path = $conf->{"serial$i"}) {
2989             if ($path eq 'socket') {
2990                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2991                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2992                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2993             } else {
2994                 die "no such serial device\n" if ! -c $path;
2995                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2996                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2997             }
2998         }
2999     }
3000
3001     # parallel devices
3002     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3003         if (my $path = $conf->{"parallel$i"}) {
3004             die "no such parallel device\n" if ! -c $path;
3005             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3006             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3007             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3008         }
3009     }
3010
3011     my $vmname = $conf->{name} || "vm$vmid";
3012
3013     push @$cmd, '-name', $vmname;
3014
3015     my $sockets = 1;
3016     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3017     $sockets = $conf->{sockets} if  $conf->{sockets};
3018
3019     my $cores = $conf->{cores} || 1;
3020
3021     my $maxcpus = $sockets * $cores;
3022
3023     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3024
3025     my $allowed_vcpus = $cpuinfo->{cpus};
3026
3027     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3028         if ($allowed_vcpus < $maxcpus);
3029
3030     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3031
3032         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3033         for (my $i = 2; $i <= $vcpus; $i++)  {
3034             my $cpustr = print_cpu_device($conf,$i);
3035             push @$cmd, '-device', $cpustr;
3036         }
3037
3038     } else {
3039
3040         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3041     }
3042     push @$cmd, '-nodefaults';
3043
3044     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3045
3046     my $bootindex_hash = {};
3047     my $i = 1;
3048     foreach my $o (split(//, $bootorder)) {
3049         $bootindex_hash->{$o} = $i*100;
3050         $i++;
3051     }
3052
3053     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3054
3055     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3056
3057     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3058
3059     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3060
3061     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3062         my $socket = vnc_socket($vmid);
3063         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3064     } else {
3065         push @$cmd, '-nographic';
3066     }
3067
3068     # time drift fix
3069     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3070
3071     my $useLocaltime = $conf->{localtime};
3072
3073     if ($winversion >= 5) { # windows
3074         $useLocaltime = 1 if !defined($conf->{localtime});
3075
3076         # use time drift fix when acpi is enabled
3077         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3078             $tdf = 1 if !defined($conf->{tdf});
3079         }
3080     }
3081
3082     if ($winversion >= 6) {
3083         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3084         push @$cmd, '-no-hpet';
3085     }
3086
3087     push @$rtcFlags, 'driftfix=slew' if $tdf;
3088
3089     if (!$kvm) {
3090         push @$machineFlags, 'accel=tcg';
3091     }
3092
3093     if ($machine_type) {
3094         push @$machineFlags, "type=${machine_type}";
3095     }
3096
3097     if ($conf->{startdate}) {
3098         push @$rtcFlags, "base=$conf->{startdate}";
3099     } elsif ($useLocaltime) {
3100         push @$rtcFlags, 'base=localtime';
3101     }
3102
3103     my $cpu = $kvm ? "kvm64" : "qemu64";
3104     if (my $cputype = $conf->{cpu}) {
3105         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3106             or die "Cannot parse cpu description: $cputype\n";
3107         $cpu = $cpuconf->{cputype};
3108         $kvm_off = 1 if $cpuconf->{hidden};
3109     }
3110
3111     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3112
3113     push @$cpuFlags , '-x2apic'
3114         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3115
3116     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3117
3118     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3119
3120     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3121
3122         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3123         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3124     }
3125
3126     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3127
3128     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3129
3130     push @$cpuFlags, 'kvm=off' if $kvm_off;
3131
3132     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3133         die "internal error"; # should not happen
3134
3135     push @$cpuFlags, "vendor=${cpu_vendor}"
3136         if $cpu_vendor ne 'default';
3137
3138     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3139
3140     push @$cmd, '-cpu', $cpu;
3141
3142     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3143
3144     push @$cmd, '-S' if $conf->{freeze};
3145
3146     # set keyboard layout
3147     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3148     push @$cmd, '-k', $kb if $kb;
3149
3150     # enable sound
3151     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3152     #push @$cmd, '-soundhw', 'es1370';
3153     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3154
3155     if($conf->{agent}) {
3156         my $qgasocket = qmp_socket($vmid, 1);
3157         my $pciaddr = print_pci_addr("qga0", $bridges);
3158         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3159         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3160         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3161     }
3162
3163     my $spice_port;
3164
3165     if ($qxlnum) {
3166         if ($qxlnum > 1) {
3167             if ($winversion){
3168                 for(my $i = 1; $i < $qxlnum; $i++){
3169                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3170                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3171                 }
3172             } else {
3173                 # assume other OS works like Linux
3174                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3175                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3176             }
3177         }
3178
3179         my $pciaddr = print_pci_addr("spice", $bridges);
3180
3181         my $nodename = PVE::INotify::nodename();
3182         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3183         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3184         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3185         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3186         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3187
3188         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3189
3190         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3191         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3192         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3193     }
3194
3195     # enable balloon by default, unless explicitly disabled
3196     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3197         $pciaddr = print_pci_addr("balloon0", $bridges);
3198         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3199     }
3200
3201     if ($conf->{watchdog}) {
3202         my $wdopts = parse_watchdog($conf->{watchdog});
3203         $pciaddr = print_pci_addr("watchdog", $bridges);
3204         my $watchdog = $wdopts->{model} || 'i6300esb';
3205         push @$devices, '-device', "$watchdog$pciaddr";
3206         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3207     }
3208
3209     my $vollist = [];
3210     my $scsicontroller = {};
3211     my $ahcicontroller = {};
3212     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3213
3214     # Add iscsi initiator name if available
3215     if (my $initiator = get_initiator_name()) {
3216         push @$devices, '-iscsi', "initiator-name=$initiator";
3217     }
3218
3219     foreach_drive($conf, sub {
3220         my ($ds, $drive) = @_;
3221
3222         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3223             push @$vollist, $drive->{file};
3224         }
3225
3226         # ignore efidisk here, already added in bios/fw handling code above
3227         return if $drive->{interface} eq 'efidisk';
3228
3229         $use_virtio = 1 if $ds =~ m/^virtio/;
3230
3231         if (drive_is_cdrom ($drive)) {
3232             if ($bootindex_hash->{d}) {
3233                 $drive->{bootindex} = $bootindex_hash->{d};
3234                 $bootindex_hash->{d} += 1;
3235             }
3236         } else {
3237             if ($bootindex_hash->{c}) {
3238                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3239                 $bootindex_hash->{c} += 1;
3240             }
3241         }
3242
3243         if($drive->{interface} eq 'virtio'){
3244            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3245         }
3246
3247         if ($drive->{interface} eq 'scsi') {
3248
3249             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3250
3251             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3252             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3253
3254             my $iothread = '';
3255             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3256                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3257                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3258             } elsif ($drive->{iothread}) {
3259                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3260             }
3261
3262             my $queues = '';
3263             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3264                 $queues = ",num_queues=$drive->{queues}";
3265             }
3266
3267             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3268             $scsicontroller->{$controller}=1;
3269         }
3270
3271         if ($drive->{interface} eq 'sata') {
3272            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3273            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3274            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3275            $ahcicontroller->{$controller}=1;
3276         }
3277
3278         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3279         push @$devices, '-drive',$drive_cmd;
3280         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3281     });
3282
3283     for (my $i = 0; $i < $MAX_NETS; $i++) {
3284          next if !$conf->{"net$i"};
3285          my $d = parse_net($conf->{"net$i"});
3286          next if !$d;
3287
3288          $use_virtio = 1 if $d->{model} eq 'virtio';
3289
3290          if ($bootindex_hash->{n}) {
3291             $d->{bootindex} = $bootindex_hash->{n};
3292             $bootindex_hash->{n} += 1;
3293          }
3294
3295          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3296          push @$devices, '-netdev', $netdevfull;
3297
3298          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3299          push @$devices, '-device', $netdevicefull;
3300     }
3301
3302     if (!$q35) {
3303         # add pci bridges
3304         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3305            $bridges->{1} = 1;
3306            $bridges->{2} = 1;
3307         }
3308
3309         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3310
3311         while (my ($k, $v) = each %$bridges) {
3312             $pciaddr = print_pci_addr("pci.$k");
3313             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3314         }
3315     }
3316
3317     # add custom args
3318     if ($conf->{args}) {
3319         my $aa = PVE::Tools::split_args($conf->{args});
3320         push @$cmd, @$aa;
3321     }
3322
3323     push @$cmd, @$devices;
3324     push @$cmd, '-rtc', join(',', @$rtcFlags)
3325         if scalar(@$rtcFlags);
3326     push @$cmd, '-machine', join(',', @$machineFlags)
3327         if scalar(@$machineFlags);
3328     push @$cmd, '-global', join(',', @$globalFlags)
3329         if scalar(@$globalFlags);
3330
3331     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3332 }
3333
3334 sub vnc_socket {
3335     my ($vmid) = @_;
3336     return "${var_run_tmpdir}/$vmid.vnc";
3337 }
3338
3339 sub spice_port {
3340     my ($vmid) = @_;
3341
3342     my $res = vm_mon_cmd($vmid, 'query-spice');
3343
3344     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3345 }
3346
3347 sub qmp_socket {
3348     my ($vmid, $qga) = @_;
3349     my $sockettype = $qga ? 'qga' : 'qmp';
3350     return "${var_run_tmpdir}/$vmid.$sockettype";
3351 }
3352
3353 sub pidfile_name {
3354     my ($vmid) = @_;
3355     return "${var_run_tmpdir}/$vmid.pid";
3356 }
3357
3358 sub vm_devices_list {
3359     my ($vmid) = @_;
3360
3361     my $res = vm_mon_cmd($vmid, 'query-pci');
3362     my $devices = {};
3363     foreach my $pcibus (@$res) {
3364         foreach my $device (@{$pcibus->{devices}}) {
3365             next if !$device->{'qdev_id'};
3366             if ($device->{'pci_bridge'}) {
3367                 $devices->{$device->{'qdev_id'}} = 1;
3368                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3369                     next if !$bridge_device->{'qdev_id'};
3370                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3371                     $devices->{$device->{'qdev_id'}}++;
3372                 }
3373             } else {
3374                 $devices->{$device->{'qdev_id'}} = 1;
3375             }
3376         }
3377     }
3378
3379     my $resblock = vm_mon_cmd($vmid, 'query-block');
3380     foreach my $block (@$resblock) {
3381         if($block->{device} =~ m/^drive-(\S+)/){
3382                 $devices->{$1} = 1;
3383         }
3384     }
3385
3386     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3387     foreach my $mice (@$resmice) {
3388         if ($mice->{name} eq 'QEMU HID Tablet') {
3389             $devices->{tablet} = 1;
3390             last;
3391         }
3392     }
3393
3394     # for usb devices there is no query-usb
3395     # but we can iterate over the entries in
3396     # qom-list path=/machine/peripheral
3397     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3398     foreach my $per (@$resperipheral) {
3399         if ($per->{name} =~ m/^usb\d+$/) {
3400             $devices->{$per->{name}} = 1;
3401         }
3402     }
3403
3404     return $devices;
3405 }
3406
3407 sub vm_deviceplug {
3408     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3409
3410     my $q35 = machine_type_is_q35($conf);
3411
3412     my $devices_list = vm_devices_list($vmid);
3413     return 1 if defined($devices_list->{$deviceid});
3414
3415     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3416
3417     if ($deviceid eq 'tablet') {
3418
3419         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3420
3421     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3422
3423         die "usb hotplug currently not reliable\n";
3424         # since we can't reliably hot unplug all added usb devices
3425         # and usb passthrough disables live migration
3426         # we disable usb hotplugging for now
3427         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3428
3429     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3430
3431         qemu_iothread_add($vmid, $deviceid, $device);
3432
3433         qemu_driveadd($storecfg, $vmid, $device);
3434         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3435
3436         qemu_deviceadd($vmid, $devicefull);
3437         eval { qemu_deviceaddverify($vmid, $deviceid); };
3438         if (my $err = $@) {
3439             eval { qemu_drivedel($vmid, $deviceid); };
3440             warn $@ if $@;
3441             die $err;
3442         }
3443
3444     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3445
3446
3447         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3448         my $pciaddr = print_pci_addr($deviceid);
3449         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3450
3451         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3452
3453         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3454             qemu_iothread_add($vmid, $deviceid, $device);
3455             $devicefull .= ",iothread=iothread-$deviceid";
3456         }
3457
3458         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3459             $devicefull .= ",num_queues=$device->{queues}";
3460         }
3461
3462         qemu_deviceadd($vmid, $devicefull);
3463         qemu_deviceaddverify($vmid, $deviceid);
3464
3465     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3466
3467         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3468         qemu_driveadd($storecfg, $vmid, $device);
3469
3470         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3471         eval { qemu_deviceadd($vmid, $devicefull); };
3472         if (my $err = $@) {
3473             eval { qemu_drivedel($vmid, $deviceid); };
3474             warn $@ if $@;
3475             die $err;
3476         }
3477
3478     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3479
3480         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3481
3482         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3483         my $use_old_bios_files = undef;
3484         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3485
3486         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3487         qemu_deviceadd($vmid, $netdevicefull);
3488         eval { qemu_deviceaddverify($vmid, $deviceid); };
3489         if (my $err = $@) {
3490             eval { qemu_netdevdel($vmid, $deviceid); };
3491             warn $@ if $@;
3492             die $err;
3493         }
3494
3495     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3496
3497         my $bridgeid = $2;
3498         my $pciaddr = print_pci_addr($deviceid);
3499         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3500
3501         qemu_deviceadd($vmid, $devicefull);
3502         qemu_deviceaddverify($vmid, $deviceid);
3503
3504     } else {
3505         die "can't hotplug device '$deviceid'\n";
3506     }
3507
3508     return 1;
3509 }
3510
3511 # fixme: this should raise exceptions on error!
3512 sub vm_deviceunplug {
3513     my ($vmid, $conf, $deviceid) = @_;
3514
3515     my $devices_list = vm_devices_list($vmid);
3516     return 1 if !defined($devices_list->{$deviceid});
3517
3518     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3519
3520     if ($deviceid eq 'tablet') {
3521
3522         qemu_devicedel($vmid, $deviceid);
3523
3524     } elsif ($deviceid =~ m/^usb\d+$/) {
3525
3526         die "usb hotplug currently not reliable\n";
3527         # when unplugging usb devices this way,
3528         # there may be remaining usb controllers/hubs
3529         # so we disable it for now
3530         qemu_devicedel($vmid, $deviceid);
3531         qemu_devicedelverify($vmid, $deviceid);
3532
3533     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3534
3535         qemu_devicedel($vmid, $deviceid);
3536         qemu_devicedelverify($vmid, $deviceid);
3537         qemu_drivedel($vmid, $deviceid);
3538         qemu_iothread_del($conf, $vmid, $deviceid);
3539
3540     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3541
3542         qemu_devicedel($vmid, $deviceid);
3543         qemu_devicedelverify($vmid, $deviceid);
3544         qemu_iothread_del($conf, $vmid, $deviceid);
3545
3546     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3547
3548         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3549         my $device = parse_drive($deviceid, $conf->{$deviceid});
3550         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3551
3552         qemu_devicedel($vmid, $deviceid);
3553         qemu_drivedel($vmid, $deviceid);
3554         qemu_deletescsihw($conf, $vmid, $deviceid);
3555
3556     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3557
3558         qemu_devicedel($vmid, $deviceid);
3559         qemu_devicedelverify($vmid, $deviceid);
3560         qemu_netdevdel($vmid, $deviceid);
3561
3562     } else {
3563         die "can't unplug device '$deviceid'\n";
3564     }
3565
3566     return 1;
3567 }
3568
3569 sub qemu_deviceadd {
3570     my ($vmid, $devicefull) = @_;
3571
3572     $devicefull = "driver=".$devicefull;
3573     my %options =  split(/[=,]/, $devicefull);
3574
3575     vm_mon_cmd($vmid, "device_add" , %options);
3576 }
3577
3578 sub qemu_devicedel {
3579     my ($vmid, $deviceid) = @_;
3580
3581     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3582 }
3583
3584 sub qemu_iothread_add {
3585     my($vmid, $deviceid, $device) = @_;
3586
3587     if ($device->{iothread}) {
3588         my $iothreads = vm_iothreads_list($vmid);
3589         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3590     }
3591 }
3592
3593 sub qemu_iothread_del {
3594     my($conf, $vmid, $deviceid) = @_;
3595
3596     my $device = parse_drive($deviceid, $conf->{$deviceid});
3597     if ($device->{iothread}) {
3598         my $iothreads = vm_iothreads_list($vmid);
3599         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3600     }
3601 }
3602
3603 sub qemu_objectadd {
3604     my($vmid, $objectid, $qomtype) = @_;
3605
3606     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3607
3608     return 1;
3609 }
3610
3611 sub qemu_objectdel {
3612     my($vmid, $objectid) = @_;
3613
3614     vm_mon_cmd($vmid, "object-del", id => $objectid);
3615
3616     return 1;
3617 }
3618
3619 sub qemu_driveadd {
3620     my ($storecfg, $vmid, $device) = @_;
3621
3622     my $drive = print_drive_full($storecfg, $vmid, $device);
3623     $drive =~ s/\\/\\\\/g;
3624     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3625
3626     # If the command succeeds qemu prints: "OK"
3627     return 1 if $ret =~ m/OK/s;
3628
3629     die "adding drive failed: $ret\n";
3630 }
3631
3632 sub qemu_drivedel {
3633     my($vmid, $deviceid) = @_;
3634
3635     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3636     $ret =~ s/^\s+//;
3637
3638     return 1 if $ret eq "";
3639
3640     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3641     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3642
3643     die "deleting drive $deviceid failed : $ret\n";
3644 }
3645
3646 sub qemu_deviceaddverify {
3647     my ($vmid, $deviceid) = @_;
3648
3649     for (my $i = 0; $i <= 5; $i++) {
3650          my $devices_list = vm_devices_list($vmid);
3651          return 1 if defined($devices_list->{$deviceid});
3652          sleep 1;
3653     }
3654
3655     die "error on hotplug device '$deviceid'\n";
3656 }
3657
3658
3659 sub qemu_devicedelverify {
3660     my ($vmid, $deviceid) = @_;
3661
3662     # need to verify that the device is correctly removed as device_del
3663     # is async and empty return is not reliable
3664
3665     for (my $i = 0; $i <= 5; $i++) {
3666          my $devices_list = vm_devices_list($vmid);
3667          return 1 if !defined($devices_list->{$deviceid});
3668          sleep 1;
3669     }
3670
3671     die "error on hot-unplugging device '$deviceid'\n";
3672 }
3673
3674 sub qemu_findorcreatescsihw {
3675     my ($storecfg, $conf, $vmid, $device) = @_;
3676
3677     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3678
3679     my $scsihwid="$controller_prefix$controller";
3680     my $devices_list = vm_devices_list($vmid);
3681
3682     if(!defined($devices_list->{$scsihwid})) {
3683         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3684     }
3685
3686     return 1;
3687 }
3688
3689 sub qemu_deletescsihw {
3690     my ($conf, $vmid, $opt) = @_;
3691
3692     my $device = parse_drive($opt, $conf->{$opt});
3693
3694     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3695         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3696         return 1;
3697     }
3698
3699     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3700
3701     my $devices_list = vm_devices_list($vmid);
3702     foreach my $opt (keys %{$devices_list}) {
3703         if (PVE::QemuServer::is_valid_drivename($opt)) {
3704             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3705             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3706                 return 1;
3707             }
3708         }
3709     }
3710
3711     my $scsihwid="scsihw$controller";
3712
3713     vm_deviceunplug($vmid, $conf, $scsihwid);
3714
3715     return 1;
3716 }
3717
3718 sub qemu_add_pci_bridge {
3719     my ($storecfg, $conf, $vmid, $device) = @_;
3720
3721     my $bridges = {};
3722
3723     my $bridgeid;
3724
3725     print_pci_addr($device, $bridges);
3726
3727     while (my ($k, $v) = each %$bridges) {
3728         $bridgeid = $k;
3729     }
3730     return 1 if !defined($bridgeid) || $bridgeid < 1;
3731
3732     my $bridge = "pci.$bridgeid";
3733     my $devices_list = vm_devices_list($vmid);
3734
3735     if (!defined($devices_list->{$bridge})) {
3736         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3737     }
3738
3739     return 1;
3740 }
3741
3742 sub qemu_set_link_status {
3743     my ($vmid, $device, $up) = @_;
3744
3745     vm_mon_cmd($vmid, "set_link", name => $device,
3746                up => $up ? JSON::true : JSON::false);
3747 }
3748
3749 sub qemu_netdevadd {
3750     my ($vmid, $conf, $device, $deviceid) = @_;
3751
3752     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3753     my %options =  split(/[=,]/, $netdev);
3754
3755     vm_mon_cmd($vmid, "netdev_add",  %options);
3756     return 1;
3757 }
3758
3759 sub qemu_netdevdel {
3760     my ($vmid, $deviceid) = @_;
3761
3762     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3763 }
3764
3765 sub qemu_usb_hotplug {
3766     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3767
3768     return if !$device;
3769
3770     # remove the old one first
3771     vm_deviceunplug($vmid, $conf, $deviceid);
3772
3773     # check if xhci controller is necessary and available
3774     if ($device->{usb3}) {
3775
3776         my $devicelist = vm_devices_list($vmid);
3777
3778         if (!$devicelist->{xhci}) {
3779             my $pciaddr = print_pci_addr("xhci");
3780             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3781         }
3782     }
3783     my $d = parse_usb_device($device->{host});
3784     $d->{usb3} = $device->{usb3};
3785
3786     # add the new one
3787     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3788 }
3789
3790 sub qemu_cpu_hotplug {
3791     my ($vmid, $conf, $vcpus) = @_;
3792
3793     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3794
3795     my $sockets = 1;
3796     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3797     $sockets = $conf->{sockets} if  $conf->{sockets};
3798     my $cores = $conf->{cores} || 1;
3799     my $maxcpus = $sockets * $cores;
3800
3801     $vcpus = $maxcpus if !$vcpus;
3802
3803     die "you can't add more vcpus than maxcpus\n"
3804         if $vcpus > $maxcpus;
3805
3806     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3807
3808     if ($vcpus < $currentvcpus) {
3809
3810         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3811
3812             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3813                 qemu_devicedel($vmid, "cpu$i");
3814                 my $retry = 0;
3815                 my $currentrunningvcpus = undef;
3816                 while (1) {
3817                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3818                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3819                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3820                     $retry++;
3821                     sleep 1;
3822                 }
3823                 #update conf after each succesfull cpu unplug
3824                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3825                 PVE::QemuConfig->write_config($vmid, $conf);
3826             }
3827         } else {
3828             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3829         }
3830
3831         return;
3832     }
3833
3834     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3835     die "vcpus in running vm does not match its configuration\n"
3836         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3837
3838     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3839
3840         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3841             my $cpustr = print_cpu_device($conf, $i);
3842             qemu_deviceadd($vmid, $cpustr);
3843
3844             my $retry = 0;
3845             my $currentrunningvcpus = undef;
3846             while (1) {
3847                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3848                 last if scalar(@{$currentrunningvcpus}) == $i;
3849                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3850                 sleep 1;
3851                 $retry++;
3852             }
3853             #update conf after each succesfull cpu hotplug
3854             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3855             PVE::QemuConfig->write_config($vmid, $conf);
3856         }
3857     } else {
3858
3859         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3860             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3861         }
3862     }
3863 }
3864
3865 sub qemu_block_set_io_throttle {
3866     my ($vmid, $deviceid,
3867         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3868         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3869         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3870         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3871
3872     return if !check_running($vmid) ;
3873
3874     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3875         bps => int($bps),
3876         bps_rd => int($bps_rd),
3877         bps_wr => int($bps_wr),
3878         iops => int($iops),
3879         iops_rd => int($iops_rd),
3880         iops_wr => int($iops_wr),
3881         bps_max => int($bps_max),
3882         bps_rd_max => int($bps_rd_max),
3883         bps_wr_max => int($bps_wr_max),
3884         iops_max => int($iops_max),
3885         iops_rd_max => int($iops_rd_max),
3886         iops_wr_max => int($iops_wr_max),
3887         bps_max_length => int($bps_max_length),
3888         bps_rd_max_length => int($bps_rd_max_length),
3889         bps_wr_max_length => int($bps_wr_max_length),
3890         iops_max_length => int($iops_max_length),
3891         iops_rd_max_length => int($iops_rd_max_length),
3892         iops_wr_max_length => int($iops_wr_max_length),
3893     );
3894
3895 }
3896
3897 # old code, only used to shutdown old VM after update
3898 sub __read_avail {
3899     my ($fh, $timeout) = @_;
3900
3901     my $sel = new IO::Select;
3902     $sel->add($fh);
3903
3904     my $res = '';
3905     my $buf;
3906
3907     my @ready;
3908     while (scalar (@ready = $sel->can_read($timeout))) {
3909         my $count;
3910         if ($count = $fh->sysread($buf, 8192)) {
3911             if ($buf =~ /^(.*)\(qemu\) $/s) {
3912                 $res .= $1;
3913                 last;
3914             } else {
3915                 $res .= $buf;
3916             }
3917         } else {
3918             if (!defined($count)) {
3919                 die "$!\n";
3920             }
3921             last;
3922         }
3923     }
3924
3925     die "monitor read timeout\n" if !scalar(@ready);
3926
3927     return $res;
3928 }
3929
3930 # old code, only used to shutdown old VM after update
3931 sub vm_monitor_command {
3932     my ($vmid, $cmdstr, $nocheck) = @_;
3933
3934     my $res;
3935
3936     eval {
3937         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3938
3939         my $sname = "${var_run_tmpdir}/$vmid.mon";
3940
3941         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3942             die "unable to connect to VM $vmid socket - $!\n";
3943
3944         my $timeout = 3;
3945
3946         # hack: migrate sometime blocks the monitor (when migrate_downtime
3947         # is set)
3948         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3949             $timeout = 60*60; # 1 hour
3950         }
3951
3952         # read banner;
3953         my $data = __read_avail($sock, $timeout);
3954
3955         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3956             die "got unexpected qemu monitor banner\n";
3957         }
3958
3959         my $sel = new IO::Select;
3960         $sel->add($sock);
3961
3962         if (!scalar(my @ready = $sel->can_write($timeout))) {
3963             die "monitor write error - timeout";
3964         }
3965
3966         my $fullcmd = "$cmdstr\r";
3967
3968         # syslog('info', "VM $vmid monitor command: $cmdstr");
3969
3970         my $b;
3971         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3972             die "monitor write error - $!";
3973         }
3974
3975         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3976
3977         $timeout = 20;
3978
3979         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3980             $timeout = 60*60; # 1 hour
3981         } elsif ($cmdstr =~ m/^(eject|change)/) {
3982             $timeout = 60; # note: cdrom mount command is slow
3983         }
3984         if ($res = __read_avail($sock, $timeout)) {
3985
3986             my @lines = split("\r?\n", $res);
3987
3988             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3989
3990             $res = join("\n", @lines);
3991             $res .= "\n";
3992         }
3993     };
3994
3995     my $err = $@;
3996
3997     if ($err) {
3998         syslog("err", "VM $vmid monitor command failed - $err");
3999         die $err;
4000     }
4001
4002     return $res;
4003 }
4004
4005 sub qemu_block_resize {
4006     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4007
4008     my $running = check_running($vmid);
4009
4010     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4011
4012     return if !$running;
4013
4014     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4015
4016 }
4017
4018 sub qemu_volume_snapshot {
4019     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4020
4021     my $running = check_running($vmid);
4022
4023     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4024         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4025     } else {
4026         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4027     }
4028 }
4029
4030 sub qemu_volume_snapshot_delete {
4031     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4032
4033     my $running = check_running($vmid);
4034
4035     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4036         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4037     } else {
4038         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4039     }
4040 }
4041
4042 sub set_migration_caps {
4043     my ($vmid) = @_;
4044
4045     my $cap_ref = [];
4046
4047     my $enabled_cap = {
4048         "auto-converge" => 1,
4049         "xbzrle" => 1,
4050         "x-rdma-pin-all" => 0,
4051         "zero-blocks" => 0,
4052         "compress" => 0
4053     };
4054
4055     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4056
4057     for my $supported_capability (@$supported_capabilities) {
4058         push @$cap_ref, {
4059             capability => $supported_capability->{capability},
4060             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4061         };
4062     }
4063
4064     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4065 }
4066
4067 my $fast_plug_option = {
4068     'lock' => 1,
4069     'name' => 1,
4070     'onboot' => 1,
4071     'shares' => 1,
4072     'startup' => 1,
4073     'description' => 1,
4074     'protection' => 1,
4075 };
4076
4077 # hotplug changes in [PENDING]
4078 # $selection hash can be used to only apply specified options, for
4079 # example: { cores => 1 } (only apply changed 'cores')
4080 # $errors ref is used to return error messages
4081 sub vmconfig_hotplug_pending {
4082     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4083
4084     my $defaults = load_defaults();
4085
4086     # commit values which do not have any impact on running VM first
4087     # Note: those option cannot raise errors, we we do not care about
4088     # $selection and always apply them.
4089
4090     my $add_error = sub {
4091         my ($opt, $msg) = @_;
4092         $errors->{$opt} = "hotplug problem - $msg";
4093     };
4094
4095     my $changes = 0;
4096     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4097         if ($fast_plug_option->{$opt}) {
4098             $conf->{$opt} = $conf->{pending}->{$opt};
4099             delete $conf->{pending}->{$opt};
4100             $changes = 1;
4101         }
4102     }
4103
4104     if ($changes) {
4105         PVE::QemuConfig->write_config($vmid, $conf);
4106         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4107     }
4108
4109     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4110
4111     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4112     while (my ($opt, $force) = each %$pending_delete_hash) {
4113         next if $selection && !$selection->{$opt};
4114         eval {
4115             if ($opt eq 'hotplug') {
4116                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4117             } elsif ($opt eq 'tablet') {
4118                 die "skip\n" if !$hotplug_features->{usb};
4119                 if ($defaults->{tablet}) {
4120                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4121                 } else {
4122                     vm_deviceunplug($vmid, $conf, $opt);
4123                 }
4124             } elsif ($opt =~ m/^usb\d+/) {
4125                 die "skip\n";
4126                 # since we cannot reliably hot unplug usb devices
4127                 # we are disabling it
4128                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4129                 vm_deviceunplug($vmid, $conf, $opt);
4130             } elsif ($opt eq 'vcpus') {
4131                 die "skip\n" if !$hotplug_features->{cpu};
4132                 qemu_cpu_hotplug($vmid, $conf, undef);
4133             } elsif ($opt eq 'balloon') {
4134                 # enable balloon device is not hotpluggable
4135                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4136             } elsif ($fast_plug_option->{$opt}) {
4137                 # do nothing
4138             } elsif ($opt =~ m/^net(\d+)$/) {
4139                 die "skip\n" if !$hotplug_features->{network};
4140                 vm_deviceunplug($vmid, $conf, $opt);
4141             } elsif (is_valid_drivename($opt)) {
4142                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4143                 vm_deviceunplug($vmid, $conf, $opt);
4144                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4145             } elsif ($opt =~ m/^memory$/) {
4146                 die "skip\n" if !$hotplug_features->{memory};
4147                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4148             } elsif ($opt eq 'cpuunits') {
4149                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4150             } elsif ($opt eq 'cpulimit') {
4151                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4152             } else {
4153                 die "skip\n";
4154             }
4155         };
4156         if (my $err = $@) {
4157             &$add_error($opt, $err) if $err ne "skip\n";
4158         } else {
4159             # save new config if hotplug was successful
4160             delete $conf->{$opt};
4161             vmconfig_undelete_pending_option($conf, $opt);
4162             PVE::QemuConfig->write_config($vmid, $conf);
4163             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4164         }
4165     }
4166
4167     foreach my $opt (keys %{$conf->{pending}}) {
4168         next if $selection && !$selection->{$opt};
4169         my $value = $conf->{pending}->{$opt};
4170         eval {
4171             if ($opt eq 'hotplug') {
4172                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4173             } elsif ($opt eq 'tablet') {
4174                 die "skip\n" if !$hotplug_features->{usb};
4175                 if ($value == 1) {
4176                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4177                 } elsif ($value == 0) {
4178                     vm_deviceunplug($vmid, $conf, $opt);
4179                 }
4180             } elsif ($opt =~ m/^usb\d+$/) {
4181                 die "skip\n";
4182                 # since we cannot reliably hot unplug usb devices
4183                 # we are disabling it
4184                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4185                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4186                 die "skip\n" if !$d;
4187                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4188             } elsif ($opt eq 'vcpus') {
4189                 die "skip\n" if !$hotplug_features->{cpu};
4190                 qemu_cpu_hotplug($vmid, $conf, $value);
4191             } elsif ($opt eq 'balloon') {
4192                 # enable/disable balloning device is not hotpluggable
4193                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4194                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4195                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4196
4197                 # allow manual ballooning if shares is set to zero
4198                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4199                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4200                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4201                 }
4202             } elsif ($opt =~ m/^net(\d+)$/) {
4203                 # some changes can be done without hotplug
4204                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4205                                     $vmid, $opt, $value);
4206             } elsif (is_valid_drivename($opt)) {
4207                 # some changes can be done without hotplug
4208                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4209                                      $vmid, $opt, $value, 1);
4210             } elsif ($opt =~ m/^memory$/) { #dimms
4211                 die "skip\n" if !$hotplug_features->{memory};
4212                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4213             } elsif ($opt eq 'cpuunits') {
4214                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4215             } elsif ($opt eq 'cpulimit') {
4216                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4217                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4218             } else {
4219                 die "skip\n";  # skip non-hot-pluggable options
4220             }
4221         };
4222         if (my $err = $@) {
4223             &$add_error($opt, $err) if $err ne "skip\n";
4224         } else {
4225             # save new config if hotplug was successful
4226             $conf->{$opt} = $value;
4227             delete $conf->{pending}->{$opt};
4228             PVE::QemuConfig->write_config($vmid, $conf);
4229             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4230         }
4231     }
4232 }
4233
4234 sub try_deallocate_drive {
4235     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4236
4237     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4238         my $volid = $drive->{file};
4239         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4240             my $sid = PVE::Storage::parse_volume_id($volid);
4241             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4242
4243             # check if the disk is really unused
4244             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4245                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4246             PVE::Storage::vdisk_free($storecfg, $volid);
4247             return 1;
4248         } else {
4249             # If vm is not owner of this disk remove from config
4250             return 1;
4251         }
4252     }
4253
4254     return undef;
4255 }
4256
4257 sub vmconfig_delete_or_detach_drive {
4258     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4259
4260     my $drive = parse_drive($opt, $conf->{$opt});
4261
4262     my $rpcenv = PVE::RPCEnvironment::get();
4263     my $authuser = $rpcenv->get_user();
4264
4265     if ($force) {
4266         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4267         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4268     } else {
4269         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4270     }
4271 }
4272
4273 sub vmconfig_apply_pending {
4274     my ($vmid, $conf, $storecfg) = @_;
4275
4276     # cold plug
4277
4278     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4279     while (my ($opt, $force) = each %$pending_delete_hash) {
4280         die "internal error" if $opt =~ m/^unused/;
4281         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4282         if (!defined($conf->{$opt})) {
4283             vmconfig_undelete_pending_option($conf, $opt);
4284             PVE::QemuConfig->write_config($vmid, $conf);
4285         } elsif (is_valid_drivename($opt)) {
4286             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4287             vmconfig_undelete_pending_option($conf, $opt);
4288             delete $conf->{$opt};
4289             PVE::QemuConfig->write_config($vmid, $conf);
4290         } else {
4291             vmconfig_undelete_pending_option($conf, $opt);
4292             delete $conf->{$opt};
4293             PVE::QemuConfig->write_config($vmid, $conf);
4294         }
4295     }
4296
4297     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4298
4299     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4300         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4301
4302         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4303             # skip if nothing changed
4304         } elsif (is_valid_drivename($opt)) {
4305             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4306                 if defined($conf->{$opt});
4307             $conf->{$opt} = $conf->{pending}->{$opt};
4308         } else {
4309             $conf->{$opt} = $conf->{pending}->{$opt};
4310         }
4311
4312         delete $conf->{pending}->{$opt};
4313         PVE::QemuConfig->write_config($vmid, $conf);
4314     }
4315 }
4316
4317 my $safe_num_ne = sub {
4318     my ($a, $b) = @_;
4319
4320     return 0 if !defined($a) && !defined($b);
4321     return 1 if !defined($a);
4322     return 1 if !defined($b);
4323
4324     return $a != $b;
4325 };
4326
4327 my $safe_string_ne = sub {
4328     my ($a, $b) = @_;
4329
4330     return 0 if !defined($a) && !defined($b);
4331     return 1 if !defined($a);
4332     return 1 if !defined($b);
4333
4334     return $a ne $b;
4335 };
4336
4337 sub vmconfig_update_net {
4338     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4339
4340     my $newnet = parse_net($value);
4341
4342     if ($conf->{$opt}) {
4343         my $oldnet = parse_net($conf->{$opt});
4344
4345         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4346             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4347             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4348             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4349
4350             # for non online change, we try to hot-unplug
4351             die "skip\n" if !$hotplug;
4352             vm_deviceunplug($vmid, $conf, $opt);
4353         } else {
4354
4355             die "internal error" if $opt !~ m/net(\d+)/;
4356             my $iface = "tap${vmid}i$1";
4357
4358             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4359                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4360                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4361                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4362                 PVE::Network::tap_unplug($iface);
4363                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4364             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4365                 # Rate can be applied on its own but any change above needs to
4366                 # include the rate in tap_plug since OVS resets everything.
4367                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4368             }
4369
4370             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4371                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4372             }
4373
4374             return 1;
4375         }
4376     }
4377
4378     if ($hotplug) {
4379         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4380     } else {
4381         die "skip\n";
4382     }
4383 }
4384
4385 sub vmconfig_update_disk {
4386     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4387
4388     # fixme: do we need force?
4389
4390     my $drive = parse_drive($opt, $value);
4391
4392     if ($conf->{$opt}) {
4393
4394         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4395
4396             my $media = $drive->{media} || 'disk';
4397             my $oldmedia = $old_drive->{media} || 'disk';
4398             die "unable to change media type\n" if $media ne $oldmedia;
4399
4400             if (!drive_is_cdrom($old_drive)) {
4401
4402                 if ($drive->{file} ne $old_drive->{file}) {
4403
4404                     die "skip\n" if !$hotplug;
4405
4406                     # unplug and register as unused
4407                     vm_deviceunplug($vmid, $conf, $opt);
4408                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4409
4410                 } else {
4411                     # update existing disk
4412
4413                     # skip non hotpluggable value
4414                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4415                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4416                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4417                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4418                         die "skip\n";
4419                     }
4420
4421                     # apply throttle
4422                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4423                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4424                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4425                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4426                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4427                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4428                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4429                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4430                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4431                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4432                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4433                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4434                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4435                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4436                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4437                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4438                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4439                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4440
4441                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4442                                                    ($drive->{mbps} || 0)*1024*1024,
4443                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4444                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4445                                                    $drive->{iops} || 0,
4446                                                    $drive->{iops_rd} || 0,
4447                                                    $drive->{iops_wr} || 0,
4448                                                    ($drive->{mbps_max} || 0)*1024*1024,
4449                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4450                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4451                                                    $drive->{iops_max} || 0,
4452                                                    $drive->{iops_rd_max} || 0,
4453                                                    $drive->{iops_wr_max} || 0,
4454                                                    $drive->{bps_max_length} || 1,
4455                                                    $drive->{bps_rd_max_length} || 1,
4456                                                    $drive->{bps_wr_max_length} || 1,
4457                                                    $drive->{iops_max_length} || 1,
4458                                                    $drive->{iops_rd_max_length} || 1,
4459                                                    $drive->{iops_wr_max_length} || 1);
4460
4461                     }
4462
4463                     return 1;
4464                 }
4465
4466             } else { # cdrom
4467
4468                 if ($drive->{file} eq 'none') {
4469                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4470                 } else {
4471                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4472                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4473                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4474                 }
4475
4476                 return 1;
4477             }
4478         }
4479     }
4480
4481     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4482     # hotplug new disks
4483     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4484     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4485 }
4486
4487 sub vm_start {
4488     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4489         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4490
4491     PVE::QemuConfig->lock_config($vmid, sub {
4492         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4493
4494         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4495
4496         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4497
4498         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4499
4500         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4501             vmconfig_apply_pending($vmid, $conf, $storecfg);
4502             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4503         }
4504
4505         my $defaults = load_defaults();
4506
4507         # set environment variable useful inside network script
4508         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4509
4510         my $local_volumes = {};
4511
4512         if ($targetstorage) {
4513             foreach_drive($conf, sub {
4514                 my ($ds, $drive) = @_;
4515
4516                 return if drive_is_cdrom($drive);
4517
4518                 my $volid = $drive->{file};
4519
4520                 return if !$volid;
4521
4522                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4523
4524                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4525                 return if $scfg->{shared};
4526                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4527             });
4528
4529             my $format = undef;
4530
4531             foreach my $opt (sort keys %$local_volumes) {
4532
4533                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4534                 my $drive = parse_drive($opt, $conf->{$opt});
4535
4536                 #if remote storage is specified, use default format
4537                 if ($targetstorage && $targetstorage ne "1") {
4538                     $storeid = $targetstorage;
4539                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4540                     $format = $defFormat;
4541                 } else {
4542                     #else we use same format than original
4543                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4544                     $format = qemu_img_format($scfg, $volid);
4545                 }
4546
4547                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4548                 my $newdrive = $drive;
4549                 $newdrive->{format} = $format;
4550                 $newdrive->{file} = $newvolid;
4551                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4552                 $local_volumes->{$opt} = $drivestr;
4553                 #pass drive to conf for command line
4554                 $conf->{$opt} = $drivestr;
4555             }
4556         }
4557
4558         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4559
4560         my $migrate_port = 0;
4561         my $migrate_uri;
4562         if ($statefile) {
4563             if ($statefile eq 'tcp') {
4564                 my $localip = "localhost";
4565                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4566                 my $nodename = PVE::INotify::nodename();
4567
4568                 if (!defined($migration_type)) {
4569                     if (defined($datacenterconf->{migration}->{type})) {
4570                         $migration_type = $datacenterconf->{migration}->{type};
4571                     } else {
4572                         $migration_type = 'secure';
4573                     }
4574                 }
4575
4576                 if ($migration_type eq 'insecure') {
4577                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4578                     if ($migrate_network_addr) {
4579                         $localip = $migrate_network_addr;
4580                     } else {
4581                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4582                     }
4583
4584                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4585                 }
4586
4587                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4588                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4589                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4590                 push @$cmd, '-incoming', $migrate_uri;
4591                 push @$cmd, '-S';
4592
4593             } elsif ($statefile eq 'unix') {
4594                 # should be default for secure migrations as a ssh TCP forward
4595                 # tunnel is not deterministic reliable ready and fails regurarly
4596                 # to set up in time, so use UNIX socket forwards
4597                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4598                 unlink $socket_addr;
4599
4600                 $migrate_uri = "unix:$socket_addr";
4601
4602                 push @$cmd, '-incoming', $migrate_uri;
4603                 push @$cmd, '-S';
4604
4605             } else {
4606                 push @$cmd, '-loadstate', $statefile;
4607             }
4608         } elsif ($paused) {
4609             push @$cmd, '-S';
4610         }
4611
4612         # host pci devices
4613         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4614           my $d = parse_hostpci($conf->{"hostpci$i"});
4615           next if !$d;
4616           my $pcidevices = $d->{pciid};
4617           foreach my $pcidevice (@$pcidevices) {
4618                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4619
4620                 my $info = pci_device_info("0000:$pciid");
4621                 die "IOMMU not present\n" if !check_iommu_support();
4622                 die "no pci device info for device '$pciid'\n" if !$info;
4623                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4624                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4625           }
4626         }
4627
4628         PVE::Storage::activate_volumes($storecfg, $vollist);
4629
4630         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4631             my $cmd = [];
4632             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4633             eval  { run_command($cmd); };
4634         }
4635
4636         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4637                                                   : $defaults->{cpuunits};
4638
4639         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4640
4641         my %properties = (
4642             Slice => 'qemu.slice',
4643             KillMode => 'none',
4644             CPUShares => $cpuunits
4645         );
4646
4647         if (my $cpulimit = $conf->{cpulimit}) {
4648             $properties{CPUQuota} = int($cpulimit * 100);
4649         }
4650         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4651
4652         if ($conf->{hugepages}) {
4653
4654             my $code = sub {
4655                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4656                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4657
4658                 PVE::QemuServer::Memory::hugepages_mount();
4659                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4660
4661                 eval  {
4662                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4663                     run_command($cmd, %run_params);
4664                 };
4665
4666                 if (my $err = $@) {
4667                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4668                     die $err;
4669                 }
4670
4671                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4672             };
4673             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4674
4675         } else {
4676             eval  {
4677                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4678                 run_command($cmd, %run_params);
4679             };
4680         }
4681
4682         if (my $err = $@) {
4683             # deactivate volumes if start fails
4684             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4685             die "start failed: $err";
4686         }
4687
4688         print "migration listens on $migrate_uri\n" if $migrate_uri;
4689
4690         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4691             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4692             warn $@ if $@;
4693         }
4694
4695         #start nbd server for storage migration
4696         if ($targetstorage) {
4697             my $nodename = PVE::INotify::nodename();
4698             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4699             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4700             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4701             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4702
4703             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4704
4705             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4706
4707             foreach my $opt (sort keys %$local_volumes) {
4708                 my $volid = $local_volumes->{$opt};
4709                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4710                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4711                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4712             }
4713         }
4714
4715         if ($migratedfrom) {
4716             eval {
4717                 set_migration_caps($vmid);
4718             };
4719             warn $@ if $@;
4720
4721             if ($spice_port) {
4722                 print "spice listens on port $spice_port\n";
4723                 if ($spice_ticket) {
4724                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4725                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4726                 }
4727             }
4728
4729         } else {
4730             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4731                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4732                     if $conf->{balloon};
4733             }
4734
4735             foreach my $opt (keys %$conf) {
4736                 next if $opt !~  m/^net\d+$/;
4737                 my $nicconf = parse_net($conf->{$opt});
4738                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4739             }
4740         }
4741
4742         vm_mon_cmd_nocheck($vmid, 'qom-set',
4743                     path => "machine/peripheral/balloon0",
4744                     property => "guest-stats-polling-interval",
4745                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4746
4747     });
4748 }
4749
4750 sub vm_mon_cmd {
4751     my ($vmid, $execute, %params) = @_;
4752
4753     my $cmd = { execute => $execute, arguments => \%params };
4754     vm_qmp_command($vmid, $cmd);
4755 }
4756
4757 sub vm_mon_cmd_nocheck {
4758     my ($vmid, $execute, %params) = @_;
4759
4760     my $cmd = { execute => $execute, arguments => \%params };
4761     vm_qmp_command($vmid, $cmd, 1);
4762 }
4763
4764 sub vm_qmp_command {
4765     my ($vmid, $cmd, $nocheck) = @_;
4766
4767     my $res;
4768
4769     my $timeout;
4770     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4771         $timeout = $cmd->{arguments}->{timeout};
4772         delete $cmd->{arguments}->{timeout};
4773     }
4774
4775     eval {
4776         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4777         my $sname = qmp_socket($vmid);
4778         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4779             my $qmpclient = PVE::QMPClient->new();
4780
4781             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4782         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4783             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4784                 if scalar(%{$cmd->{arguments}});
4785             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4786         } else {
4787             die "unable to open monitor socket\n";
4788         }
4789     };
4790     if (my $err = $@) {
4791         syslog("err", "VM $vmid qmp command failed - $err");
4792         die $err;
4793     }
4794
4795     return $res;
4796 }
4797
4798 sub vm_human_monitor_command {
4799     my ($vmid, $cmdline) = @_;
4800
4801     my $res;
4802
4803     my $cmd = {
4804         execute => 'human-monitor-command',
4805         arguments => { 'command-line' => $cmdline},
4806     };
4807
4808     return vm_qmp_command($vmid, $cmd);
4809 }
4810
4811 sub vm_commandline {
4812     my ($storecfg, $vmid) = @_;
4813
4814     my $conf = PVE::QemuConfig->load_config($vmid);
4815
4816     my $defaults = load_defaults();
4817
4818     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4819
4820     return PVE::Tools::cmd2string($cmd);
4821 }
4822
4823 sub vm_reset {
4824     my ($vmid, $skiplock) = @_;
4825
4826     PVE::QemuConfig->lock_config($vmid, sub {
4827
4828         my $conf = PVE::QemuConfig->load_config($vmid);
4829
4830         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4831
4832         vm_mon_cmd($vmid, "system_reset");
4833     });
4834 }
4835
4836 sub get_vm_volumes {
4837     my ($conf) = @_;
4838
4839     my $vollist = [];
4840     foreach_volid($conf, sub {
4841         my ($volid, $attr) = @_;
4842
4843         return if $volid =~ m|^/|;
4844
4845         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4846         return if !$sid;
4847
4848         push @$vollist, $volid;
4849     });
4850
4851     return $vollist;
4852 }
4853
4854 sub vm_stop_cleanup {
4855     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4856
4857     eval {
4858
4859         if (!$keepActive) {
4860             my $vollist = get_vm_volumes($conf);
4861             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4862         }
4863
4864         foreach my $ext (qw(mon qmp pid vnc qga)) {
4865             unlink "/var/run/qemu-server/${vmid}.$ext";
4866         }
4867
4868         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4869     };
4870     warn $@ if $@; # avoid errors - just warn
4871 }
4872
4873 # Note: use $nockeck to skip tests if VM configuration file exists.
4874 # We need that when migration VMs to other nodes (files already moved)
4875 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4876 sub vm_stop {
4877     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4878
4879     $force = 1 if !defined($force) && !$shutdown;
4880
4881     if ($migratedfrom){
4882         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4883         kill 15, $pid if $pid;
4884         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4885         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4886         return;
4887     }
4888
4889     PVE::QemuConfig->lock_config($vmid, sub {
4890
4891         my $pid = check_running($vmid, $nocheck);
4892         return if !$pid;
4893
4894         my $conf;
4895         if (!$nocheck) {
4896             $conf = PVE::QemuConfig->load_config($vmid);
4897             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4898             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4899                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4900                 $timeout = $opts->{down} if $opts->{down};
4901             }
4902         }
4903
4904         $timeout = 60 if !defined($timeout);
4905
4906         eval {
4907             if ($shutdown) {
4908                 if (defined($conf) && $conf->{agent}) {
4909                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4910                 } else {
4911                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4912                 }
4913             } else {
4914                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4915             }
4916         };
4917         my $err = $@;
4918
4919         if (!$err) {
4920             my $count = 0;
4921             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4922                 $count++;
4923                 sleep 1;
4924             }
4925
4926             if ($count >= $timeout) {
4927                 if ($force) {
4928                     warn "VM still running - terminating now with SIGTERM\n";
4929                     kill 15, $pid;
4930                 } else {
4931                     die "VM quit/powerdown failed - got timeout\n";
4932                 }
4933             } else {
4934                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4935                 return;
4936             }
4937         } else {
4938             if ($force) {
4939                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4940                 kill 15, $pid;
4941             } else {
4942                 die "VM quit/powerdown failed\n";
4943             }
4944         }
4945
4946         # wait again
4947         $timeout = 10;
4948
4949         my $count = 0;
4950         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4951             $count++;
4952             sleep 1;
4953         }
4954
4955         if ($count >= $timeout) {
4956             warn "VM still running - terminating now with SIGKILL\n";
4957             kill 9, $pid;
4958             sleep 1;
4959         }
4960
4961         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4962    });
4963 }
4964
4965 sub vm_suspend {
4966     my ($vmid, $skiplock) = @_;
4967
4968     PVE::QemuConfig->lock_config($vmid, sub {
4969
4970         my $conf = PVE::QemuConfig->load_config($vmid);
4971
4972         PVE::QemuConfig->check_lock($conf)
4973             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4974
4975         vm_mon_cmd($vmid, "stop");
4976     });
4977 }
4978
4979 sub vm_resume {
4980     my ($vmid, $skiplock, $nocheck) = @_;
4981
4982     PVE::QemuConfig->lock_config($vmid, sub {
4983
4984         if (!$nocheck) {
4985
4986             my $conf = PVE::QemuConfig->load_config($vmid);
4987
4988             PVE::QemuConfig->check_lock($conf)
4989                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4990
4991             vm_mon_cmd($vmid, "cont");
4992
4993         } else {
4994             vm_mon_cmd_nocheck($vmid, "cont");
4995         }
4996     });
4997 }
4998
4999 sub vm_sendkey {
5000     my ($vmid, $skiplock, $key) = @_;
5001
5002     PVE::QemuConfig->lock_config($vmid, sub {
5003
5004         my $conf = PVE::QemuConfig->load_config($vmid);
5005
5006         # there is no qmp command, so we use the human monitor command
5007         vm_human_monitor_command($vmid, "sendkey $key");
5008     });
5009 }
5010
5011 sub vm_destroy {
5012     my ($storecfg, $vmid, $skiplock) = @_;
5013
5014     PVE::QemuConfig->lock_config($vmid, sub {
5015
5016         my $conf = PVE::QemuConfig->load_config($vmid);
5017
5018         if (!check_running($vmid)) {
5019             destroy_vm($storecfg, $vmid, undef, $skiplock);
5020         } else {
5021             die "VM $vmid is running - destroy failed\n";
5022         }
5023     });
5024 }
5025
5026 # pci helpers
5027
5028 sub file_write {
5029     my ($filename, $buf) = @_;
5030
5031     my $fh = IO::File->new($filename, "w");
5032     return undef if !$fh;
5033
5034     my $res = print $fh $buf;
5035
5036     $fh->close();
5037
5038     return $res;
5039 }
5040
5041 sub pci_device_info {
5042     my ($name) = @_;
5043
5044     my $res;
5045
5046     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5047     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5048
5049     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5050     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5051
5052     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5053     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5054
5055     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5056     return undef if !defined($product) || $product !~ s/^0x//;
5057
5058     $res = {
5059         name => $name,
5060         vendor => $vendor,
5061         product => $product,
5062         domain => $domain,
5063         bus => $bus,
5064         slot => $slot,
5065         func => $func,
5066         irq => $irq,
5067         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5068     };
5069
5070     return $res;
5071 }
5072
5073 sub pci_dev_reset {
5074     my ($dev) = @_;
5075
5076     my $name = $dev->{name};
5077
5078     my $fn = "$pcisysfs/devices/$name/reset";
5079
5080     return file_write($fn, "1");
5081 }
5082
5083 sub pci_dev_bind_to_vfio {
5084     my ($dev) = @_;
5085
5086     my $name = $dev->{name};
5087
5088     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5089
5090     if (!-d $vfio_basedir) {
5091         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5092     }
5093     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5094
5095     my $testdir = "$vfio_basedir/$name";
5096     return 1 if -d $testdir;
5097
5098     my $data = "$dev->{vendor} $dev->{product}";
5099     return undef if !file_write("$vfio_basedir/new_id", $data);
5100
5101     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5102     if (!file_write($fn, $name)) {
5103         return undef if -f $fn;
5104     }
5105
5106     $fn = "$vfio_basedir/bind";
5107     if (! -d $testdir) {
5108         return undef if !file_write($fn, $name);
5109     }
5110
5111     return -d $testdir;
5112 }
5113
5114 sub pci_dev_group_bind_to_vfio {
5115     my ($pciid) = @_;
5116
5117     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5118
5119     if (!-d $vfio_basedir) {
5120         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5121     }
5122     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5123
5124     # get IOMMU group devices
5125     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5126       my @devs = grep /^0000:/, readdir($D);
5127     closedir($D);
5128
5129     foreach my $pciid (@devs) {
5130         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5131
5132         # pci bridges, switches or root ports are not supported
5133         # they have a pci_bus subdirectory so skip them
5134         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5135
5136         my $info = pci_device_info($1);
5137         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5138     }
5139
5140     return 1;
5141 }
5142
5143 # vzdump restore implementaion
5144
5145 sub tar_archive_read_firstfile {
5146     my $archive = shift;
5147
5148     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5149
5150     # try to detect archive type first
5151     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5152         die "unable to open file '$archive'\n";
5153     my $firstfile = <$fh>;
5154     kill 15, $pid;
5155     close $fh;
5156
5157     die "ERROR: archive contaions no data\n" if !$firstfile;
5158     chomp $firstfile;
5159
5160     return $firstfile;
5161 }
5162
5163 sub tar_restore_cleanup {
5164     my ($storecfg, $statfile) = @_;
5165
5166     print STDERR "starting cleanup\n";
5167
5168     if (my $fd = IO::File->new($statfile, "r")) {
5169         while (defined(my $line = <$fd>)) {
5170             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5171                 my $volid = $2;
5172                 eval {
5173                     if ($volid =~ m|^/|) {
5174                         unlink $volid || die 'unlink failed\n';
5175                     } else {
5176                         PVE::Storage::vdisk_free($storecfg, $volid);
5177                     }
5178                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5179                 };
5180                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5181             } else {
5182                 print STDERR "unable to parse line in statfile - $line";
5183             }
5184         }
5185         $fd->close();
5186     }
5187 }
5188
5189 sub restore_archive {
5190     my ($archive, $vmid, $user, $opts) = @_;
5191
5192     my $format = $opts->{format};
5193     my $comp;
5194
5195     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5196         $format = 'tar' if !$format;
5197         $comp = 'gzip';
5198     } elsif ($archive =~ m/\.tar$/) {
5199         $format = 'tar' if !$format;
5200     } elsif ($archive =~ m/.tar.lzo$/) {
5201         $format = 'tar' if !$format;
5202         $comp = 'lzop';
5203     } elsif ($archive =~ m/\.vma$/) {
5204         $format = 'vma' if !$format;
5205     } elsif ($archive =~ m/\.vma\.gz$/) {
5206         $format = 'vma' if !$format;
5207         $comp = 'gzip';
5208     } elsif ($archive =~ m/\.vma\.lzo$/) {
5209         $format = 'vma' if !$format;
5210         $comp = 'lzop';
5211     } else {
5212         $format = 'vma' if !$format; # default
5213     }
5214
5215     # try to detect archive format
5216     if ($format eq 'tar') {
5217         return restore_tar_archive($archive, $vmid, $user, $opts);
5218     } else {
5219         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5220     }
5221 }
5222
5223 sub restore_update_config_line {
5224     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5225
5226     return if $line =~ m/^\#qmdump\#/;
5227     return if $line =~ m/^\#vzdump\#/;
5228     return if $line =~ m/^lock:/;
5229     return if $line =~ m/^unused\d+:/;
5230     return if $line =~ m/^parent:/;
5231     return if $line =~ m/^template:/; # restored VM is never a template
5232
5233     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5234     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5235         # try to convert old 1.X settings
5236         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5237         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5238             my ($model, $macaddr) = split(/\=/, $devconfig);
5239             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5240             my $net = {
5241                 model => $model,
5242                 bridge => "vmbr$ind",
5243                 macaddr => $macaddr,
5244             };
5245             my $netstr = print_net($net);
5246
5247             print $outfd "net$cookie->{netcount}: $netstr\n";
5248             $cookie->{netcount}++;
5249         }
5250     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5251         my ($id, $netstr) = ($1, $2);
5252         my $net = parse_net($netstr);
5253         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5254         $netstr = print_net($net);
5255         print $outfd "$id: $netstr\n";
5256     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5257         my $virtdev = $1;
5258         my $value = $3;
5259         my $di = parse_drive($virtdev, $value);
5260         if (defined($di->{backup}) && !$di->{backup}) {
5261             print $outfd "#$line";
5262         } elsif ($map->{$virtdev}) {
5263             delete $di->{format}; # format can change on restore
5264             $di->{file} = $map->{$virtdev};
5265             $value = print_drive($vmid, $di);
5266             print $outfd "$virtdev: $value\n";
5267         } else {
5268             print $outfd $line;
5269         }
5270     } else {
5271         print $outfd $line;
5272     }
5273 }
5274
5275 sub scan_volids {
5276     my ($cfg, $vmid) = @_;
5277
5278     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5279
5280     my $volid_hash = {};
5281     foreach my $storeid (keys %$info) {
5282         foreach my $item (@{$info->{$storeid}}) {
5283             next if !($item->{volid} && $item->{size});
5284             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5285             $volid_hash->{$item->{volid}} = $item;
5286         }
5287     }
5288
5289     return $volid_hash;
5290 }
5291
5292 sub is_volume_in_use {
5293     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5294
5295     my $path = PVE::Storage::path($storecfg, $volid);
5296
5297     my $scan_config = sub {
5298         my ($cref, $snapname) = @_;
5299
5300         foreach my $key (keys %$cref) {
5301             my $value = $cref->{$key};
5302             if (is_valid_drivename($key)) {
5303                 next if $skip_drive && $key eq $skip_drive;
5304                 my $drive = parse_drive($key, $value);
5305                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5306                 return 1 if $volid eq $drive->{file};
5307                 if ($drive->{file} =~ m!^/!) {
5308                     return 1 if $drive->{file} eq $path;
5309                 } else {
5310                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5311                     next if !$storeid;
5312                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5313                     next if !$scfg;
5314                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5315                 }
5316             }
5317         }
5318
5319         return 0;
5320     };
5321
5322     return 1 if &$scan_config($conf);
5323
5324     undef $skip_drive;
5325
5326     foreach my $snapname (keys %{$conf->{snapshots}}) {
5327         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5328     }
5329
5330     return 0;
5331 }
5332
5333 sub update_disksize {
5334     my ($vmid, $conf, $volid_hash) = @_;
5335
5336     my $changes;
5337
5338     my $used = {};
5339
5340     # Note: it is allowed to define multiple storages with same path (alias), so
5341     # we need to check both 'volid' and real 'path' (two different volid can point
5342     # to the same path).
5343
5344     my $usedpath = {};
5345
5346     # update size info
5347     foreach my $opt (keys %$conf) {
5348         if (is_valid_drivename($opt)) {
5349             my $drive = parse_drive($opt, $conf->{$opt});
5350             my $volid = $drive->{file};
5351             next if !$volid;
5352
5353             $used->{$volid} = 1;
5354             if ($volid_hash->{$volid} &&
5355                 (my $path = $volid_hash->{$volid}->{path})) {
5356                 $usedpath->{$path} = 1;
5357             }
5358
5359             next if drive_is_cdrom($drive);
5360             next if !$volid_hash->{$volid};
5361
5362             $drive->{size} = $volid_hash->{$volid}->{size};
5363             my $new = print_drive($vmid, $drive);
5364             if ($new ne $conf->{$opt}) {
5365                 $changes = 1;
5366                 $conf->{$opt} = $new;
5367             }
5368         }
5369     }
5370
5371     # remove 'unusedX' entry if volume is used
5372     foreach my $opt (keys %$conf) {
5373         next if $opt !~ m/^unused\d+$/;
5374         my $volid = $conf->{$opt};
5375         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5376         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5377             $changes = 1;
5378             delete $conf->{$opt};
5379         }
5380     }
5381
5382     foreach my $volid (sort keys %$volid_hash) {
5383         next if $volid =~ m/vm-$vmid-state-/;
5384         next if $used->{$volid};
5385         my $path = $volid_hash->{$volid}->{path};
5386         next if !$path; # just to be sure
5387         next if $usedpath->{$path};
5388         $changes = 1;
5389         PVE::QemuConfig->add_unused_volume($conf, $volid);
5390         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5391     }
5392
5393     return $changes;
5394 }
5395
5396 sub rescan {
5397     my ($vmid, $nolock) = @_;
5398
5399     my $cfg = PVE::Storage::config();
5400
5401     my $volid_hash = scan_volids($cfg, $vmid);
5402
5403     my $updatefn =  sub {
5404         my ($vmid) = @_;
5405
5406         my $conf = PVE::QemuConfig->load_config($vmid);
5407
5408         PVE::QemuConfig->check_lock($conf);
5409
5410         my $vm_volids = {};
5411         foreach my $volid (keys %$volid_hash) {
5412             my $info = $volid_hash->{$volid};
5413             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5414         }
5415
5416         my $changes = update_disksize($vmid, $conf, $vm_volids);
5417
5418         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5419     };
5420
5421     if (defined($vmid)) {
5422         if ($nolock) {
5423             &$updatefn($vmid);
5424         } else {
5425             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5426         }
5427     } else {
5428         my $vmlist = config_list();
5429         foreach my $vmid (keys %$vmlist) {
5430             if ($nolock) {
5431                 &$updatefn($vmid);
5432             } else {
5433                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5434             }
5435         }
5436     }
5437 }
5438
5439 sub restore_vma_archive {
5440     my ($archive, $vmid, $user, $opts, $comp) = @_;
5441
5442     my $input = $archive eq '-' ? "<&STDIN" : undef;
5443     my $readfrom = $archive;
5444
5445     my $uncomp = '';
5446     if ($comp) {
5447         $readfrom = '-';
5448         my $qarchive = PVE::Tools::shellquote($archive);
5449         if ($comp eq 'gzip') {
5450             $uncomp = "zcat $qarchive|";
5451         } elsif ($comp eq 'lzop') {
5452             $uncomp = "lzop -d -c $qarchive|";
5453         } else {
5454             die "unknown compression method '$comp'\n";
5455         }
5456
5457     }
5458
5459     my $tmpdir = "/var/tmp/vzdumptmp$$";
5460     rmtree $tmpdir;
5461
5462     # disable interrupts (always do cleanups)
5463     local $SIG{INT} =
5464         local $SIG{TERM} =
5465         local $SIG{QUIT} =
5466         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5467
5468     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5469     POSIX::mkfifo($mapfifo, 0600);
5470     my $fifofh;
5471
5472     my $openfifo = sub {
5473         open($fifofh, '>', $mapfifo) || die $!;
5474     };
5475
5476     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5477
5478     my $oldtimeout;
5479     my $timeout = 5;
5480
5481     my $devinfo = {};
5482
5483     my $rpcenv = PVE::RPCEnvironment::get();
5484
5485     my $conffile = PVE::QemuConfig->config_file($vmid);
5486     my $tmpfn = "$conffile.$$.tmp";
5487
5488     # Note: $oldconf is undef if VM does not exists
5489     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5490     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5491
5492     my $print_devmap = sub {
5493         my $virtdev_hash = {};
5494
5495         my $cfgfn = "$tmpdir/qemu-server.conf";
5496
5497         # we can read the config - that is already extracted
5498         my $fh = IO::File->new($cfgfn, "r") ||
5499             "unable to read qemu-server.conf - $!\n";
5500
5501         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5502         if (-f $fwcfgfn) {
5503             my $pve_firewall_dir = '/etc/pve/firewall';
5504             mkdir $pve_firewall_dir; # make sure the dir exists
5505             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5506         }
5507
5508         while (defined(my $line = <$fh>)) {
5509             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5510                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5511                 die "archive does not contain data for drive '$virtdev'\n"
5512                     if !$devinfo->{$devname};
5513                 if (defined($opts->{storage})) {
5514                     $storeid = $opts->{storage} || 'local';
5515                 } elsif (!$storeid) {
5516                     $storeid = 'local';
5517                 }
5518                 $format = 'raw' if !$format;
5519                 $devinfo->{$devname}->{devname} = $devname;
5520                 $devinfo->{$devname}->{virtdev} = $virtdev;
5521                 $devinfo->{$devname}->{format} = $format;
5522                 $devinfo->{$devname}->{storeid} = $storeid;
5523
5524                 # check permission on storage
5525                 my $pool = $opts->{pool}; # todo: do we need that?
5526                 if ($user ne 'root@pam') {
5527                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5528                 }
5529
5530                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5531             }
5532         }
5533
5534         foreach my $devname (keys %$devinfo) {
5535             die "found no device mapping information for device '$devname'\n"
5536                 if !$devinfo->{$devname}->{virtdev};
5537         }
5538
5539         my $cfg = PVE::Storage::config();
5540
5541         # create empty/temp config
5542         if ($oldconf) {
5543             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5544             foreach_drive($oldconf, sub {
5545                 my ($ds, $drive) = @_;
5546
5547                 return if drive_is_cdrom($drive);
5548
5549                 my $volid = $drive->{file};
5550
5551                 return if !$volid || $volid =~ m|^/|;
5552
5553                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5554                 return if !$path || !$owner || ($owner != $vmid);
5555
5556                 # Note: only delete disk we want to restore
5557                 # other volumes will become unused
5558                 if ($virtdev_hash->{$ds}) {
5559                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5560                     if (my $err = $@) {
5561                         warn $err;
5562                     }
5563                 }
5564             });
5565
5566             # delete vmstate files
5567             # since after the restore we have no snapshots anymore
5568             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5569                 my $snap = $oldconf->{snapshots}->{$snapname};
5570                 if ($snap->{vmstate}) {
5571                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5572                     if (my $err = $@) {
5573                         warn $err;
5574                     }
5575                 }
5576             }
5577         }
5578
5579         my $map = {};
5580         foreach my $virtdev (sort keys %$virtdev_hash) {
5581             my $d = $virtdev_hash->{$virtdev};
5582             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5583             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5584
5585             # test if requested format is supported
5586             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5587             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5588             $d->{format} = $defFormat if !$supported;
5589
5590             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5591                                                   $d->{format}, undef, $alloc_size);
5592             print STDERR "new volume ID is '$volid'\n";
5593             $d->{volid} = $volid;
5594             my $path = PVE::Storage::path($cfg, $volid);
5595
5596             PVE::Storage::activate_volumes($cfg,[$volid]);
5597
5598             my $write_zeros = 1;
5599             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5600                 $write_zeros = 0;
5601             }
5602
5603             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5604
5605             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5606             $map->{$virtdev} = $volid;
5607         }
5608
5609         $fh->seek(0, 0) || die "seek failed - $!\n";
5610
5611         my $outfd = new IO::File ($tmpfn, "w") ||
5612             die "unable to write config for VM $vmid\n";
5613
5614         my $cookie = { netcount => 0 };
5615         while (defined(my $line = <$fh>)) {
5616             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5617         }
5618
5619         $fh->close();
5620         $outfd->close();
5621     };
5622
5623     eval {
5624         # enable interrupts
5625         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5626             die "interrupted by signal\n";
5627         };
5628         local $SIG{ALRM} = sub { die "got timeout\n"; };
5629
5630         $oldtimeout = alarm($timeout);
5631
5632         my $parser = sub {
5633             my $line = shift;
5634
5635             print "$line\n";
5636
5637             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5638                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5639                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5640             } elsif ($line =~ m/^CTIME: /) {
5641                 # we correctly received the vma config, so we can disable
5642                 # the timeout now for disk allocation (set to 10 minutes, so
5643                 # that we always timeout if something goes wrong)
5644                 alarm(600);
5645                 &$print_devmap();
5646                 print $fifofh "done\n";
5647                 my $tmp = $oldtimeout || 0;
5648                 $oldtimeout = undef;
5649                 alarm($tmp);
5650                 close($fifofh);
5651             }
5652         };
5653
5654         print "restore vma archive: $cmd\n";
5655         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5656     };
5657     my $err = $@;
5658
5659     alarm($oldtimeout) if $oldtimeout;
5660
5661     my $vollist = [];
5662     foreach my $devname (keys %$devinfo) {
5663         my $volid = $devinfo->{$devname}->{volid};
5664         push @$vollist, $volid if $volid;
5665     }
5666
5667     my $cfg = PVE::Storage::config();
5668     PVE::Storage::deactivate_volumes($cfg, $vollist);
5669
5670     unlink $mapfifo;
5671
5672     if ($err) {
5673         rmtree $tmpdir;
5674         unlink $tmpfn;
5675
5676         foreach my $devname (keys %$devinfo) {
5677             my $volid = $devinfo->{$devname}->{volid};
5678             next if !$volid;
5679             eval {
5680                 if ($volid =~ m|^/|) {
5681                     unlink $volid || die 'unlink failed\n';
5682                 } else {
5683                     PVE::Storage::vdisk_free($cfg, $volid);
5684                 }
5685                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5686             };
5687             print STDERR "unable to cleanup '$volid' - $@" if $@;
5688         }
5689         die $err;
5690     }
5691
5692     rmtree $tmpdir;
5693
5694     rename($tmpfn, $conffile) ||
5695         die "unable to commit configuration file '$conffile'\n";
5696
5697     PVE::Cluster::cfs_update(); # make sure we read new file
5698
5699     eval { rescan($vmid, 1); };
5700     warn $@ if $@;
5701 }
5702
5703 sub restore_tar_archive {
5704     my ($archive, $vmid, $user, $opts) = @_;
5705
5706     if ($archive ne '-') {
5707         my $firstfile = tar_archive_read_firstfile($archive);
5708         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5709             if $firstfile ne 'qemu-server.conf';
5710     }
5711
5712     my $storecfg = PVE::Storage::config();
5713
5714     # destroy existing data - keep empty config
5715     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5716     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5717
5718     my $tocmd = "/usr/lib/qemu-server/qmextract";
5719
5720     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5721     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5722     $tocmd .= ' --prealloc' if $opts->{prealloc};
5723     $tocmd .= ' --info' if $opts->{info};
5724
5725     # tar option "xf" does not autodetect compression when read from STDIN,
5726     # so we pipe to zcat
5727     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5728         PVE::Tools::shellquote("--to-command=$tocmd");
5729
5730     my $tmpdir = "/var/tmp/vzdumptmp$$";
5731     mkpath $tmpdir;
5732
5733     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5734     local $ENV{VZDUMP_VMID} = $vmid;
5735     local $ENV{VZDUMP_USER} = $user;
5736
5737     my $conffile = PVE::QemuConfig->config_file($vmid);
5738     my $tmpfn = "$conffile.$$.tmp";
5739
5740     # disable interrupts (always do cleanups)
5741     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5742         print STDERR "got interrupt - ignored\n";
5743     };
5744
5745     eval {
5746         # enable interrupts
5747         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5748             die "interrupted by signal\n";
5749         };
5750
5751         if ($archive eq '-') {
5752             print "extracting archive from STDIN\n";
5753             run_command($cmd, input => "<&STDIN");
5754         } else {
5755             print "extracting archive '$archive'\n";
5756             run_command($cmd);
5757         }
5758
5759         return if $opts->{info};
5760
5761         # read new mapping
5762         my $map = {};
5763         my $statfile = "$tmpdir/qmrestore.stat";
5764         if (my $fd = IO::File->new($statfile, "r")) {
5765             while (defined (my $line = <$fd>)) {
5766                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5767                     $map->{$1} = $2 if $1;
5768                 } else {
5769                     print STDERR "unable to parse line in statfile - $line\n";
5770                 }
5771             }
5772             $fd->close();
5773         }
5774
5775         my $confsrc = "$tmpdir/qemu-server.conf";
5776
5777         my $srcfd = new IO::File($confsrc, "r") ||
5778             die "unable to open file '$confsrc'\n";
5779
5780         my $outfd = new IO::File ($tmpfn, "w") ||
5781             die "unable to write config for VM $vmid\n";
5782
5783         my $cookie = { netcount => 0 };
5784         while (defined (my $line = <$srcfd>)) {
5785             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5786         }
5787
5788         $srcfd->close();
5789         $outfd->close();
5790     };
5791     my $err = $@;
5792
5793     if ($err) {
5794
5795         unlink $tmpfn;
5796
5797         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5798
5799         die $err;
5800     }
5801
5802     rmtree $tmpdir;
5803
5804     rename $tmpfn, $conffile ||
5805         die "unable to commit configuration file '$conffile'\n";
5806
5807     PVE::Cluster::cfs_update(); # make sure we read new file
5808
5809     eval { rescan($vmid, 1); };
5810     warn $@ if $@;
5811 };
5812
5813 sub foreach_writable_storage {
5814     my ($conf, $func) = @_;
5815
5816     my $sidhash = {};
5817
5818     foreach my $ds (keys %$conf) {
5819         next if !is_valid_drivename($ds);
5820
5821         my $drive = parse_drive($ds, $conf->{$ds});
5822         next if !$drive;
5823         next if drive_is_cdrom($drive);
5824
5825         my $volid = $drive->{file};
5826
5827         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5828         $sidhash->{$sid} = $sid if $sid;
5829     }
5830
5831     foreach my $sid (sort keys %$sidhash) {
5832         &$func($sid);
5833     }
5834 }
5835
5836 sub do_snapshots_with_qemu {
5837     my ($storecfg, $volid) = @_;
5838
5839     my $storage_name = PVE::Storage::parse_volume_id($volid);
5840
5841     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5842         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5843         return 1;
5844     }
5845
5846     if ($volid =~ m/\.(qcow2|qed)$/){
5847         return 1;
5848     }
5849
5850     return undef;
5851 }
5852
5853 sub qga_check_running {
5854     my ($vmid) = @_;
5855
5856     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5857     if ($@) {
5858         warn "Qemu Guest Agent are not running - $@";
5859         return 0;
5860     }
5861     return 1;
5862 }
5863
5864 sub template_create {
5865     my ($vmid, $conf, $disk) = @_;
5866
5867     my $storecfg = PVE::Storage::config();
5868
5869     foreach_drive($conf, sub {
5870         my ($ds, $drive) = @_;
5871
5872         return if drive_is_cdrom($drive);
5873         return if $disk && $ds ne $disk;
5874
5875         my $volid = $drive->{file};
5876         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5877
5878         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5879         $drive->{file} = $voliddst;
5880         $conf->{$ds} = print_drive($vmid, $drive);
5881         PVE::QemuConfig->write_config($vmid, $conf);
5882     });
5883 }
5884
5885 sub qemu_img_convert {
5886     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5887
5888     my $storecfg = PVE::Storage::config();
5889     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5890     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5891
5892     if ($src_storeid && $dst_storeid) {
5893
5894         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5895
5896         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5897         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5898
5899         my $src_format = qemu_img_format($src_scfg, $src_volname);
5900         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5901
5902         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5903         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5904
5905         my $cmd = [];
5906         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5907         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5908         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5909         if ($is_zero_initialized) {
5910             push @$cmd, "zeroinit:$dst_path";
5911         } else {
5912             push @$cmd, $dst_path;
5913         }
5914
5915         my $parser = sub {
5916             my $line = shift;
5917             if($line =~ m/\((\S+)\/100\%\)/){
5918                 my $percent = $1;
5919                 my $transferred = int($size * $percent / 100);
5920                 my $remaining = $size - $transferred;
5921
5922                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5923             }
5924
5925         };
5926
5927         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5928         my $err = $@;
5929         die "copy failed: $err" if $err;
5930     }
5931 }
5932
5933 sub qemu_img_format {
5934     my ($scfg, $volname) = @_;
5935
5936     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5937         return $1;
5938     } else {
5939         return "raw";
5940     }
5941 }
5942
5943 sub qemu_drive_mirror {
5944     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5945
5946     $jobs = {} if !$jobs;
5947
5948     my $qemu_target;
5949     my $format;
5950     $jobs->{"drive-$drive"} = {};
5951
5952     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5953         my $server = $1;
5954         my $port = $2;
5955         my $exportname = $3;
5956
5957         $format = "nbd";
5958         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5959         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5960         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5961
5962         my $pid = fork();
5963         if (!defined($pid)) {
5964             die "forking socat tunnel failed\n";
5965         } elsif ($pid == 0) {
5966             exec(@$cmd);
5967             warn "exec failed: $!\n";
5968             POSIX::_exit(-1);
5969         }
5970         $jobs->{"drive-$drive"}->{pid} = $pid;
5971
5972         my $timeout = 0;
5973         while (!-S $unixsocket) {
5974             die "nbd connection helper timed out\n"
5975                 if $timeout++ > 5;
5976             sleep 1;
5977         }
5978     } else {
5979         my $storecfg = PVE::Storage::config();
5980         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5981
5982         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5983
5984         $format = qemu_img_format($dst_scfg, $dst_volname);
5985
5986         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5987
5988         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5989     }
5990
5991     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5992     $opts->{format} = $format if $format;
5993
5994     print "drive mirror is starting for drive-$drive\n";
5995
5996     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
5997
5998     if (my $err = $@) {
5999         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6000         die "mirroring error: $err";
6001     }
6002
6003     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6004 }
6005
6006 sub qemu_drive_mirror_monitor {
6007     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6008
6009     eval {
6010         my $err_complete = 0;
6011
6012         while (1) {
6013             die "storage migration timed out\n" if $err_complete > 300;
6014
6015             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6016
6017             my $running_mirror_jobs = {};
6018             foreach my $stat (@$stats) {
6019                 next if $stat->{type} ne 'mirror';
6020                 $running_mirror_jobs->{$stat->{device}} = $stat;
6021             }
6022
6023             my $readycounter = 0;
6024
6025             foreach my $job (keys %$jobs) {
6026
6027                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6028                     print "$job : finished\n";
6029                     delete $jobs->{$job};
6030                     next;
6031                 }
6032
6033                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6034
6035                 my $busy = $running_mirror_jobs->{$job}->{busy};
6036                 my $ready = $running_mirror_jobs->{$job}->{ready};
6037                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6038                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6039                     my $remaining = $total - $transferred;
6040                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6041
6042                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6043                 }
6044
6045                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6046             }
6047
6048             last if scalar(keys %$jobs) == 0;
6049
6050             if ($readycounter == scalar(keys %$jobs)) {
6051                 print "all mirroring jobs are ready \n";
6052                 last if $skipcomplete; #do the complete later
6053
6054                 if ($vmiddst && $vmiddst != $vmid) {
6055                     if ($qga) {
6056                         print "freeze filesystem\n";
6057                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6058                     } else {
6059                         print "suspend vm\n";
6060                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6061                     }
6062
6063                     # if we clone a disk for a new target vm, we don't switch the disk
6064                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6065
6066                     if ($qga) {
6067                         print "unfreeze filesystem\n";
6068                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6069                     } else {
6070                         print "resume vm\n";
6071                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6072                     }
6073
6074                     last;
6075                 } else {
6076
6077                     foreach my $job (keys %$jobs) {
6078                         # try to switch the disk if source and destination are on the same guest
6079                         print "$job: Completing block job...\n";
6080
6081                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6082                         if ($@ =~ m/cannot be completed/) {
6083                             print "$job: Block job cannot be completed, try again.\n";
6084                             $err_complete++;
6085                         }else {
6086                             print "$job: Completed successfully.\n";
6087                             $jobs->{$job}->{complete} = 1;
6088                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6089                         }
6090                     }
6091                 }
6092             }
6093             sleep 1;
6094         }
6095     };
6096     my $err = $@;
6097
6098     if ($err) {
6099         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6100         die "mirroring error: $err";
6101     }
6102
6103 }
6104
6105 sub qemu_blockjobs_cancel {
6106     my ($vmid, $jobs) = @_;
6107
6108     foreach my $job (keys %$jobs) {
6109         print "$job: Cancelling block job\n";
6110         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6111         $jobs->{$job}->{cancel} = 1;
6112     }
6113
6114     while (1) {
6115         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6116
6117         my $running_jobs = {};
6118         foreach my $stat (@$stats) {
6119             $running_jobs->{$stat->{device}} = $stat;
6120         }
6121
6122         foreach my $job (keys %$jobs) {
6123
6124             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6125                 print "$job: Done.\n";
6126                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6127                 delete $jobs->{$job};
6128             }
6129         }
6130
6131         last if scalar(keys %$jobs) == 0;
6132
6133         sleep 1;
6134     }
6135 }
6136
6137 sub qemu_blockjobs_finish_tunnel {
6138    my ($vmid, $job, $cpid) = @_;
6139
6140    return if !$cpid;
6141
6142    for (my $i = 1; $i < 20; $i++) {
6143         my $waitpid = waitpid($cpid, WNOHANG);
6144         last if (defined($waitpid) && ($waitpid == $cpid));
6145
6146         if ($i == 10) {
6147             kill(15, $cpid);
6148         } elsif ($i >= 15) {
6149             kill(9, $cpid);
6150         }
6151         sleep (1);
6152     }
6153     unlink "/run/qemu-server/$vmid.mirror-$job";
6154 }
6155
6156 sub clone_disk {
6157     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6158         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6159
6160     my $newvolid;
6161
6162     if (!$full) {
6163         print "create linked clone of drive $drivename ($drive->{file})\n";
6164         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6165         push @$newvollist, $newvolid;
6166     } else {
6167
6168         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6169         $storeid = $storage if $storage;
6170
6171         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6172         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6173
6174         print "create full clone of drive $drivename ($drive->{file})\n";
6175         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6176         push @$newvollist, $newvolid;
6177
6178         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6179
6180         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6181         if (!$running || $snapname) {
6182             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6183         } else {
6184
6185             my $kvmver = get_running_qemu_version ($vmid);
6186             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6187                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6188                     if $drive->{iothread};
6189             }
6190
6191             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6192         }
6193     }
6194
6195     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6196
6197     my $disk = $drive;
6198     $disk->{format} = undef;
6199     $disk->{file} = $newvolid;
6200     $disk->{size} = $size;
6201
6202     return $disk;
6203 }
6204
6205 # this only works if VM is running
6206 sub get_current_qemu_machine {
6207     my ($vmid) = @_;
6208
6209     my $cmd = { execute => 'query-machines', arguments => {} };
6210     my $res = vm_qmp_command($vmid, $cmd);
6211
6212     my ($current, $default);
6213     foreach my $e (@$res) {
6214         $default = $e->{name} if $e->{'is-default'};
6215         $current = $e->{name} if $e->{'is-current'};
6216     }
6217
6218     # fallback to the default machine if current is not supported by qemu
6219     return $current || $default || 'pc';
6220 }
6221
6222 sub get_running_qemu_version {
6223     my ($vmid) = @_;
6224     my $cmd = { execute => 'query-version', arguments => {} };
6225     my $res = vm_qmp_command($vmid, $cmd);
6226     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6227 }
6228
6229 sub qemu_machine_feature_enabled {
6230     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6231
6232     my $current_major;
6233     my $current_minor;
6234
6235     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6236
6237         $current_major = $3;
6238         $current_minor = $4;
6239
6240     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6241
6242         $current_major = $1;
6243         $current_minor = $2;
6244     }
6245
6246     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6247
6248
6249 }
6250
6251 sub qemu_machine_pxe {
6252     my ($vmid, $conf, $machine) = @_;
6253
6254     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6255
6256     foreach my $opt (keys %$conf) {
6257         next if $opt !~ m/^net(\d+)$/;
6258         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6259         next if !$net;
6260         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6261         return $machine.".pxe" if $romfile =~ m/pxe/;
6262         last;
6263     }
6264
6265     return $machine;
6266 }
6267
6268 sub qemu_use_old_bios_files {
6269     my ($machine_type) = @_;
6270
6271     return if !$machine_type;
6272
6273     my $use_old_bios_files = undef;
6274
6275     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6276         $machine_type = $1;
6277         $use_old_bios_files = 1;
6278     } else {
6279         my $kvmver = kvm_user_version();
6280         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6281         # load new efi bios files on migration. So this hack is required to allow
6282         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6283         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6284         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6285     }
6286
6287     return ($use_old_bios_files, $machine_type);
6288 }
6289
6290 sub create_efidisk {
6291     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6292
6293     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6294
6295     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6296     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6297     PVE::Storage::activate_volumes($storecfg, [$volid]);
6298
6299     my $path = PVE::Storage::path($storecfg, $volid);
6300     eval {
6301         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6302     };
6303     die "Copying EFI vars image failed: $@" if $@;
6304
6305     return ($volid, $vars_size);
6306 }
6307
6308 sub lspci {
6309
6310     my $devices = {};
6311
6312     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6313             my (undef, $id, $function) = @_;
6314             my $res = { id => $id, function => $function};
6315             push @{$devices->{$id}}, $res;
6316     });
6317
6318     # Entries should be sorted by functions.
6319     foreach my $id (keys %$devices) {
6320         my $dev = $devices->{$id};
6321         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6322     }
6323
6324     return $devices;
6325 }
6326
6327 sub vm_iothreads_list {
6328     my ($vmid) = @_;
6329
6330     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6331
6332     my $iothreads = {};
6333     foreach my $iothread (@$res) {
6334         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6335     }
6336
6337     return $iothreads;
6338 }
6339
6340 sub scsihw_infos {
6341     my ($conf, $drive) = @_;
6342
6343     my $maxdev = 0;
6344
6345     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6346         $maxdev = 7;
6347     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6348         $maxdev = 1;
6349     } else {
6350         $maxdev = 256;
6351     }
6352
6353     my $controller = int($drive->{index} / $maxdev);
6354     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6355
6356     return ($maxdev, $controller, $controller_prefix);
6357 }
6358
6359 sub add_hyperv_enlightenments {
6360     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6361
6362     return if $winversion < 6;
6363     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6364
6365     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6366
6367     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6368         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6369         push @$cpuFlags , 'hv_vapic';
6370         push @$cpuFlags , 'hv_time';
6371     } else {
6372         push @$cpuFlags , 'hv_spinlocks=0xffff';
6373     }
6374
6375     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6376         push @$cpuFlags , 'hv_reset';
6377         push @$cpuFlags , 'hv_vpindex';
6378         push @$cpuFlags , 'hv_runtime';
6379     }
6380
6381     if ($winversion >= 7) {
6382         push @$cpuFlags , 'hv_relaxed';
6383     }
6384 }
6385
6386 sub windows_version {
6387     my ($ostype) = @_;
6388
6389     return 0 if !$ostype;
6390
6391     my $winversion = 0;
6392
6393     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6394         $winversion = 5;
6395     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6396         $winversion = 6;
6397     } elsif ($ostype =~ m/^win(\d+)$/) {
6398         $winversion = $1;
6399     }
6400
6401     return $winversion;
6402 }
6403
6404 sub resolve_dst_disk_format {
6405         my ($storecfg, $storeid, $src_volname, $format) = @_;
6406         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6407
6408         if (!$format) {
6409             # if no target format is specified, use the source disk format as hint
6410             if ($src_volname) {
6411                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6412                 $format = qemu_img_format($scfg, $src_volname);
6413             } else {
6414                 return $defFormat;
6415             }
6416         }
6417
6418         # test if requested format is supported - else use default
6419         my $supported = grep { $_ eq $format } @$validFormats;
6420         $format = $defFormat if !$supported;
6421         return $format;
6422 }
6423
6424 # bash completion helper
6425
6426 sub complete_backup_archives {
6427     my ($cmdname, $pname, $cvalue) = @_;
6428
6429     my $cfg = PVE::Storage::config();
6430
6431     my $storeid;
6432
6433     if ($cvalue =~ m/^([^:]+):/) {
6434         $storeid = $1;
6435     }
6436
6437     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6438
6439     my $res = [];
6440     foreach my $id (keys %$data) {
6441         foreach my $item (@{$data->{$id}}) {
6442             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6443             push @$res, $item->{volid} if defined($item->{volid});
6444         }
6445     }
6446
6447     return $res;
6448 }
6449
6450 my $complete_vmid_full = sub {
6451     my ($running) = @_;
6452
6453     my $idlist = vmstatus();
6454
6455     my $res = [];
6456
6457     foreach my $id (keys %$idlist) {
6458         my $d = $idlist->{$id};
6459         if (defined($running)) {
6460             next if $d->{template};
6461             next if $running && $d->{status} ne 'running';
6462             next if !$running && $d->{status} eq 'running';
6463         }
6464         push @$res, $id;
6465
6466     }
6467     return $res;
6468 };
6469
6470 sub complete_vmid {
6471     return &$complete_vmid_full();
6472 }
6473
6474 sub complete_vmid_stopped {
6475     return &$complete_vmid_full(0);
6476 }
6477
6478 sub complete_vmid_running {
6479     return &$complete_vmid_full(1);
6480 }
6481
6482 sub complete_storage {
6483
6484     my $cfg = PVE::Storage::config();
6485     my $ids = $cfg->{ids};
6486
6487     my $res = [];
6488     foreach my $sid (keys %$ids) {
6489         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6490         next if !$ids->{$sid}->{content}->{images};
6491         push @$res, $sid;
6492     }
6493
6494     return $res;
6495 }
6496
6497 sub nbd_stop {
6498     my ($vmid) = @_;
6499
6500     vm_mon_cmd($vmid, 'nbd-server-stop');
6501 }
6502
6503 1;