PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120     'Skylake-Client' => 'GenuineIntel',
 121
 122     # AMD CPUs
 123     athlon => 'AuthenticAMD',
 124     phenom  => 'AuthenticAMD',
 125     Opteron_G1  => 'AuthenticAMD',
 126     Opteron_G2  => 'AuthenticAMD',
 127     Opteron_G3  => 'AuthenticAMD',
 128     Opteron_G4  => 'AuthenticAMD',
 129     Opteron_G5  => 'AuthenticAMD',
 130
 131     # generic types, use vendor from host node
 132     host => 'default',
 133     kvm32 => 'default',
 134     kvm64 => 'default',
 135     qemu32 => 'default',
 136     qemu64 => 'default',
 137 };
 138
 139 my $cpu_fmt = {
 140     cputype => {
 141         description => "Emulated CPU type.",
 142         type => 'string',
 143         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 218         minimum => 0,
 219         maximum => 500000,
 220         default => 1024,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 247         enum => PVE::Tools::kvmkeymaplist(),
 248         default => 'en-us',
 249     },
 250     name => {
 251         optional => 1,
 252         type => 'string', format => 'dns-name',
 253         description => "Set a name for the VM. Only used on the configuration web interface.",
 254     },
 255     scsihw => {
 256         optional => 1,
 257         type => 'string',
 258         description => "SCSI controller model",
 259         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 260         default => 'lsi',
 261     },
 262     description => {
 263         optional => 1,
 264         type => 'string',
 265         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 266     },
 267     ostype => {
 268         optional => 1,
 269         type => 'string',
 270         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 271         description => "Specify guest operating system.",
 272         verbose_description => <<EODESC,
 273 Specify guest operating system. This is used to enable special
 274 optimization/features for specific operating systems:
 275
 276 [horizontal]
 277 other;; unspecified OS
 278 wxp;; Microsoft Windows XP
 279 w2k;; Microsoft Windows 2000
 280 w2k3;; Microsoft Windows 2003
 281 w2k8;; Microsoft Windows 2008
 282 wvista;; Microsoft Windows Vista
 283 win7;; Microsoft Windows 7
 284 win8;; Microsoft Windows 8/2012
 285 l24;; Linux 2.4 Kernel
 286 l26;; Linux 2.6/3.X Kernel
 287 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 288 EODESC
 289     },
 290     boot => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 294         pattern => '[acdn]{1,4}',
 295         default => 'cdn',
 296     },
 297     bootdisk => {
 298         optional => 1,
 299         type => 'string', format => 'pve-qm-bootdisk',
 300         description => "Enable booting from specified disk.",
 301         pattern => '(ide|sata|scsi|virtio)\d+',
 302     },
 303     smp => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "The number of CPUs. Please use option -sockets instead.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     sockets => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "The number of CPU sockets.",
 314         minimum => 1,
 315         default => 1,
 316     },
 317     cores => {
 318         optional => 1,
 319         type => 'integer',
 320         description => "The number of cores per socket.",
 321         minimum => 1,
 322         default => 1,
 323     },
 324     numa => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Enable/disable NUMA.",
 328         default => 0,
 329     },
 330     hugepages => {
 331         optional => 1,
 332         type => 'string',
 333         description => "Enable/disable hugepages memory.",
 334         enum => [qw(any 2 1024)],
 335     },
 336     vcpus => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Number of hotplugged vcpus.",
 340         minimum => 1,
 341         default => 0,
 342     },
 343     acpi => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable ACPI.",
 347         default => 1,
 348     },
 349     agent => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable Qemu GuestAgent.",
 353         default => 0,
 354     },
 355     kvm => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable KVM hardware virtualization.",
 359         default => 1,
 360     },
 361     tdf => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable time drift fix.",
 365         default => 0,
 366     },
 367     localtime => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 371     },
 372     freeze => {
 373         optional => 1,
 374         type => 'boolean',
 375         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 376     },
 377     vga => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Select the VGA type.",
 381         verbose_description => "Select the VGA type. If you want to use high resolution" .
 382             " modes (>= 1280x1024x16) then you should use the options " .
 383             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 384             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 385             "display sever. For win* OS you can select how many independent " .
 386             "displays you want, Linux guests can add displays them self. " .
 387             "You can also run without any graphic card, using a serial device" .
 388             " as terminal.",
 389         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 390     },
 391     watchdog => {
 392         optional => 1,
 393         type => 'string', format => 'pve-qm-watchdog',
 394         description => "Create a virtual hardware watchdog device.",
 395         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 396             " (by a guest action), the watchdog must be periodically polled " .
 397             "by an agent inside the guest or else the watchdog will reset " .
 398             "the guest (or execute the respective action specified)",
 399     },
 400     startdate => {
 401         optional => 1,
 402         type => 'string',
 403         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 404         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 405         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 406         default => 'now',
 407     },
 408     startup =>  get_standard_option('pve-startup-order'),
 409     template => {
 410         optional => 1,
 411         type => 'boolean',
 412         description => "Enable/disable Template.",
 413         default => 0,
 414     },
 415     args => {
 416         optional => 1,
 417         type => 'string',
 418         description => "Arbitrary arguments passed to kvm.",
 419         verbose_description => <<EODESCR,
 420 Arbitrary arguments passed to kvm, for example:
 421
 422 args: -no-reboot -no-hpet
 423
 424 NOTE: this option is for experts only.
 425 EODESCR
 426     },
 427     tablet => {
 428         optional => 1,
 429         type => 'boolean',
 430         default => 1,
 431         description => "Enable/disable the USB tablet device.",
 432         verbose_description => "Enable/disable the USB tablet device. This device is " .
 433             "usually needed to allow absolute mouse positioning with VNC. " .
 434             "Else the mouse runs out of sync with normal VNC clients. " .
 435             "If you're running lots of console-only guests on one host, " .
 436             "you may consider disabling this to save some context switches. " .
 437             "This is turned off by default if you use spice (-vga=qxl).",
 438     },
 439     migrate_speed => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 443         minimum => 0,
 444         default => 0,
 445     },
 446     migrate_downtime => {
 447         optional => 1,
 448         type => 'number',
 449         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 450         minimum => 0,
 451         default => 0.1,
 452     },
 453     cdrom => {
 454         optional => 1,
 455         type => 'string', format => 'pve-qm-ide',
 456         typetext => '<volume>',
 457         description => "This is an alias for option -ide2",
 458     },
 459     cpu => {
 460         optional => 1,
 461         description => "Emulated CPU type.",
 462         type => 'string',
 463         format => $cpu_fmt,
 464     },
 465     parent => get_standard_option('pve-snapshot-name', {
 466         optional => 1,
 467         description => "Parent snapshot name. This is used internally, and should not be modified.",
 468     }),
 469     snaptime => {
 470         optional => 1,
 471         description => "Timestamp for snapshots.",
 472         type => 'integer',
 473         minimum => 0,
 474     },
 475     vmstate => {
 476         optional => 1,
 477         type => 'string', format => 'pve-volume-id',
 478         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 479     },
 480     vmstatestorage => get_standard_option('pve-storage-id', {
 481         description => "Default storage for VM state volumes/files.",
 482         optional => 1,
 483     }),
 484     machine => {
 485         description => "Specific the Qemu machine type.",
 486         type => 'string',
 487         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 488         maxLength => 40,
 489         optional => 1,
 490     },
 491     smbios1 => {
 492         description => "Specify SMBIOS type 1 fields.",
 493         type => 'string', format => 'pve-qm-smbios1',
 494         maxLength => 256,
 495         optional => 1,
 496     },
 497     protection => {
 498         optional => 1,
 499         type => 'boolean',
 500         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 501         default => 0,
 502     },
 503     bios => {
 504         optional => 1,
 505         type => 'string',
 506         enum => [ qw(seabios ovmf) ],
 507         description => "Select BIOS implementation.",
 508         default => 'seabios',
 509     },
 510 };
 511
 512 # what about other qemu settings ?
 513 #cpu => 'string',
 514 #machine => 'string',
 515 #fda => 'file',
 516 #fdb => 'file',
 517 #mtdblock => 'file',
 518 #sd => 'file',
 519 #pflash => 'file',
 520 #snapshot => 'bool',
 521 #bootp => 'file',
 522 ##tftp => 'dir',
 523 ##smb => 'dir',
 524 #kernel => 'file',
 525 #append => 'string',
 526 #initrd => 'file',
 527 ##soundhw => 'string',
 528
 529 while (my ($k, $v) = each %$confdesc) {
 530     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 531 }
 532
 533 my $MAX_IDE_DISKS = 4;
 534 my $MAX_SCSI_DISKS = 14;
 535 my $MAX_VIRTIO_DISKS = 16;
 536 my $MAX_SATA_DISKS = 6;
 537 my $MAX_USB_DEVICES = 5;
 538 my $MAX_NETS = 32;
 539 my $MAX_UNUSED_DISKS = 8;
 540 my $MAX_HOSTPCI_DEVICES = 4;
 541 my $MAX_SERIAL_PORTS = 4;
 542 my $MAX_PARALLEL_PORTS = 3;
 543 my $MAX_NUMA = 8;
 544
 545 my $numa_fmt = {
 546     cpus => {
 547         type => "string",
 548         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 549         description => "CPUs accessing this NUMA node.",
 550         format_description => "id[-id];...",
 551     },
 552     memory => {
 553         type => "number",
 554         description => "Amount of memory this NUMA node provides.",
 555         optional => 1,
 556     },
 557     hostnodes => {
 558         type => "string",
 559         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 560         description => "Host NUMA nodes to use.",
 561         format_description => "id[-id];...",
 562         optional => 1,
 563     },
 564     policy => {
 565         type => 'string',
 566         enum => [qw(preferred bind interleave)],
 567         description => "NUMA allocation policy.",
 568         optional => 1,
 569     },
 570 };
 571 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 572 my $numadesc = {
 573     optional => 1,
 574     type => 'string', format => $numa_fmt,
 575     description => "NUMA topology.",
 576 };
 577 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 578
 579 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 580     $confdesc->{"numa$i"} = $numadesc;
 581 }
 582
 583 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 584                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 585                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 586 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 587
 588 my $net_fmt_bridge_descr = <<__EOD__;
 589 Bridge to attach the network device to. The Proxmox VE standard bridge
 590 is called 'vmbr0'.
 591
 592 If you do not specify a bridge, we create a kvm user (NATed) network
 593 device, which provides DHCP and DNS services. The following addresses
 594 are used:
 595
 596  10.0.2.2   Gateway
 597  10.0.2.3   DNS Server
 598  10.0.2.4   SMB Server
 599
 600 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 601 __EOD__
 602
 603 my $net_fmt = {
 604     macaddr => {
 605         type => 'string',
 606         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 607         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 608         format_description => "XX:XX:XX:XX:XX:XX",
 609         optional => 1,
 610     },
 611     model => {
 612         type => 'string',
 613         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 614         enum => $nic_model_list,
 615         default_key => 1,
 616     },
 617     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 618     bridge => {
 619         type => 'string',
 620         description => $net_fmt_bridge_descr,
 621         format_description => 'bridge',
 622         optional => 1,
 623     },
 624     queues => {
 625         type => 'integer',
 626         minimum => 0, maximum => 16,
 627         description => 'Number of packet queues to be used on the device.',
 628         optional => 1,
 629     },
 630     rate => {
 631         type => 'number',
 632         minimum => 0,
 633         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 634         optional => 1,
 635     },
 636     tag => {
 637         type => 'integer',
 638         minimum => 1, maximum => 4094,
 639         description => 'VLAN tag to apply to packets on this interface.',
 640         optional => 1,
 641     },
 642     trunks => {
 643         type => 'string',
 644         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 645         description => 'VLAN trunks to pass through this interface.',
 646         format_description => 'vlanid[;vlanid...]',
 647         optional => 1,
 648     },
 649     firewall => {
 650         type => 'boolean',
 651         description => 'Whether this interface should be protected by the firewall.',
 652         optional => 1,
 653     },
 654     link_down => {
 655         type => 'boolean',
 656         description => 'Whether this interface should be disconnected (like pulling the plug).',
 657         optional => 1,
 658     },
 659 };
 660
 661 my $netdesc = {
 662     optional => 1,
 663     type => 'string', format => $net_fmt,
 664     description => "Specify network devices.",
 665 };
 666
 667 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 668
 669 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 670     $confdesc->{"net$i"} = $netdesc;
 671 }
 672
 673 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 674 sub verify_volume_id_or_qm_path {
 675     my ($volid, $noerr) = @_;
 676
 677     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 678         return $volid;
 679     }
 680
 681     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 682     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 683     if ($@) {
 684         return undef if $noerr;
 685         die $@;
 686     }
 687     return $volid;
 688 }
 689
 690 my $drivename_hash;
 691
 692 my %drivedesc_base = (
 693     volume => { alias => 'file' },
 694     file => {
 695         type => 'string',
 696         format => 'pve-volume-id-or-qm-path',
 697         default_key => 1,
 698         format_description => 'volume',
 699         description => "The drive's backing volume.",
 700     },
 701     media => {
 702         type => 'string',
 703         enum => [qw(cdrom disk)],
 704         description => "The drive's media type.",
 705         default => 'disk',
 706         optional => 1
 707     },
 708     cyls => {
 709         type => 'integer',
 710         description => "Force the drive's physical geometry to have a specific cylinder count.",
 711         optional => 1
 712     },
 713     heads => {
 714         type => 'integer',
 715         description => "Force the drive's physical geometry to have a specific head count.",
 716         optional => 1
 717     },
 718     secs => {
 719         type => 'integer',
 720         description => "Force the drive's physical geometry to have a specific sector count.",
 721         optional => 1
 722     },
 723     trans => {
 724         type => 'string',
 725         enum => [qw(none lba auto)],
 726         description => "Force disk geometry bios translation mode.",
 727         optional => 1,
 728     },
 729     snapshot => {
 730         type => 'boolean',
 731         description => "Whether the drive should be included when making snapshots.",
 732         optional => 1,
 733     },
 734     cache => {
 735         type => 'string',
 736         enum => [qw(none writethrough writeback unsafe directsync)],
 737         description => "The drive's cache mode",
 738         optional => 1,
 739     },
 740     format => get_standard_option('pve-qm-image-format'),
 741     size => {
 742         type => 'string',
 743         format => 'disk-size',
 744         format_description => 'DiskSize',
 745         description => "Disk size. This is purely informational and has no effect.",
 746         optional => 1,
 747     },
 748     backup => {
 749         type => 'boolean',
 750         description => "Whether the drive should be included when making backups.",
 751         optional => 1,
 752     },
 753     replicate => {
 754         type => 'boolean',
 755         description => 'Whether the drive should considered for replication jobs.',
 756         optional => 1,
 757         default => 1,
 758     },
 759     rerror => {
 760         type => 'string',
 761         enum => [qw(ignore report stop)],
 762         description => 'Read error action.',
 763         optional => 1,
 764     },
 765     werror => {
 766         type => 'string',
 767         enum => [qw(enospc ignore report stop)],
 768         description => 'Write error action.',
 769         optional => 1,
 770     },
 771     aio => {
 772         type => 'string',
 773         enum => [qw(native threads)],
 774         description => 'AIO type to use.',
 775         optional => 1,
 776     },
 777     discard => {
 778         type => 'string',
 779         enum => [qw(ignore on)],
 780         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 781         optional => 1,
 782     },
 783     detect_zeroes => {
 784         type => 'boolean',
 785         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 786         optional => 1,
 787     },
 788     serial => {
 789         type => 'string',
 790         format => 'urlencoded',
 791         format_description => 'serial',
 792         maxLength => 20*3, # *3 since it's %xx url enoded
 793         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 794         optional => 1,
 795     }
 796 );
 797
 798 my %iothread_fmt = ( iothread => {
 799         type => 'boolean',
 800         description => "Whether to use iothreads for this drive",
 801         optional => 1,
 802 });
 803
 804 my %model_fmt = (
 805     model => {
 806         type => 'string',
 807         format => 'urlencoded',
 808         format_description => 'model',
 809         maxLength => 40*3, # *3 since it's %xx url enoded
 810         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 811         optional => 1,
 812     },
 813 );
 814
 815 my %queues_fmt = (
 816     queues => {
 817         type => 'integer',
 818         description => "Number of queues.",
 819         minimum => 2,
 820         optional => 1
 821     }
 822 );
 823
 824 my %scsiblock_fmt = (
 825     scsiblock => {
 826         type => 'boolean',
 827         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 828         optional => 1,
 829         default => 0,
 830     },
 831 );
 832
 833 my $add_throttle_desc = sub {
 834     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 835     my $d = {
 836         type => $type,
 837         format_description => $unit,
 838         description => "Maximum $what in $longunit.",
 839         optional => 1,
 840     };
 841     $d->{minimum} = $minimum if defined($minimum);
 842     $drivedesc_base{$key} = $d;
 843 };
 844 # throughput: (leaky bucket)
 845 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 846 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 847 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 848 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 849 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 850 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 851 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 852 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 853 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 854
 855 # pools: (pool of IO before throttling starts taking effect)
 856 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 857 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 858 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 859 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 860 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 861 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 862
 863 # burst lengths
 864 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 865 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 866 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 867 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 868 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 869 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 870
 871 # legacy support
 872 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 873 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 874 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 875 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 876
 877 my $ide_fmt = {
 878     %drivedesc_base,
 879     %model_fmt,
 880 };
 881 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 882
 883 my $idedesc = {
 884     optional => 1,
 885     type => 'string', format => $ide_fmt,
 886     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 887 };
 888 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 889
 890 my $scsi_fmt = {
 891     %drivedesc_base,
 892     %iothread_fmt,
 893     %queues_fmt,
 894     %scsiblock_fmt,
 895 };
 896 my $scsidesc = {
 897     optional => 1,
 898     type => 'string', format => $scsi_fmt,
 899     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 900 };
 901 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 902
 903 my $sata_fmt = {
 904     %drivedesc_base,
 905 };
 906 my $satadesc = {
 907     optional => 1,
 908     type => 'string', format => $sata_fmt,
 909     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 910 };
 911 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 912
 913 my $virtio_fmt = {
 914     %drivedesc_base,
 915     %iothread_fmt,
 916 };
 917 my $virtiodesc = {
 918     optional => 1,
 919     type => 'string', format => $virtio_fmt,
 920     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 921 };
 922 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 923
 924 my $alldrive_fmt = {
 925     %drivedesc_base,
 926     %iothread_fmt,
 927     %model_fmt,
 928     %queues_fmt,
 929     %scsiblock_fmt,
 930 };
 931
 932 my $efidisk_fmt = {
 933     volume => { alias => 'file' },
 934     file => {
 935         type => 'string',
 936         format => 'pve-volume-id-or-qm-path',
 937         default_key => 1,
 938         format_description => 'volume',
 939         description => "The drive's backing volume.",
 940     },
 941     format => get_standard_option('pve-qm-image-format'),
 942     size => {
 943         type => 'string',
 944         format => 'disk-size',
 945         format_description => 'DiskSize',
 946         description => "Disk size. This is purely informational and has no effect.",
 947         optional => 1,
 948     },
 949 };
 950
 951 my $efidisk_desc = {
 952     optional => 1,
 953     type => 'string', format => $efidisk_fmt,
 954     description => "Configure a Disk for storing EFI vars",
 955 };
 956
 957 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 958
 959 my $usb_fmt = {
 960     host => {
 961         default_key => 1,
 962         type => 'string', format => 'pve-qm-usb-device',
 963         format_description => 'HOSTUSBDEVICE|spice',
 964         description => <<EODESCR,
 965 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 966
 967  'bus-port(.port)*' (decimal numbers) or
 968  'vendor_id:product_id' (hexadeciaml numbers) or
 969  'spice'
 970
 971 You can use the 'lsusb -t' command to list existing usb devices.
 972
 973 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 974
 975 The value 'spice' can be used to add a usb redirection devices for spice.
 976 EODESCR
 977     },
 978     usb3 => {
 979         optional => 1,
 980         type => 'boolean',
 981         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 982         default => 0,
 983     },
 984 };
 985
 986 my $usbdesc = {
 987     optional => 1,
 988     type => 'string', format => $usb_fmt,
 989     description => "Configure an USB device (n is 0 to 4).",
 990 };
 991 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 992
 993 # NOTE: the match-groups of this regex are used in parse_hostpci
 994 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 995 my $hostpci_fmt = {
 996     host => {
 997         default_key => 1,
 998         type => 'string',
 999         pattern => qr/$PCIRE(;$PCIRE)*/,
1000         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1001         description => <<EODESCR,
1002 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1003 of PCI virtual functions of the host. HOSTPCIID syntax is:
1004
1005 'bus:dev.func' (hexadecimal numbers)
1006
1007 You can us the 'lspci' command to list existing PCI devices.
1008 EODESCR
1009     },
1010     rombar => {
1011         type => 'boolean',
1012         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1013         optional => 1,
1014         default => 1,
1015     },
1016     romfile => {
1017         type => 'string',
1018         pattern => '[^,;]+',
1019         format_description => 'string',
1020         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1021         optional => 1,
1022     },
1023     pcie => {
1024         type => 'boolean',
1025         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1026         optional => 1,
1027         default => 0,
1028     },
1029     'x-vga' => {
1030         type => 'boolean',
1031         description =>  "Enable vfio-vga device support.",
1032         optional => 1,
1033         default => 0,
1034     },
1035 };
1036 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1037
1038 my $hostpcidesc = {
1039         optional => 1,
1040         type => 'string', format => 'pve-qm-hostpci',
1041         description => "Map host PCI devices into guest.",
1042         verbose_description =>  <<EODESCR,
1043 Map host PCI devices into guest.
1044
1045 NOTE: This option allows direct access to host hardware. So it is no longer
1046 possible to migrate such machines - use with special care.
1047
1048 CAUTION: Experimental! User reported problems with this option.
1049 EODESCR
1050 };
1051 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1052
1053 my $serialdesc = {
1054         optional => 1,
1055         type => 'string',
1056         pattern => '(/dev/.+|socket)',
1057         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1058         verbose_description =>  <<EODESCR,
1059 Create a serial device inside the VM (n is 0 to 3), and pass through a
1060 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1061 host side (use 'qm terminal' to open a terminal connection).
1062
1063 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1064
1065 CAUTION: Experimental! User reported problems with this option.
1066 EODESCR
1067 };
1068
1069 my $paralleldesc= {
1070         optional => 1,
1071         type => 'string',
1072         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1073         description =>  "Map host parallel devices (n is 0 to 2).",
1074         verbose_description =>  <<EODESCR,
1075 Map host parallel devices (n is 0 to 2).
1076
1077 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1078
1079 CAUTION: Experimental! User reported problems with this option.
1080 EODESCR
1081 };
1082
1083 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1084     $confdesc->{"parallel$i"} = $paralleldesc;
1085 }
1086
1087 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1088     $confdesc->{"serial$i"} = $serialdesc;
1089 }
1090
1091 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1092     $confdesc->{"hostpci$i"} = $hostpcidesc;
1093 }
1094
1095 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1096     $drivename_hash->{"ide$i"} = 1;
1097     $confdesc->{"ide$i"} = $idedesc;
1098 }
1099
1100 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1101     $drivename_hash->{"sata$i"} = 1;
1102     $confdesc->{"sata$i"} = $satadesc;
1103 }
1104
1105 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1106     $drivename_hash->{"scsi$i"} = 1;
1107     $confdesc->{"scsi$i"} = $scsidesc ;
1108 }
1109
1110 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1111     $drivename_hash->{"virtio$i"} = 1;
1112     $confdesc->{"virtio$i"} = $virtiodesc;
1113 }
1114
1115 $drivename_hash->{efidisk0} = 1;
1116 $confdesc->{efidisk0} = $efidisk_desc;
1117
1118 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1119     $confdesc->{"usb$i"} = $usbdesc;
1120 }
1121
1122 my $unuseddesc = {
1123     optional => 1,
1124     type => 'string', format => 'pve-volume-id',
1125     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1126 };
1127
1128 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1129     $confdesc->{"unused$i"} = $unuseddesc;
1130 }
1131
1132 my $kvm_api_version = 0;
1133
1134 sub kvm_version {
1135
1136     return $kvm_api_version if $kvm_api_version;
1137
1138     my $fh = IO::File->new("</dev/kvm") ||
1139         return 0;
1140
1141     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1142         $kvm_api_version = $v;
1143     }
1144
1145     $fh->close();
1146
1147     return  $kvm_api_version;
1148 }
1149
1150 my $kvm_user_version;
1151
1152 sub kvm_user_version {
1153
1154     return $kvm_user_version if $kvm_user_version;
1155
1156     $kvm_user_version = 'unknown';
1157
1158     my $code = sub {
1159         my $line = shift;
1160         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1161             $kvm_user_version = $2;
1162         }
1163     };
1164
1165     eval { run_command("kvm -version", outfunc => $code); };
1166     warn $@ if $@;
1167
1168     return $kvm_user_version;
1169
1170 }
1171
1172 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1173
1174 sub valid_drive_names {
1175     # order is important - used to autoselect boot disk
1176     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1177             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1178             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1179             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1180             'efidisk0');
1181 }
1182
1183 sub is_valid_drivename {
1184     my $dev = shift;
1185
1186     return defined($drivename_hash->{$dev});
1187 }
1188
1189 sub option_exists {
1190     my $key = shift;
1191     return defined($confdesc->{$key});
1192 }
1193
1194 sub nic_models {
1195     return $nic_model_list;
1196 }
1197
1198 sub os_list_description {
1199
1200     return {
1201         other => 'Other',
1202         wxp => 'Windows XP',
1203         w2k => 'Windows 2000',
1204         w2k3 =>, 'Windows 2003',
1205         w2k8 => 'Windows 2008',
1206         wvista => 'Windows Vista',
1207         win7 => 'Windows 7',
1208         win8 => 'Windows 8/2012',
1209         win10 => 'Windows 10/2016',
1210         l24 => 'Linux 2.4',
1211         l26 => 'Linux 2.6',
1212     };
1213 }
1214
1215 my $cdrom_path;
1216
1217 sub get_cdrom_path {
1218
1219     return  $cdrom_path if $cdrom_path;
1220
1221     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1222     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1223     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1224 }
1225
1226 sub get_iso_path {
1227     my ($storecfg, $vmid, $cdrom) = @_;
1228
1229     if ($cdrom eq 'cdrom') {
1230         return get_cdrom_path();
1231     } elsif ($cdrom eq 'none') {
1232         return '';
1233     } elsif ($cdrom =~ m|^/|) {
1234         return $cdrom;
1235     } else {
1236         return PVE::Storage::path($storecfg, $cdrom);
1237     }
1238 }
1239
1240 # try to convert old style file names to volume IDs
1241 sub filename_to_volume_id {
1242     my ($vmid, $file, $media) = @_;
1243
1244     if (!($file eq 'none' || $file eq 'cdrom' ||
1245           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1246
1247         return undef if $file =~ m|/|;
1248
1249         if ($media && $media eq 'cdrom') {
1250             $file = "local:iso/$file";
1251         } else {
1252             $file = "local:$vmid/$file";
1253         }
1254     }
1255
1256     return $file;
1257 }
1258
1259 sub verify_media_type {
1260     my ($opt, $vtype, $media) = @_;
1261
1262     return if !$media;
1263
1264     my $etype;
1265     if ($media eq 'disk') {
1266         $etype = 'images';
1267     } elsif ($media eq 'cdrom') {
1268         $etype = 'iso';
1269     } else {
1270         die "internal error";
1271     }
1272
1273     return if ($vtype eq $etype);
1274
1275     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1276 }
1277
1278 sub cleanup_drive_path {
1279     my ($opt, $storecfg, $drive) = @_;
1280
1281     # try to convert filesystem paths to volume IDs
1282
1283     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1284         ($drive->{file} !~ m|^/dev/.+|) &&
1285         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1286         ($drive->{file} !~ m/^\d+$/)) {
1287         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1288         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1289         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1290         verify_media_type($opt, $vtype, $drive->{media});
1291         $drive->{file} = $volid;
1292     }
1293
1294     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1295 }
1296
1297 sub parse_hotplug_features {
1298     my ($data) = @_;
1299
1300     my $res = {};
1301
1302     return $res if $data eq '0';
1303
1304     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1305
1306     foreach my $feature (PVE::Tools::split_list($data)) {
1307         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1308             $res->{$1} = 1;
1309         } else {
1310             die "invalid hotplug feature '$feature'\n";
1311         }
1312     }
1313     return $res;
1314 }
1315
1316 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1317 sub pve_verify_hotplug_features {
1318     my ($value, $noerr) = @_;
1319
1320     return $value if parse_hotplug_features($value);
1321
1322     return undef if $noerr;
1323
1324     die "unable to parse hotplug option\n";
1325 }
1326
1327 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1328 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1329 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1330 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1331 #        [,iothread=on][,serial=serial][,model=model]
1332
1333 sub parse_drive {
1334     my ($key, $data) = @_;
1335
1336     my ($interface, $index);
1337
1338     if ($key =~ m/^([^\d]+)(\d+)$/) {
1339         $interface = $1;
1340         $index = $2;
1341     } else {
1342         return undef;
1343     }
1344
1345     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1346                                      : $confdesc->{$key}->{format};
1347     if (!$desc) {
1348         warn "invalid drive key: $key\n";
1349         return undef;
1350     }
1351     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1352     return undef if !$res;
1353     $res->{interface} = $interface;
1354     $res->{index} = $index;
1355
1356     my $error = 0;
1357     foreach my $opt (qw(bps bps_rd bps_wr)) {
1358         if (my $bps = defined(delete $res->{$opt})) {
1359             if (defined($res->{"m$opt"})) {
1360                 warn "both $opt and m$opt specified\n";
1361                 ++$error;
1362                 next;
1363             }
1364             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1365         }
1366     }
1367
1368     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1369     for my $requirement (
1370         [mbps_max => 'mbps'],
1371         [mbps_rd_max => 'mbps_rd'],
1372         [mbps_wr_max => 'mbps_wr'],
1373         [miops_max => 'miops'],
1374         [miops_rd_max => 'miops_rd'],
1375         [miops_wr_max => 'miops_wr'],
1376         [bps_max_length => 'mbps_max'],
1377         [bps_rd_max_length => 'mbps_rd_max'],
1378         [bps_wr_max_length => 'mbps_wr_max'],
1379         [iops_max_length => 'iops_max'],
1380         [iops_rd_max_length => 'iops_rd_max'],
1381         [iops_wr_max_length => 'iops_wr_max']) {
1382         my ($option, $requires) = @$requirement;
1383         if ($res->{$option} && !$res->{$requires}) {
1384             warn "$option requires $requires\n";
1385             ++$error;
1386         }
1387     }
1388
1389     return undef if $error;
1390
1391     return undef if $res->{mbps_rd} && $res->{mbps};
1392     return undef if $res->{mbps_wr} && $res->{mbps};
1393     return undef if $res->{iops_rd} && $res->{iops};
1394     return undef if $res->{iops_wr} && $res->{iops};
1395
1396     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1397         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1398         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1399         return undef if $res->{interface} eq 'virtio';
1400     }
1401
1402     if (my $size = $res->{size}) {
1403         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1404     }
1405
1406     return $res;
1407 }
1408
1409 sub print_drive {
1410     my ($vmid, $drive) = @_;
1411     my $data = { %$drive };
1412     delete $data->{$_} for qw(index interface);
1413     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1414 }
1415
1416 sub scsi_inquiry {
1417     my($fh, $noerr) = @_;
1418
1419     my $SG_IO = 0x2285;
1420     my $SG_GET_VERSION_NUM = 0x2282;
1421
1422     my $versionbuf = "\x00" x 8;
1423     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1424     if (!$ret) {
1425         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1426         return undef;
1427     }
1428     my $version = unpack("I", $versionbuf);
1429     if ($version < 30000) {
1430         die "scsi generic interface too old\n"  if !$noerr;
1431         return undef;
1432     }
1433
1434     my $buf = "\x00" x 36;
1435     my $sensebuf = "\x00" x 8;
1436     my $cmd = pack("C x3 C x1", 0x12, 36);
1437
1438     # see /usr/include/scsi/sg.h
1439     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1440
1441     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1442                       length($sensebuf), 0, length($buf), $buf,
1443                       $cmd, $sensebuf, 6000);
1444
1445     $ret = ioctl($fh, $SG_IO, $packet);
1446     if (!$ret) {
1447         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1448         return undef;
1449     }
1450
1451     my @res = unpack($sg_io_hdr_t, $packet);
1452     if ($res[17] || $res[18]) {
1453         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1454         return undef;
1455     }
1456
1457     my $res = {};
1458     (my $byte0, my $byte1, $res->{vendor},
1459      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1460
1461     $res->{removable} = $byte1 & 128 ? 1 : 0;
1462     $res->{type} = $byte0 & 31;
1463
1464     return $res;
1465 }
1466
1467 sub path_is_scsi {
1468     my ($path) = @_;
1469
1470     my $fh = IO::File->new("+<$path") || return undef;
1471     my $res = scsi_inquiry($fh, 1);
1472     close($fh);
1473
1474     return $res;
1475 }
1476
1477 sub machine_type_is_q35 {
1478     my ($conf) = @_;
1479
1480     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1481 }
1482
1483 sub print_tabletdevice_full {
1484     my ($conf) = @_;
1485
1486     my $q35 = machine_type_is_q35($conf);
1487
1488     # we use uhci for old VMs because tablet driver was buggy in older qemu
1489     my $usbbus = $q35 ? "ehci" : "uhci";
1490
1491     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1492 }
1493
1494 sub print_drivedevice_full {
1495     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1496
1497     my $device = '';
1498     my $maxdev = 0;
1499
1500     if ($drive->{interface} eq 'virtio') {
1501         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1502         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1503         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1504     } elsif ($drive->{interface} eq 'scsi') {
1505
1506         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1507         my $unit = $drive->{index} % $maxdev;
1508         my $devicetype = 'hd';
1509         my $path = '';
1510         if (drive_is_cdrom($drive)) {
1511             $devicetype = 'cd';
1512         } else {
1513             if ($drive->{file} =~ m|^/|) {
1514                 $path = $drive->{file};
1515                 if (my $info = path_is_scsi($path)) {
1516                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1517                         $devicetype = 'block';
1518                     } elsif ($info->{type} == 1) { # tape
1519                         $devicetype = 'generic';
1520                     }
1521                 }
1522             } else {
1523                  $path = PVE::Storage::path($storecfg, $drive->{file});
1524             }
1525
1526             if($path =~ m/^iscsi\:\/\//){
1527                 $devicetype = 'generic';
1528             }
1529         }
1530
1531         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1532             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1533         } else {
1534             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1535         }
1536
1537     } elsif ($drive->{interface} eq 'ide'){
1538         $maxdev = 2;
1539         my $controller = int($drive->{index} / $maxdev);
1540         my $unit = $drive->{index} % $maxdev;
1541         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1542
1543         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1544         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1545             $model = URI::Escape::uri_unescape($model);
1546             $device .= ",model=$model";
1547         }
1548     } elsif ($drive->{interface} eq 'sata'){
1549         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1550         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1551         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1552     } elsif ($drive->{interface} eq 'usb') {
1553         die "implement me";
1554         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1555     } else {
1556         die "unsupported interface type";
1557     }
1558
1559     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1560
1561     return $device;
1562 }
1563
1564 sub get_initiator_name {
1565     my $initiator;
1566
1567     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1568     while (defined(my $line = <$fh>)) {
1569         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1570         $initiator = $1;
1571         last;
1572     }
1573     $fh->close();
1574
1575     return $initiator;
1576 }
1577
1578 sub print_drive_full {
1579     my ($storecfg, $vmid, $drive) = @_;
1580
1581     my $path;
1582     my $volid = $drive->{file};
1583     my $format;
1584
1585     if (drive_is_cdrom($drive)) {
1586         $path = get_iso_path($storecfg, $vmid, $volid);
1587     } else {
1588         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1589         if ($storeid) {
1590             $path = PVE::Storage::path($storecfg, $volid);
1591             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1592             $format = qemu_img_format($scfg, $volname);
1593         } else {
1594             $path = $volid;
1595             $format = "raw";
1596         }
1597    }
1598
1599     my $opts = '';
1600     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1601     foreach my $o (@qemu_drive_options) {
1602         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1603     }
1604     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1605         my ($dir, $qmpname) = @$type;
1606         if (my $v = $drive->{"mbps$dir"}) {
1607             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1608         }
1609         if (my $v = $drive->{"mbps${dir}_max"}) {
1610             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1611         }
1612         if (my $v = $drive->{"bps${dir}_max_length"}) {
1613             $opts .= ",throttling.bps$qmpname-max-length=$v";
1614         }
1615         if (my $v = $drive->{"iops${dir}"}) {
1616             $opts .= ",throttling.iops$qmpname=$v";
1617         }
1618         if (my $v = $drive->{"iops${dir}_max"}) {
1619             $opts .= ",throttling.iops$qmpname-max=$v";
1620         }
1621         if (my $v = $drive->{"iops${dir}_max_length"}) {
1622             $opts .= ",throttling.iops$qmpname-max-length=$v";
1623         }
1624     }
1625
1626     if (my $serial = $drive->{serial}) {
1627         $serial = URI::Escape::uri_unescape($serial);
1628         $opts .= ",serial=$serial";
1629     }
1630
1631     $opts .= ",format=$format" if $format && !$drive->{format};
1632
1633     my $cache_direct = 0;
1634
1635     if (my $cache = $drive->{cache}) {
1636         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1637     } elsif (!drive_is_cdrom($drive)) {
1638         $opts .= ",cache=none";
1639         $cache_direct = 1;
1640     }
1641
1642     # aio native works only with O_DIRECT
1643     if (!$drive->{aio}) {
1644         if($cache_direct) {
1645             $opts .= ",aio=native";
1646         } else {
1647             $opts .= ",aio=threads";
1648         }
1649     }
1650
1651     if (!drive_is_cdrom($drive)) {
1652         my $detectzeroes;
1653         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1654             $detectzeroes = 'off';
1655         } elsif ($drive->{discard}) {
1656             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1657         } else {
1658             # This used to be our default with discard not being specified:
1659             $detectzeroes = 'on';
1660         }
1661         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1662     }
1663
1664     my $pathinfo = $path ? "file=$path," : '';
1665
1666     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1667 }
1668
1669 sub print_netdevice_full {
1670     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1671
1672     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1673
1674     my $device = $net->{model};
1675     if ($net->{model} eq 'virtio') {
1676          $device = 'virtio-net-pci';
1677      };
1678
1679     my $pciaddr = print_pci_addr("$netid", $bridges);
1680     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1681     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1682         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1683         my $vectors = $net->{queues} * 2 + 2;
1684         $tmpstr .= ",vectors=$vectors,mq=on";
1685     }
1686     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1687
1688     if ($use_old_bios_files) {
1689         my $romfile;
1690         if ($device eq 'virtio-net-pci') {
1691             $romfile = 'pxe-virtio.rom';
1692         } elsif ($device eq 'e1000') {
1693             $romfile = 'pxe-e1000.rom';
1694         } elsif ($device eq 'ne2k') {
1695             $romfile = 'pxe-ne2k_pci.rom';
1696         } elsif ($device eq 'pcnet') {
1697             $romfile = 'pxe-pcnet.rom';
1698         } elsif ($device eq 'rtl8139') {
1699             $romfile = 'pxe-rtl8139.rom';
1700         }
1701         $tmpstr .= ",romfile=$romfile" if $romfile;
1702     }
1703
1704     return $tmpstr;
1705 }
1706
1707 sub print_netdev_full {
1708     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1709
1710     my $i = '';
1711     if ($netid =~ m/^net(\d+)$/) {
1712         $i = int($1);
1713     }
1714
1715     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1716
1717     my $ifname = "tap${vmid}i$i";
1718
1719     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1720     die "interface name '$ifname' is too long (max 15 character)\n"
1721         if length($ifname) >= 16;
1722
1723     my $vhostparam = '';
1724     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1725
1726     my $vmname = $conf->{name} || "vm$vmid";
1727
1728     my $netdev = "";
1729     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1730
1731     if ($net->{bridge}) {
1732         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1733     } else {
1734         $netdev = "type=user,id=$netid,hostname=$vmname";
1735     }
1736
1737     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1738
1739     return $netdev;
1740 }
1741
1742
1743 sub print_cpu_device {
1744     my ($conf, $id) = @_;
1745
1746     my $kvm = $conf->{kvm} // 1;
1747     my $cpu = $kvm ? "kvm64" : "qemu64";
1748     if (my $cputype = $conf->{cpu}) {
1749         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1750             or die "Cannot parse cpu description: $cputype\n";
1751         $cpu = $cpuconf->{cputype};
1752     }
1753
1754     my $cores = $conf->{cores} || 1;
1755
1756     my $current_core = ($id - 1) % $cores;
1757     my $current_socket = int(($id - 1 - $current_core)/$cores);
1758
1759     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1760 }
1761
1762 sub drive_is_cdrom {
1763     my ($drive) = @_;
1764
1765     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1766
1767 }
1768
1769 sub parse_number_sets {
1770     my ($set) = @_;
1771     my $res = [];
1772     foreach my $part (split(/;/, $set)) {
1773         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1774             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1775             push @$res, [ $1, $2 ];
1776         } else {
1777             die "invalid range: $part\n";
1778         }
1779     }
1780     return $res;
1781 }
1782
1783 sub parse_numa {
1784     my ($data) = @_;
1785
1786     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1787     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1788     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1789     return $res;
1790 }
1791
1792 sub parse_hostpci {
1793     my ($value) = @_;
1794
1795     return undef if !$value;
1796
1797     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1798
1799     my @idlist = split(/;/, $res->{host});
1800     delete $res->{host};
1801     foreach my $id (@idlist) {
1802         if ($id =~ /^$PCIRE$/) {
1803             if (defined($2)) {
1804                 push @{$res->{pciid}}, { id => $1, function => $2 };
1805             } else {
1806                 my $pcidevices = lspci($1);
1807                 $res->{pciid} = $pcidevices->{$1};
1808             }
1809         } else {
1810             # should have been caught by parse_property_string already
1811             die "failed to parse PCI id: $id\n";
1812         }
1813     }
1814     return $res;
1815 }
1816
1817 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1818 sub parse_net {
1819     my ($data) = @_;
1820
1821     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1822     if ($@) {
1823         warn $@;
1824         return undef;
1825     }
1826     if (!defined($res->{macaddr})) {
1827         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1828         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1829     }
1830     return $res;
1831 }
1832
1833 sub print_net {
1834     my $net = shift;
1835
1836     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1837 }
1838
1839 sub add_random_macs {
1840     my ($settings) = @_;
1841
1842     foreach my $opt (keys %$settings) {
1843         next if $opt !~ m/^net(\d+)$/;
1844         my $net = parse_net($settings->{$opt});
1845         next if !$net;
1846         $settings->{$opt} = print_net($net);
1847     }
1848 }
1849
1850 sub vm_is_volid_owner {
1851     my ($storecfg, $vmid, $volid) = @_;
1852
1853     if ($volid !~  m|^/|) {
1854         my ($path, $owner);
1855         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1856         if ($owner && ($owner == $vmid)) {
1857             return 1;
1858         }
1859     }
1860
1861     return undef;
1862 }
1863
1864 sub split_flagged_list {
1865     my $text = shift || '';
1866     $text =~ s/[,;]/ /g;
1867     $text =~ s/^\s+//;
1868     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1869 }
1870
1871 sub join_flagged_list {
1872     my ($how, $lst) = @_;
1873     join $how, map { $lst->{$_} . $_ } keys %$lst;
1874 }
1875
1876 sub vmconfig_delete_pending_option {
1877     my ($conf, $key, $force) = @_;
1878
1879     delete $conf->{pending}->{$key};
1880     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1881     $pending_delete_hash->{$key} = $force ? '!' : '';
1882     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1883 }
1884
1885 sub vmconfig_undelete_pending_option {
1886     my ($conf, $key) = @_;
1887
1888     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1889     delete $pending_delete_hash->{$key};
1890
1891     if (%$pending_delete_hash) {
1892         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1893     } else {
1894         delete $conf->{pending}->{delete};
1895     }
1896 }
1897
1898 sub vmconfig_register_unused_drive {
1899     my ($storecfg, $vmid, $conf, $drive) = @_;
1900
1901     if (!drive_is_cdrom($drive)) {
1902         my $volid = $drive->{file};
1903         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1904             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1905         }
1906     }
1907 }
1908
1909 sub vmconfig_cleanup_pending {
1910     my ($conf) = @_;
1911
1912     # remove pending changes when nothing changed
1913     my $changes;
1914     foreach my $opt (keys %{$conf->{pending}}) {
1915         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1916             $changes = 1;
1917             delete $conf->{pending}->{$opt};
1918         }
1919     }
1920
1921     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1922     my $pending_delete_hash = {};
1923     while (my ($opt, $force) = each %$current_delete_hash) {
1924         if (defined($conf->{$opt})) {
1925             $pending_delete_hash->{$opt} = $force;
1926         } else {
1927             $changes = 1;
1928         }
1929     }
1930
1931     if (%$pending_delete_hash) {
1932         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1933     } else {
1934         delete $conf->{pending}->{delete};
1935     }
1936
1937     return $changes;
1938 }
1939
1940 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1941 my $smbios1_fmt = {
1942     uuid => {
1943         type => 'string',
1944         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1945         format_description => 'UUID',
1946         description => "Set SMBIOS1 UUID.",
1947         optional => 1,
1948     },
1949     version => {
1950         type => 'string',
1951         pattern => '\S+',
1952         format_description => 'string',
1953         description => "Set SMBIOS1 version.",
1954         optional => 1,
1955     },
1956     serial => {
1957         type => 'string',
1958         pattern => '\S+',
1959         format_description => 'string',
1960         description => "Set SMBIOS1 serial number.",
1961         optional => 1,
1962     },
1963     manufacturer => {
1964         type => 'string',
1965         pattern => '\S+',
1966         format_description => 'string',
1967         description => "Set SMBIOS1 manufacturer.",
1968         optional => 1,
1969     },
1970     product => {
1971         type => 'string',
1972         pattern => '\S+',
1973         format_description => 'string',
1974         description => "Set SMBIOS1 product ID.",
1975         optional => 1,
1976     },
1977     sku => {
1978         type => 'string',
1979         pattern => '\S+',
1980         format_description => 'string',
1981         description => "Set SMBIOS1 SKU string.",
1982         optional => 1,
1983     },
1984     family => {
1985         type => 'string',
1986         pattern => '\S+',
1987         format_description => 'string',
1988         description => "Set SMBIOS1 family string.",
1989         optional => 1,
1990     },
1991 };
1992
1993 sub parse_smbios1 {
1994     my ($data) = @_;
1995
1996     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1997     warn $@ if $@;
1998     return $res;
1999 }
2000
2001 sub print_smbios1 {
2002     my ($smbios1) = @_;
2003     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2004 }
2005
2006 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2007
2008 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2009 sub verify_bootdisk {
2010     my ($value, $noerr) = @_;
2011
2012     return $value if is_valid_drivename($value);
2013
2014     return undef if $noerr;
2015
2016     die "invalid boot disk '$value'\n";
2017 }
2018
2019 sub parse_watchdog {
2020     my ($value) = @_;
2021
2022     return undef if !$value;
2023
2024     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2025     warn $@ if $@;
2026     return $res;
2027 }
2028
2029 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2030 sub verify_usb_device {
2031     my ($value, $noerr) = @_;
2032
2033     return $value if parse_usb_device($value);
2034
2035     return undef if $noerr;
2036
2037     die "unable to parse usb device\n";
2038 }
2039
2040 # add JSON properties for create and set function
2041 sub json_config_properties {
2042     my $prop = shift;
2043
2044     foreach my $opt (keys %$confdesc) {
2045         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2046         $prop->{$opt} = $confdesc->{$opt};
2047     }
2048
2049     return $prop;
2050 }
2051
2052 sub check_type {
2053     my ($key, $value) = @_;
2054
2055     die "unknown setting '$key'\n" if !$confdesc->{$key};
2056
2057     my $type = $confdesc->{$key}->{type};
2058
2059     if (!defined($value)) {
2060         die "got undefined value\n";
2061     }
2062
2063     if ($value =~ m/[\n\r]/) {
2064         die "property contains a line feed\n";
2065     }
2066
2067     if ($type eq 'boolean') {
2068         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2069         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2070         die "type check ('boolean') failed - got '$value'\n";
2071     } elsif ($type eq 'integer') {
2072         return int($1) if $value =~ m/^(\d+)$/;
2073         die "type check ('integer') failed - got '$value'\n";
2074     } elsif ($type eq 'number') {
2075         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2076         die "type check ('number') failed - got '$value'\n";
2077     } elsif ($type eq 'string') {
2078         if (my $fmt = $confdesc->{$key}->{format}) {
2079             PVE::JSONSchema::check_format($fmt, $value);
2080             return $value;
2081         }
2082         $value =~ s/^\"(.*)\"$/$1/;
2083         return $value;
2084     } else {
2085         die "internal error"
2086     }
2087 }
2088
2089 sub check_iommu_support{
2090     #fixme : need to check IOMMU support
2091     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2092
2093     my $iommu=1;
2094     return $iommu;
2095
2096 }
2097
2098 sub touch_config {
2099     my ($vmid) = @_;
2100
2101     my $conf = PVE::QemuConfig->config_file($vmid);
2102     utime undef, undef, $conf;
2103 }
2104
2105 sub destroy_vm {
2106     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2107
2108     my $conffile = PVE::QemuConfig->config_file($vmid);
2109
2110     my $conf = PVE::QemuConfig->load_config($vmid);
2111
2112     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2113
2114     # only remove disks owned by this VM
2115     foreach_drive($conf, sub {
2116         my ($ds, $drive) = @_;
2117
2118         return if drive_is_cdrom($drive);
2119
2120         my $volid = $drive->{file};
2121
2122         return if !$volid || $volid =~ m|^/|;
2123
2124         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2125         return if !$path || !$owner || ($owner != $vmid);
2126
2127         eval {
2128             PVE::Storage::vdisk_free($storecfg, $volid);
2129         };
2130         warn "Could not remove disk '$volid', check manually: $@" if $@;
2131
2132     });
2133
2134     if ($keep_empty_config) {
2135         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2136     } else {
2137         unlink $conffile;
2138     }
2139
2140     # also remove unused disk
2141     eval {
2142         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2143
2144         eval {
2145             PVE::Storage::foreach_volid($dl, sub {
2146                 my ($volid, $sid, $volname, $d) = @_;
2147                 PVE::Storage::vdisk_free($storecfg, $volid);
2148             });
2149         };
2150         warn $@ if $@;
2151
2152     };
2153     warn $@ if $@;
2154 }
2155
2156 sub parse_vm_config {
2157     my ($filename, $raw) = @_;
2158
2159     return undef if !defined($raw);
2160
2161     my $res = {
2162         digest => Digest::SHA::sha1_hex($raw),
2163         snapshots => {},
2164         pending => {},
2165     };
2166
2167     $filename =~ m|/qemu-server/(\d+)\.conf$|
2168         || die "got strange filename '$filename'";
2169
2170     my $vmid = $1;
2171
2172     my $conf = $res;
2173     my $descr;
2174     my $section = '';
2175
2176     my @lines = split(/\n/, $raw);
2177     foreach my $line (@lines) {
2178         next if $line =~ m/^\s*$/;
2179
2180         if ($line =~ m/^\[PENDING\]\s*$/i) {
2181             $section = 'pending';
2182             if (defined($descr)) {
2183                 $descr =~ s/\s+$//;
2184                 $conf->{description} = $descr;
2185             }
2186             $descr = undef;
2187             $conf = $res->{$section} = {};
2188             next;
2189
2190         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2191             $section = $1;
2192             if (defined($descr)) {
2193                 $descr =~ s/\s+$//;
2194                 $conf->{description} = $descr;
2195             }
2196             $descr = undef;
2197             $conf = $res->{snapshots}->{$section} = {};
2198             next;
2199         }
2200
2201         if ($line =~ m/^\#(.*)\s*$/) {
2202             $descr = '' if !defined($descr);
2203             $descr .= PVE::Tools::decode_text($1) . "\n";
2204             next;
2205         }
2206
2207         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2208             $descr = '' if !defined($descr);
2209             $descr .= PVE::Tools::decode_text($2);
2210         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2211             $conf->{snapstate} = $1;
2212         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2213             my $key = $1;
2214             my $value = $2;
2215             $conf->{$key} = $value;
2216         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2217             my $value = $1;
2218             if ($section eq 'pending') {
2219                 $conf->{delete} = $value; # we parse this later
2220             } else {
2221                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2222             }
2223         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2224             my $key = $1;
2225             my $value = $2;
2226             eval { $value = check_type($key, $value); };
2227             if ($@) {
2228                 warn "vm $vmid - unable to parse value of '$key' - $@";
2229             } else {
2230                 $key = 'ide2' if $key eq 'cdrom';
2231                 my $fmt = $confdesc->{$key}->{format};
2232                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2233                     my $v = parse_drive($key, $value);
2234                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2235                         $v->{file} = $volid;
2236                         $value = print_drive($vmid, $v);
2237                     } else {
2238                         warn "vm $vmid - unable to parse value of '$key'\n";
2239                         next;
2240                     }
2241                 }
2242
2243                 $conf->{$key} = $value;
2244             }
2245         }
2246     }
2247
2248     if (defined($descr)) {
2249         $descr =~ s/\s+$//;
2250         $conf->{description} = $descr;
2251     }
2252     delete $res->{snapstate}; # just to be sure
2253
2254     return $res;
2255 }
2256
2257 sub write_vm_config {
2258     my ($filename, $conf) = @_;
2259
2260     delete $conf->{snapstate}; # just to be sure
2261
2262     if ($conf->{cdrom}) {
2263         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2264         $conf->{ide2} = $conf->{cdrom};
2265         delete $conf->{cdrom};
2266     }
2267
2268     # we do not use 'smp' any longer
2269     if ($conf->{sockets}) {
2270         delete $conf->{smp};
2271     } elsif ($conf->{smp}) {
2272         $conf->{sockets} = $conf->{smp};
2273         delete $conf->{cores};
2274         delete $conf->{smp};
2275     }
2276
2277     my $used_volids = {};
2278
2279     my $cleanup_config = sub {
2280         my ($cref, $pending, $snapname) = @_;
2281
2282         foreach my $key (keys %$cref) {
2283             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2284                 $key eq 'snapstate' || $key eq 'pending';
2285             my $value = $cref->{$key};
2286             if ($key eq 'delete') {
2287                 die "propertry 'delete' is only allowed in [PENDING]\n"
2288                     if !$pending;
2289                 # fixme: check syntax?
2290                 next;
2291             }
2292             eval { $value = check_type($key, $value); };
2293             die "unable to parse value of '$key' - $@" if $@;
2294
2295             $cref->{$key} = $value;
2296
2297             if (!$snapname && is_valid_drivename($key)) {
2298                 my $drive = parse_drive($key, $value);
2299                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2300             }
2301         }
2302     };
2303
2304     &$cleanup_config($conf);
2305
2306     &$cleanup_config($conf->{pending}, 1);
2307
2308     foreach my $snapname (keys %{$conf->{snapshots}}) {
2309         die "internal error" if $snapname eq 'pending';
2310         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2311     }
2312
2313     # remove 'unusedX' settings if we re-add a volume
2314     foreach my $key (keys %$conf) {
2315         my $value = $conf->{$key};
2316         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2317             delete $conf->{$key};
2318         }
2319     }
2320
2321     my $generate_raw_config = sub {
2322         my ($conf, $pending) = @_;
2323
2324         my $raw = '';
2325
2326         # add description as comment to top of file
2327         if (defined(my $descr = $conf->{description})) {
2328             if ($descr) {
2329                 foreach my $cl (split(/\n/, $descr)) {
2330                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2331                 }
2332             } else {
2333                 $raw .= "#\n" if $pending;
2334             }
2335         }
2336
2337         foreach my $key (sort keys %$conf) {
2338             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2339             $raw .= "$key: $conf->{$key}\n";
2340         }
2341         return $raw;
2342     };
2343
2344     my $raw = &$generate_raw_config($conf);
2345
2346     if (scalar(keys %{$conf->{pending}})){
2347         $raw .= "\n[PENDING]\n";
2348         $raw .= &$generate_raw_config($conf->{pending}, 1);
2349     }
2350
2351     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2352         $raw .= "\n[$snapname]\n";
2353         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2354     }
2355
2356     return $raw;
2357 }
2358
2359 sub load_defaults {
2360
2361     my $res = {};
2362
2363     # we use static defaults from our JSON schema configuration
2364     foreach my $key (keys %$confdesc) {
2365         if (defined(my $default = $confdesc->{$key}->{default})) {
2366             $res->{$key} = $default;
2367         }
2368     }
2369
2370     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2371     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2372
2373     return $res;
2374 }
2375
2376 sub config_list {
2377     my $vmlist = PVE::Cluster::get_vmlist();
2378     my $res = {};
2379     return $res if !$vmlist || !$vmlist->{ids};
2380     my $ids = $vmlist->{ids};
2381
2382     foreach my $vmid (keys %$ids) {
2383         my $d = $ids->{$vmid};
2384         next if !$d->{node} || $d->{node} ne $nodename;
2385         next if !$d->{type} || $d->{type} ne 'qemu';
2386         $res->{$vmid}->{exists} = 1;
2387     }
2388     return $res;
2389 }
2390
2391 # test if VM uses local resources (to prevent migration)
2392 sub check_local_resources {
2393     my ($conf, $noerr) = @_;
2394
2395     my $loc_res = 0;
2396
2397     $loc_res = 1 if $conf->{hostusb}; # old syntax
2398     $loc_res = 1 if $conf->{hostpci}; # old syntax
2399
2400     foreach my $k (keys %$conf) {
2401         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2402         # sockets are safe: they will recreated be on the target side post-migrate
2403         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2404         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2405     }
2406
2407     die "VM uses local resources\n" if $loc_res && !$noerr;
2408
2409     return $loc_res;
2410 }
2411
2412 # check if used storages are available on all nodes (use by migrate)
2413 sub check_storage_availability {
2414     my ($storecfg, $conf, $node) = @_;
2415
2416     foreach_drive($conf, sub {
2417         my ($ds, $drive) = @_;
2418
2419         my $volid = $drive->{file};
2420         return if !$volid;
2421
2422         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2423         return if !$sid;
2424
2425         # check if storage is available on both nodes
2426         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2427         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2428    });
2429 }
2430
2431 # list nodes where all VM images are available (used by has_feature API)
2432 sub shared_nodes {
2433     my ($conf, $storecfg) = @_;
2434
2435     my $nodelist = PVE::Cluster::get_nodelist();
2436     my $nodehash = { map { $_ => 1 } @$nodelist };
2437     my $nodename = PVE::INotify::nodename();
2438
2439     foreach_drive($conf, sub {
2440         my ($ds, $drive) = @_;
2441
2442         my $volid = $drive->{file};
2443         return if !$volid;
2444
2445         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2446         if ($storeid) {
2447             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2448             if ($scfg->{disable}) {
2449                 $nodehash = {};
2450             } elsif (my $avail = $scfg->{nodes}) {
2451                 foreach my $node (keys %$nodehash) {
2452                     delete $nodehash->{$node} if !$avail->{$node};
2453                 }
2454             } elsif (!$scfg->{shared}) {
2455                 foreach my $node (keys %$nodehash) {
2456                     delete $nodehash->{$node} if $node ne $nodename
2457                 }
2458             }
2459         }
2460     });
2461
2462     return $nodehash
2463 }
2464
2465 sub check_cmdline {
2466     my ($pidfile, $pid) = @_;
2467
2468     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2469     if (defined($fh)) {
2470         my $line = <$fh>;
2471         $fh->close;
2472         return undef if !$line;
2473         my @param = split(/\0/, $line);
2474
2475         my $cmd = $param[0];
2476         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2477
2478         for (my $i = 0; $i < scalar (@param); $i++) {
2479             my $p = $param[$i];
2480             next if !$p;
2481             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2482                 my $p = $param[$i+1];
2483                 return 1 if $p && ($p eq $pidfile);
2484                 return undef;
2485             }
2486         }
2487     }
2488     return undef;
2489 }
2490
2491 sub check_running {
2492     my ($vmid, $nocheck, $node) = @_;
2493
2494     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2495
2496     die "unable to find configuration file for VM $vmid - no such machine\n"
2497         if !$nocheck && ! -f $filename;
2498
2499     my $pidfile = pidfile_name($vmid);
2500
2501     if (my $fd = IO::File->new("<$pidfile")) {
2502         my $st = stat($fd);
2503         my $line = <$fd>;
2504         close($fd);
2505
2506         my $mtime = $st->mtime;
2507         if ($mtime > time()) {
2508             warn "file '$filename' modified in future\n";
2509         }
2510
2511         if ($line =~ m/^(\d+)$/) {
2512             my $pid = $1;
2513             if (check_cmdline($pidfile, $pid)) {
2514                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2515                     return $pid;
2516                 }
2517             }
2518         }
2519     }
2520
2521     return undef;
2522 }
2523
2524 sub vzlist {
2525
2526     my $vzlist = config_list();
2527
2528     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2529
2530     while (defined(my $de = $fd->read)) {
2531         next if $de !~ m/^(\d+)\.pid$/;
2532         my $vmid = $1;
2533         next if !defined($vzlist->{$vmid});
2534         if (my $pid = check_running($vmid)) {
2535             $vzlist->{$vmid}->{pid} = $pid;
2536         }
2537     }
2538
2539     return $vzlist;
2540 }
2541
2542 sub disksize {
2543     my ($storecfg, $conf) = @_;
2544
2545     my $bootdisk = $conf->{bootdisk};
2546     return undef if !$bootdisk;
2547     return undef if !is_valid_drivename($bootdisk);
2548
2549     return undef if !$conf->{$bootdisk};
2550
2551     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2552     return undef if !defined($drive);
2553
2554     return undef if drive_is_cdrom($drive);
2555
2556     my $volid = $drive->{file};
2557     return undef if !$volid;
2558
2559     return $drive->{size};
2560 }
2561
2562 my $last_proc_pid_stat;
2563
2564 # get VM status information
2565 # This must be fast and should not block ($full == false)
2566 # We only query KVM using QMP if $full == true (this can be slow)
2567 sub vmstatus {
2568     my ($opt_vmid, $full) = @_;
2569
2570     my $res = {};
2571
2572     my $storecfg = PVE::Storage::config();
2573
2574     my $list = vzlist();
2575     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2576
2577     my $cpucount = $cpuinfo->{cpus} || 1;
2578
2579     foreach my $vmid (keys %$list) {
2580         next if $opt_vmid && ($vmid ne $opt_vmid);
2581
2582         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2583         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2584
2585         my $d = {};
2586         $d->{pid} = $list->{$vmid}->{pid};
2587
2588         # fixme: better status?
2589         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2590
2591         my $size = disksize($storecfg, $conf);
2592         if (defined($size)) {
2593             $d->{disk} = 0; # no info available
2594             $d->{maxdisk} = $size;
2595         } else {
2596             $d->{disk} = 0;
2597             $d->{maxdisk} = 0;
2598         }
2599
2600         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2601         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2602         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2603
2604         $d->{name} = $conf->{name} || "VM $vmid";
2605         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2606
2607         if ($conf->{balloon}) {
2608             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2609             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2610         }
2611
2612         $d->{uptime} = 0;
2613         $d->{cpu} = 0;
2614         $d->{mem} = 0;
2615
2616         $d->{netout} = 0;
2617         $d->{netin} = 0;
2618
2619         $d->{diskread} = 0;
2620         $d->{diskwrite} = 0;
2621
2622         $d->{template} = PVE::QemuConfig->is_template($conf);
2623
2624         $res->{$vmid} = $d;
2625     }
2626
2627     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2628     foreach my $dev (keys %$netdev) {
2629         next if $dev !~ m/^tap([1-9]\d*)i/;
2630         my $vmid = $1;
2631         my $d = $res->{$vmid};
2632         next if !$d;
2633
2634         $d->{netout} += $netdev->{$dev}->{receive};
2635         $d->{netin} += $netdev->{$dev}->{transmit};
2636
2637         if ($full) {
2638             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2639             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2640         }
2641
2642     }
2643
2644     my $ctime = gettimeofday;
2645
2646     foreach my $vmid (keys %$list) {
2647
2648         my $d = $res->{$vmid};
2649         my $pid = $d->{pid};
2650         next if !$pid;
2651
2652         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2653         next if !$pstat; # not running
2654
2655         my $used = $pstat->{utime} + $pstat->{stime};
2656
2657         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2658
2659         if ($pstat->{vsize}) {
2660             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2661         }
2662
2663         my $old = $last_proc_pid_stat->{$pid};
2664         if (!$old) {
2665             $last_proc_pid_stat->{$pid} = {
2666                 time => $ctime,
2667                 used => $used,
2668                 cpu => 0,
2669             };
2670             next;
2671         }
2672
2673         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2674
2675         if ($dtime > 1000) {
2676             my $dutime = $used -  $old->{used};
2677
2678             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2679             $last_proc_pid_stat->{$pid} = {
2680                 time => $ctime,
2681                 used => $used,
2682                 cpu => $d->{cpu},
2683             };
2684         } else {
2685             $d->{cpu} = $old->{cpu};
2686         }
2687     }
2688
2689     return $res if !$full;
2690
2691     my $qmpclient = PVE::QMPClient->new();
2692
2693     my $ballooncb = sub {
2694         my ($vmid, $resp) = @_;
2695
2696         my $info = $resp->{'return'};
2697         return if !$info->{max_mem};
2698
2699         my $d = $res->{$vmid};
2700
2701         # use memory assigned to VM
2702         $d->{maxmem} = $info->{max_mem};
2703         $d->{balloon} = $info->{actual};
2704
2705         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2706             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2707             $d->{freemem} = $info->{free_mem};
2708         }
2709
2710         $d->{ballooninfo} = $info;
2711     };
2712
2713     my $blockstatscb = sub {
2714         my ($vmid, $resp) = @_;
2715         my $data = $resp->{'return'} || [];
2716         my $totalrdbytes = 0;
2717         my $totalwrbytes = 0;
2718
2719         for my $blockstat (@$data) {
2720             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2721             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2722
2723             $blockstat->{device} =~ s/drive-//;
2724             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2725         }
2726         $res->{$vmid}->{diskread} = $totalrdbytes;
2727         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2728     };
2729
2730     my $statuscb = sub {
2731         my ($vmid, $resp) = @_;
2732
2733         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2734         # this fails if ballon driver is not loaded, so this must be
2735         # the last commnand (following command are aborted if this fails).
2736         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2737
2738         my $status = 'unknown';
2739         if (!defined($status = $resp->{'return'}->{status})) {
2740             warn "unable to get VM status\n";
2741             return;
2742         }
2743
2744         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2745     };
2746
2747     foreach my $vmid (keys %$list) {
2748         next if $opt_vmid && ($vmid ne $opt_vmid);
2749         next if !$res->{$vmid}->{pid}; # not running
2750         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2751     }
2752
2753     $qmpclient->queue_execute(undef, 2);
2754
2755     foreach my $vmid (keys %$list) {
2756         next if $opt_vmid && ($vmid ne $opt_vmid);
2757         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2758     }
2759
2760     return $res;
2761 }
2762
2763 sub foreach_drive {
2764     my ($conf, $func, @param) = @_;
2765
2766     foreach my $ds (valid_drive_names()) {
2767         next if !defined($conf->{$ds});
2768
2769         my $drive = parse_drive($ds, $conf->{$ds});
2770         next if !$drive;
2771
2772         &$func($ds, $drive, @param);
2773     }
2774 }
2775
2776 sub foreach_volid {
2777     my ($conf, $func, @param) = @_;
2778
2779     my $volhash = {};
2780
2781     my $test_volid = sub {
2782         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2783
2784         return if !$volid;
2785
2786         $volhash->{$volid}->{cdrom} //= 1;
2787         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2788
2789         $volhash->{$volid}->{replicate} //= 0;
2790         $volhash->{$volid}->{replicate} = 1 if $replicate;
2791
2792         $volhash->{$volid}->{referenced_in_config} //= 0;
2793         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2794
2795         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2796             if defined($snapname);
2797     };
2798
2799     foreach_drive($conf, sub {
2800         my ($ds, $drive) = @_;
2801         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2802     });
2803
2804     foreach my $snapname (keys %{$conf->{snapshots}}) {
2805         my $snap = $conf->{snapshots}->{$snapname};
2806         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2807         foreach_drive($snap, sub {
2808             my ($ds, $drive) = @_;
2809             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2810         });
2811     }
2812
2813     foreach my $volid (keys %$volhash) {
2814         &$func($volid, $volhash->{$volid}, @param);
2815     }
2816 }
2817
2818 sub vga_conf_has_spice {
2819     my ($vga) = @_;
2820
2821     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2822
2823     return $1 || 1;
2824 }
2825
2826 sub config_to_command {
2827     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2828
2829     my $cmd = [];
2830     my $globalFlags = [];
2831     my $machineFlags = [];
2832     my $rtcFlags = [];
2833     my $cpuFlags = [];
2834     my $devices = [];
2835     my $pciaddr = '';
2836     my $bridges = {};
2837     my $kvmver = kvm_user_version();
2838     my $vernum = 0; # unknown
2839     my $ostype = $conf->{ostype};
2840     my $winversion = windows_version($ostype);
2841     my $kvm = $conf->{kvm} // 1;
2842
2843     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2844
2845     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2846         $vernum = $1*1000000+$2*1000;
2847     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2848         $vernum = $1*1000000+$2*1000+$3;
2849     }
2850
2851     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2852
2853     my $have_ovz = -f '/proc/vz/vestat';
2854
2855     my $q35 = machine_type_is_q35($conf);
2856     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2857     my $machine_type = $forcemachine || $conf->{machine};
2858     my $use_old_bios_files = undef;
2859     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2860
2861     my $cpuunits = defined($conf->{cpuunits}) ?
2862             $conf->{cpuunits} : $defaults->{cpuunits};
2863
2864     push @$cmd, '/usr/bin/kvm';
2865
2866     push @$cmd, '-id', $vmid;
2867
2868     my $use_virtio = 0;
2869
2870     my $qmpsocket = qmp_socket($vmid);
2871     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2872     push @$cmd, '-mon', "chardev=qmp,mode=control";
2873
2874
2875     push @$cmd, '-pidfile' , pidfile_name($vmid);
2876
2877     push @$cmd, '-daemonize';
2878
2879     if ($conf->{smbios1}) {
2880         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2881     }
2882
2883     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2884         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2885
2886         my $path;
2887         if (my $efidisk = $conf->{efidisk0}) {
2888             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2889             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2890             if ($storeid) {
2891                 $path = PVE::Storage::path($storecfg, $d->{file});
2892             } else {
2893                 $path = $d->{file};
2894             }
2895         } else {
2896             warn "no efidisk configured! Using temporary efivars disk.\n";
2897             $path = "/tmp/$vmid-ovmf.fd";
2898             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2899         }
2900
2901         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2902         push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,file=$path";
2903     }
2904
2905
2906     # add usb controllers
2907     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2908     push @$devices, @usbcontrollers if @usbcontrollers;
2909     my $vga = $conf->{vga};
2910
2911     my $qxlnum = vga_conf_has_spice($vga);
2912     $vga = 'qxl' if $qxlnum;
2913
2914     if (!$vga) {
2915         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2916             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2917         } else {
2918             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2919         }
2920     }
2921
2922     # enable absolute mouse coordinates (needed by vnc)
2923     my $tablet;
2924     if (defined($conf->{tablet})) {
2925         $tablet = $conf->{tablet};
2926     } else {
2927         $tablet = $defaults->{tablet};
2928         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2929         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2930     }
2931
2932     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2933
2934     my $kvm_off = 0;
2935     my $gpu_passthrough;
2936
2937     # host pci devices
2938     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2939         my $d = parse_hostpci($conf->{"hostpci$i"});
2940         next if !$d;
2941
2942         my $pcie = $d->{pcie};
2943         if($pcie){
2944             die "q35 machine model is not enabled" if !$q35;
2945             $pciaddr = print_pcie_addr("hostpci$i");
2946         }else{
2947             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2948         }
2949
2950         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2951         my $romfile = $d->{romfile};
2952
2953         my $xvga = '';
2954         if ($d->{'x-vga'}) {
2955             $xvga = ',x-vga=on';
2956             $kvm_off = 1;
2957             $vga = 'none';
2958             $gpu_passthrough = 1;
2959
2960             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2961                 $xvga = "";
2962             }
2963         }
2964         my $pcidevices = $d->{pciid};
2965         my $multifunction = 1 if @$pcidevices > 1;
2966
2967         my $j=0;
2968         foreach my $pcidevice (@$pcidevices) {
2969
2970             my $id = "hostpci$i";
2971             $id .= ".$j" if $multifunction;
2972             my $addr = $pciaddr;
2973             $addr .= ".$j" if $multifunction;
2974             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2975
2976             if($j == 0){
2977                 $devicestr .= "$rombar$xvga";
2978                 $devicestr .= ",multifunction=on" if $multifunction;
2979                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2980             }
2981
2982             push @$devices, '-device', $devicestr;
2983             $j++;
2984         }
2985     }
2986
2987     # usb devices
2988     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2989     push @$devices, @usbdevices if @usbdevices;
2990     # serial devices
2991     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2992         if (my $path = $conf->{"serial$i"}) {
2993             if ($path eq 'socket') {
2994                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2995                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2996                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2997             } else {
2998                 die "no such serial device\n" if ! -c $path;
2999                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3000                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3001             }
3002         }
3003     }
3004
3005     # parallel devices
3006     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3007         if (my $path = $conf->{"parallel$i"}) {
3008             die "no such parallel device\n" if ! -c $path;
3009             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3010             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3011             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3012         }
3013     }
3014
3015     my $vmname = $conf->{name} || "vm$vmid";
3016
3017     push @$cmd, '-name', $vmname;
3018
3019     my $sockets = 1;
3020     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3021     $sockets = $conf->{sockets} if  $conf->{sockets};
3022
3023     my $cores = $conf->{cores} || 1;
3024
3025     my $maxcpus = $sockets * $cores;
3026
3027     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3028
3029     my $allowed_vcpus = $cpuinfo->{cpus};
3030
3031     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3032         if ($allowed_vcpus < $maxcpus);
3033
3034     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3035
3036         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3037         for (my $i = 2; $i <= $vcpus; $i++)  {
3038             my $cpustr = print_cpu_device($conf,$i);
3039             push @$cmd, '-device', $cpustr;
3040         }
3041
3042     } else {
3043
3044         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3045     }
3046     push @$cmd, '-nodefaults';
3047
3048     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3049
3050     my $bootindex_hash = {};
3051     my $i = 1;
3052     foreach my $o (split(//, $bootorder)) {
3053         $bootindex_hash->{$o} = $i*100;
3054         $i++;
3055     }
3056
3057     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3058
3059     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3060
3061     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3062
3063     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3064
3065     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3066         my $socket = vnc_socket($vmid);
3067         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3068     } else {
3069         push @$cmd, '-nographic';
3070     }
3071
3072     # time drift fix
3073     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3074
3075     my $useLocaltime = $conf->{localtime};
3076
3077     if ($winversion >= 5) { # windows
3078         $useLocaltime = 1 if !defined($conf->{localtime});
3079
3080         # use time drift fix when acpi is enabled
3081         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3082             $tdf = 1 if !defined($conf->{tdf});
3083         }
3084     }
3085
3086     if ($winversion >= 6) {
3087         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3088         push @$cmd, '-no-hpet';
3089     }
3090
3091     push @$rtcFlags, 'driftfix=slew' if $tdf;
3092
3093     if (!$kvm) {
3094         push @$machineFlags, 'accel=tcg';
3095     }
3096
3097     if ($machine_type) {
3098         push @$machineFlags, "type=${machine_type}";
3099     }
3100
3101     if ($conf->{startdate}) {
3102         push @$rtcFlags, "base=$conf->{startdate}";
3103     } elsif ($useLocaltime) {
3104         push @$rtcFlags, 'base=localtime';
3105     }
3106
3107     my $cpu = $kvm ? "kvm64" : "qemu64";
3108     if (my $cputype = $conf->{cpu}) {
3109         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3110             or die "Cannot parse cpu description: $cputype\n";
3111         $cpu = $cpuconf->{cputype};
3112         $kvm_off = 1 if $cpuconf->{hidden};
3113     }
3114
3115     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3116
3117     push @$cpuFlags , '-x2apic'
3118         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3119
3120     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3121
3122     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3123
3124     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3125
3126         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3127         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3128     }
3129
3130     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3131
3132     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3133
3134     push @$cpuFlags, 'kvm=off' if $kvm_off;
3135
3136     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3137         die "internal error"; # should not happen
3138
3139     push @$cpuFlags, "vendor=${cpu_vendor}"
3140         if $cpu_vendor ne 'default';
3141
3142     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3143
3144     push @$cmd, '-cpu', $cpu;
3145
3146     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3147
3148     push @$cmd, '-S' if $conf->{freeze};
3149
3150     # set keyboard layout
3151     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3152     push @$cmd, '-k', $kb if $kb;
3153
3154     # enable sound
3155     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3156     #push @$cmd, '-soundhw', 'es1370';
3157     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3158
3159     if($conf->{agent}) {
3160         my $qgasocket = qmp_socket($vmid, 1);
3161         my $pciaddr = print_pci_addr("qga0", $bridges);
3162         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3163         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3164         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3165     }
3166
3167     my $spice_port;
3168
3169     if ($qxlnum) {
3170         if ($qxlnum > 1) {
3171             if ($winversion){
3172                 for(my $i = 1; $i < $qxlnum; $i++){
3173                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3174                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3175                 }
3176             } else {
3177                 # assume other OS works like Linux
3178                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3179                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3180             }
3181         }
3182
3183         my $pciaddr = print_pci_addr("spice", $bridges);
3184
3185         my $nodename = PVE::INotify::nodename();
3186         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3187         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3188         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3189         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3190         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3191
3192         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3193
3194         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3195         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3196         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3197     }
3198
3199     # enable balloon by default, unless explicitly disabled
3200     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3201         $pciaddr = print_pci_addr("balloon0", $bridges);
3202         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3203     }
3204
3205     if ($conf->{watchdog}) {
3206         my $wdopts = parse_watchdog($conf->{watchdog});
3207         $pciaddr = print_pci_addr("watchdog", $bridges);
3208         my $watchdog = $wdopts->{model} || 'i6300esb';
3209         push @$devices, '-device', "$watchdog$pciaddr";
3210         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3211     }
3212
3213     my $vollist = [];
3214     my $scsicontroller = {};
3215     my $ahcicontroller = {};
3216     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3217
3218     # Add iscsi initiator name if available
3219     if (my $initiator = get_initiator_name()) {
3220         push @$devices, '-iscsi', "initiator-name=$initiator";
3221     }
3222
3223     foreach_drive($conf, sub {
3224         my ($ds, $drive) = @_;
3225
3226         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3227             push @$vollist, $drive->{file};
3228         }
3229
3230         # ignore efidisk here, already added in bios/fw handling code above
3231         return if $drive->{interface} eq 'efidisk';
3232
3233         $use_virtio = 1 if $ds =~ m/^virtio/;
3234
3235         if (drive_is_cdrom ($drive)) {
3236             if ($bootindex_hash->{d}) {
3237                 $drive->{bootindex} = $bootindex_hash->{d};
3238                 $bootindex_hash->{d} += 1;
3239             }
3240         } else {
3241             if ($bootindex_hash->{c}) {
3242                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3243                 $bootindex_hash->{c} += 1;
3244             }
3245         }
3246
3247         if($drive->{interface} eq 'virtio'){
3248            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3249         }
3250
3251         if ($drive->{interface} eq 'scsi') {
3252
3253             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3254
3255             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3256             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3257
3258             my $iothread = '';
3259             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3260                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3261                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3262             } elsif ($drive->{iothread}) {
3263                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3264             }
3265
3266             my $queues = '';
3267             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3268                 $queues = ",num_queues=$drive->{queues}";
3269             }
3270
3271             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3272             $scsicontroller->{$controller}=1;
3273         }
3274
3275         if ($drive->{interface} eq 'sata') {
3276            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3277            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3278            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3279            $ahcicontroller->{$controller}=1;
3280         }
3281
3282         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3283         push @$devices, '-drive',$drive_cmd;
3284         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3285     });
3286
3287     for (my $i = 0; $i < $MAX_NETS; $i++) {
3288          next if !$conf->{"net$i"};
3289          my $d = parse_net($conf->{"net$i"});
3290          next if !$d;
3291
3292          $use_virtio = 1 if $d->{model} eq 'virtio';
3293
3294          if ($bootindex_hash->{n}) {
3295             $d->{bootindex} = $bootindex_hash->{n};
3296             $bootindex_hash->{n} += 1;
3297          }
3298
3299          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3300          push @$devices, '-netdev', $netdevfull;
3301
3302          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3303          push @$devices, '-device', $netdevicefull;
3304     }
3305
3306     if (!$q35) {
3307         # add pci bridges
3308         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3309            $bridges->{1} = 1;
3310            $bridges->{2} = 1;
3311         }
3312
3313         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3314
3315         while (my ($k, $v) = each %$bridges) {
3316             $pciaddr = print_pci_addr("pci.$k");
3317             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3318         }
3319     }
3320
3321     # add custom args
3322     if ($conf->{args}) {
3323         my $aa = PVE::Tools::split_args($conf->{args});
3324         push @$cmd, @$aa;
3325     }
3326
3327     push @$cmd, @$devices;
3328     push @$cmd, '-rtc', join(',', @$rtcFlags)
3329         if scalar(@$rtcFlags);
3330     push @$cmd, '-machine', join(',', @$machineFlags)
3331         if scalar(@$machineFlags);
3332     push @$cmd, '-global', join(',', @$globalFlags)
3333         if scalar(@$globalFlags);
3334
3335     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3336 }
3337
3338 sub vnc_socket {
3339     my ($vmid) = @_;
3340     return "${var_run_tmpdir}/$vmid.vnc";
3341 }
3342
3343 sub spice_port {
3344     my ($vmid) = @_;
3345
3346     my $res = vm_mon_cmd($vmid, 'query-spice');
3347
3348     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3349 }
3350
3351 sub qmp_socket {
3352     my ($vmid, $qga) = @_;
3353     my $sockettype = $qga ? 'qga' : 'qmp';
3354     return "${var_run_tmpdir}/$vmid.$sockettype";
3355 }
3356
3357 sub pidfile_name {
3358     my ($vmid) = @_;
3359     return "${var_run_tmpdir}/$vmid.pid";
3360 }
3361
3362 sub vm_devices_list {
3363     my ($vmid) = @_;
3364
3365     my $res = vm_mon_cmd($vmid, 'query-pci');
3366     my $devices = {};
3367     foreach my $pcibus (@$res) {
3368         foreach my $device (@{$pcibus->{devices}}) {
3369             next if !$device->{'qdev_id'};
3370             if ($device->{'pci_bridge'}) {
3371                 $devices->{$device->{'qdev_id'}} = 1;
3372                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3373                     next if !$bridge_device->{'qdev_id'};
3374                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3375                     $devices->{$device->{'qdev_id'}}++;
3376                 }
3377             } else {
3378                 $devices->{$device->{'qdev_id'}} = 1;
3379             }
3380         }
3381     }
3382
3383     my $resblock = vm_mon_cmd($vmid, 'query-block');
3384     foreach my $block (@$resblock) {
3385         if($block->{device} =~ m/^drive-(\S+)/){
3386                 $devices->{$1} = 1;
3387         }
3388     }
3389
3390     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3391     foreach my $mice (@$resmice) {
3392         if ($mice->{name} eq 'QEMU HID Tablet') {
3393             $devices->{tablet} = 1;
3394             last;
3395         }
3396     }
3397
3398     # for usb devices there is no query-usb
3399     # but we can iterate over the entries in
3400     # qom-list path=/machine/peripheral
3401     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3402     foreach my $per (@$resperipheral) {
3403         if ($per->{name} =~ m/^usb\d+$/) {
3404             $devices->{$per->{name}} = 1;
3405         }
3406     }
3407
3408     return $devices;
3409 }
3410
3411 sub vm_deviceplug {
3412     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3413
3414     my $q35 = machine_type_is_q35($conf);
3415
3416     my $devices_list = vm_devices_list($vmid);
3417     return 1 if defined($devices_list->{$deviceid});
3418
3419     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3420
3421     if ($deviceid eq 'tablet') {
3422
3423         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3424
3425     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3426
3427         die "usb hotplug currently not reliable\n";
3428         # since we can't reliably hot unplug all added usb devices
3429         # and usb passthrough disables live migration
3430         # we disable usb hotplugging for now
3431         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3432
3433     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3434
3435         qemu_iothread_add($vmid, $deviceid, $device);
3436
3437         qemu_driveadd($storecfg, $vmid, $device);
3438         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3439
3440         qemu_deviceadd($vmid, $devicefull);
3441         eval { qemu_deviceaddverify($vmid, $deviceid); };
3442         if (my $err = $@) {
3443             eval { qemu_drivedel($vmid, $deviceid); };
3444             warn $@ if $@;
3445             die $err;
3446         }
3447
3448     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3449
3450
3451         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3452         my $pciaddr = print_pci_addr($deviceid);
3453         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3454
3455         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3456
3457         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3458             qemu_iothread_add($vmid, $deviceid, $device);
3459             $devicefull .= ",iothread=iothread-$deviceid";
3460         }
3461
3462         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3463             $devicefull .= ",num_queues=$device->{queues}";
3464         }
3465
3466         qemu_deviceadd($vmid, $devicefull);
3467         qemu_deviceaddverify($vmid, $deviceid);
3468
3469     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3470
3471         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3472         qemu_driveadd($storecfg, $vmid, $device);
3473
3474         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3475         eval { qemu_deviceadd($vmid, $devicefull); };
3476         if (my $err = $@) {
3477             eval { qemu_drivedel($vmid, $deviceid); };
3478             warn $@ if $@;
3479             die $err;
3480         }
3481
3482     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3483
3484         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3485
3486         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3487         my $use_old_bios_files = undef;
3488         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3489
3490         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3491         qemu_deviceadd($vmid, $netdevicefull);
3492         eval { qemu_deviceaddverify($vmid, $deviceid); };
3493         if (my $err = $@) {
3494             eval { qemu_netdevdel($vmid, $deviceid); };
3495             warn $@ if $@;
3496             die $err;
3497         }
3498
3499     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3500
3501         my $bridgeid = $2;
3502         my $pciaddr = print_pci_addr($deviceid);
3503         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3504
3505         qemu_deviceadd($vmid, $devicefull);
3506         qemu_deviceaddverify($vmid, $deviceid);
3507
3508     } else {
3509         die "can't hotplug device '$deviceid'\n";
3510     }
3511
3512     return 1;
3513 }
3514
3515 # fixme: this should raise exceptions on error!
3516 sub vm_deviceunplug {
3517     my ($vmid, $conf, $deviceid) = @_;
3518
3519     my $devices_list = vm_devices_list($vmid);
3520     return 1 if !defined($devices_list->{$deviceid});
3521
3522     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3523
3524     if ($deviceid eq 'tablet') {
3525
3526         qemu_devicedel($vmid, $deviceid);
3527
3528     } elsif ($deviceid =~ m/^usb\d+$/) {
3529
3530         die "usb hotplug currently not reliable\n";
3531         # when unplugging usb devices this way,
3532         # there may be remaining usb controllers/hubs
3533         # so we disable it for now
3534         qemu_devicedel($vmid, $deviceid);
3535         qemu_devicedelverify($vmid, $deviceid);
3536
3537     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3538
3539         qemu_devicedel($vmid, $deviceid);
3540         qemu_devicedelverify($vmid, $deviceid);
3541         qemu_drivedel($vmid, $deviceid);
3542         qemu_iothread_del($conf, $vmid, $deviceid);
3543
3544     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3545
3546         qemu_devicedel($vmid, $deviceid);
3547         qemu_devicedelverify($vmid, $deviceid);
3548         qemu_iothread_del($conf, $vmid, $deviceid);
3549
3550     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3551
3552         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3553         my $device = parse_drive($deviceid, $conf->{$deviceid});
3554         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3555
3556         qemu_devicedel($vmid, $deviceid);
3557         qemu_drivedel($vmid, $deviceid);
3558         qemu_deletescsihw($conf, $vmid, $deviceid);
3559
3560     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3561
3562         qemu_devicedel($vmid, $deviceid);
3563         qemu_devicedelverify($vmid, $deviceid);
3564         qemu_netdevdel($vmid, $deviceid);
3565
3566     } else {
3567         die "can't unplug device '$deviceid'\n";
3568     }
3569
3570     return 1;
3571 }
3572
3573 sub qemu_deviceadd {
3574     my ($vmid, $devicefull) = @_;
3575
3576     $devicefull = "driver=".$devicefull;
3577     my %options =  split(/[=,]/, $devicefull);
3578
3579     vm_mon_cmd($vmid, "device_add" , %options);
3580 }
3581
3582 sub qemu_devicedel {
3583     my ($vmid, $deviceid) = @_;
3584
3585     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3586 }
3587
3588 sub qemu_iothread_add {
3589     my($vmid, $deviceid, $device) = @_;
3590
3591     if ($device->{iothread}) {
3592         my $iothreads = vm_iothreads_list($vmid);
3593         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3594     }
3595 }
3596
3597 sub qemu_iothread_del {
3598     my($conf, $vmid, $deviceid) = @_;
3599
3600     my $device = parse_drive($deviceid, $conf->{$deviceid});
3601     if ($device->{iothread}) {
3602         my $iothreads = vm_iothreads_list($vmid);
3603         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3604     }
3605 }
3606
3607 sub qemu_objectadd {
3608     my($vmid, $objectid, $qomtype) = @_;
3609
3610     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3611
3612     return 1;
3613 }
3614
3615 sub qemu_objectdel {
3616     my($vmid, $objectid) = @_;
3617
3618     vm_mon_cmd($vmid, "object-del", id => $objectid);
3619
3620     return 1;
3621 }
3622
3623 sub qemu_driveadd {
3624     my ($storecfg, $vmid, $device) = @_;
3625
3626     my $drive = print_drive_full($storecfg, $vmid, $device);
3627     $drive =~ s/\\/\\\\/g;
3628     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3629
3630     # If the command succeeds qemu prints: "OK"
3631     return 1 if $ret =~ m/OK/s;
3632
3633     die "adding drive failed: $ret\n";
3634 }
3635
3636 sub qemu_drivedel {
3637     my($vmid, $deviceid) = @_;
3638
3639     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3640     $ret =~ s/^\s+//;
3641
3642     return 1 if $ret eq "";
3643
3644     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3645     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3646
3647     die "deleting drive $deviceid failed : $ret\n";
3648 }
3649
3650 sub qemu_deviceaddverify {
3651     my ($vmid, $deviceid) = @_;
3652
3653     for (my $i = 0; $i <= 5; $i++) {
3654          my $devices_list = vm_devices_list($vmid);
3655          return 1 if defined($devices_list->{$deviceid});
3656          sleep 1;
3657     }
3658
3659     die "error on hotplug device '$deviceid'\n";
3660 }
3661
3662
3663 sub qemu_devicedelverify {
3664     my ($vmid, $deviceid) = @_;
3665
3666     # need to verify that the device is correctly removed as device_del
3667     # is async and empty return is not reliable
3668
3669     for (my $i = 0; $i <= 5; $i++) {
3670          my $devices_list = vm_devices_list($vmid);
3671          return 1 if !defined($devices_list->{$deviceid});
3672          sleep 1;
3673     }
3674
3675     die "error on hot-unplugging device '$deviceid'\n";
3676 }
3677
3678 sub qemu_findorcreatescsihw {
3679     my ($storecfg, $conf, $vmid, $device) = @_;
3680
3681     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3682
3683     my $scsihwid="$controller_prefix$controller";
3684     my $devices_list = vm_devices_list($vmid);
3685
3686     if(!defined($devices_list->{$scsihwid})) {
3687         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3688     }
3689
3690     return 1;
3691 }
3692
3693 sub qemu_deletescsihw {
3694     my ($conf, $vmid, $opt) = @_;
3695
3696     my $device = parse_drive($opt, $conf->{$opt});
3697
3698     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3699         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3700         return 1;
3701     }
3702
3703     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3704
3705     my $devices_list = vm_devices_list($vmid);
3706     foreach my $opt (keys %{$devices_list}) {
3707         if (PVE::QemuServer::is_valid_drivename($opt)) {
3708             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3709             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3710                 return 1;
3711             }
3712         }
3713     }
3714
3715     my $scsihwid="scsihw$controller";
3716
3717     vm_deviceunplug($vmid, $conf, $scsihwid);
3718
3719     return 1;
3720 }
3721
3722 sub qemu_add_pci_bridge {
3723     my ($storecfg, $conf, $vmid, $device) = @_;
3724
3725     my $bridges = {};
3726
3727     my $bridgeid;
3728
3729     print_pci_addr($device, $bridges);
3730
3731     while (my ($k, $v) = each %$bridges) {
3732         $bridgeid = $k;
3733     }
3734     return 1 if !defined($bridgeid) || $bridgeid < 1;
3735
3736     my $bridge = "pci.$bridgeid";
3737     my $devices_list = vm_devices_list($vmid);
3738
3739     if (!defined($devices_list->{$bridge})) {
3740         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3741     }
3742
3743     return 1;
3744 }
3745
3746 sub qemu_set_link_status {
3747     my ($vmid, $device, $up) = @_;
3748
3749     vm_mon_cmd($vmid, "set_link", name => $device,
3750                up => $up ? JSON::true : JSON::false);
3751 }
3752
3753 sub qemu_netdevadd {
3754     my ($vmid, $conf, $device, $deviceid) = @_;
3755
3756     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3757     my %options =  split(/[=,]/, $netdev);
3758
3759     vm_mon_cmd($vmid, "netdev_add",  %options);
3760     return 1;
3761 }
3762
3763 sub qemu_netdevdel {
3764     my ($vmid, $deviceid) = @_;
3765
3766     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3767 }
3768
3769 sub qemu_usb_hotplug {
3770     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3771
3772     return if !$device;
3773
3774     # remove the old one first
3775     vm_deviceunplug($vmid, $conf, $deviceid);
3776
3777     # check if xhci controller is necessary and available
3778     if ($device->{usb3}) {
3779
3780         my $devicelist = vm_devices_list($vmid);
3781
3782         if (!$devicelist->{xhci}) {
3783             my $pciaddr = print_pci_addr("xhci");
3784             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3785         }
3786     }
3787     my $d = parse_usb_device($device->{host});
3788     $d->{usb3} = $device->{usb3};
3789
3790     # add the new one
3791     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3792 }
3793
3794 sub qemu_cpu_hotplug {
3795     my ($vmid, $conf, $vcpus) = @_;
3796
3797     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3798
3799     my $sockets = 1;
3800     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3801     $sockets = $conf->{sockets} if  $conf->{sockets};
3802     my $cores = $conf->{cores} || 1;
3803     my $maxcpus = $sockets * $cores;
3804
3805     $vcpus = $maxcpus if !$vcpus;
3806
3807     die "you can't add more vcpus than maxcpus\n"
3808         if $vcpus > $maxcpus;
3809
3810     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3811
3812     if ($vcpus < $currentvcpus) {
3813
3814         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3815
3816             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3817                 qemu_devicedel($vmid, "cpu$i");
3818                 my $retry = 0;
3819                 my $currentrunningvcpus = undef;
3820                 while (1) {
3821                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3822                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3823                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3824                     $retry++;
3825                     sleep 1;
3826                 }
3827                 #update conf after each succesfull cpu unplug
3828                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3829                 PVE::QemuConfig->write_config($vmid, $conf);
3830             }
3831         } else {
3832             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3833         }
3834
3835         return;
3836     }
3837
3838     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3839     die "vcpus in running vm does not match its configuration\n"
3840         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3841
3842     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3843
3844         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3845             my $cpustr = print_cpu_device($conf, $i);
3846             qemu_deviceadd($vmid, $cpustr);
3847
3848             my $retry = 0;
3849             my $currentrunningvcpus = undef;
3850             while (1) {
3851                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3852                 last if scalar(@{$currentrunningvcpus}) == $i;
3853                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3854                 sleep 1;
3855                 $retry++;
3856             }
3857             #update conf after each succesfull cpu hotplug
3858             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3859             PVE::QemuConfig->write_config($vmid, $conf);
3860         }
3861     } else {
3862
3863         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3864             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3865         }
3866     }
3867 }
3868
3869 sub qemu_block_set_io_throttle {
3870     my ($vmid, $deviceid,
3871         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3872         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3873         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3874         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3875
3876     return if !check_running($vmid) ;
3877
3878     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3879         bps => int($bps),
3880         bps_rd => int($bps_rd),
3881         bps_wr => int($bps_wr),
3882         iops => int($iops),
3883         iops_rd => int($iops_rd),
3884         iops_wr => int($iops_wr),
3885         bps_max => int($bps_max),
3886         bps_rd_max => int($bps_rd_max),
3887         bps_wr_max => int($bps_wr_max),
3888         iops_max => int($iops_max),
3889         iops_rd_max => int($iops_rd_max),
3890         iops_wr_max => int($iops_wr_max),
3891         bps_max_length => int($bps_max_length),
3892         bps_rd_max_length => int($bps_rd_max_length),
3893         bps_wr_max_length => int($bps_wr_max_length),
3894         iops_max_length => int($iops_max_length),
3895         iops_rd_max_length => int($iops_rd_max_length),
3896         iops_wr_max_length => int($iops_wr_max_length),
3897     );
3898
3899 }
3900
3901 # old code, only used to shutdown old VM after update
3902 sub __read_avail {
3903     my ($fh, $timeout) = @_;
3904
3905     my $sel = new IO::Select;
3906     $sel->add($fh);
3907
3908     my $res = '';
3909     my $buf;
3910
3911     my @ready;
3912     while (scalar (@ready = $sel->can_read($timeout))) {
3913         my $count;
3914         if ($count = $fh->sysread($buf, 8192)) {
3915             if ($buf =~ /^(.*)\(qemu\) $/s) {
3916                 $res .= $1;
3917                 last;
3918             } else {
3919                 $res .= $buf;
3920             }
3921         } else {
3922             if (!defined($count)) {
3923                 die "$!\n";
3924             }
3925             last;
3926         }
3927     }
3928
3929     die "monitor read timeout\n" if !scalar(@ready);
3930
3931     return $res;
3932 }
3933
3934 # old code, only used to shutdown old VM after update
3935 sub vm_monitor_command {
3936     my ($vmid, $cmdstr, $nocheck) = @_;
3937
3938     my $res;
3939
3940     eval {
3941         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3942
3943         my $sname = "${var_run_tmpdir}/$vmid.mon";
3944
3945         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3946             die "unable to connect to VM $vmid socket - $!\n";
3947
3948         my $timeout = 3;
3949
3950         # hack: migrate sometime blocks the monitor (when migrate_downtime
3951         # is set)
3952         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3953             $timeout = 60*60; # 1 hour
3954         }
3955
3956         # read banner;
3957         my $data = __read_avail($sock, $timeout);
3958
3959         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3960             die "got unexpected qemu monitor banner\n";
3961         }
3962
3963         my $sel = new IO::Select;
3964         $sel->add($sock);
3965
3966         if (!scalar(my @ready = $sel->can_write($timeout))) {
3967             die "monitor write error - timeout";
3968         }
3969
3970         my $fullcmd = "$cmdstr\r";
3971
3972         # syslog('info', "VM $vmid monitor command: $cmdstr");
3973
3974         my $b;
3975         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3976             die "monitor write error - $!";
3977         }
3978
3979         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3980
3981         $timeout = 20;
3982
3983         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3984             $timeout = 60*60; # 1 hour
3985         } elsif ($cmdstr =~ m/^(eject|change)/) {
3986             $timeout = 60; # note: cdrom mount command is slow
3987         }
3988         if ($res = __read_avail($sock, $timeout)) {
3989
3990             my @lines = split("\r?\n", $res);
3991
3992             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3993
3994             $res = join("\n", @lines);
3995             $res .= "\n";
3996         }
3997     };
3998
3999     my $err = $@;
4000
4001     if ($err) {
4002         syslog("err", "VM $vmid monitor command failed - $err");
4003         die $err;
4004     }
4005
4006     return $res;
4007 }
4008
4009 sub qemu_block_resize {
4010     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4011
4012     my $running = check_running($vmid);
4013
4014     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4015
4016     return if !$running;
4017
4018     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4019
4020 }
4021
4022 sub qemu_volume_snapshot {
4023     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4024
4025     my $running = check_running($vmid);
4026
4027     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4028         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4029     } else {
4030         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4031     }
4032 }
4033
4034 sub qemu_volume_snapshot_delete {
4035     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4036
4037     my $running = check_running($vmid);
4038
4039     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4040         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4041     } else {
4042         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4043     }
4044 }
4045
4046 sub set_migration_caps {
4047     my ($vmid) = @_;
4048
4049     my $cap_ref = [];
4050
4051     my $enabled_cap = {
4052         "auto-converge" => 1,
4053         "xbzrle" => 1,
4054         "x-rdma-pin-all" => 0,
4055         "zero-blocks" => 0,
4056         "compress" => 0
4057     };
4058
4059     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4060
4061     for my $supported_capability (@$supported_capabilities) {
4062         push @$cap_ref, {
4063             capability => $supported_capability->{capability},
4064             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4065         };
4066     }
4067
4068     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4069 }
4070
4071 my $fast_plug_option = {
4072     'lock' => 1,
4073     'name' => 1,
4074     'onboot' => 1,
4075     'shares' => 1,
4076     'startup' => 1,
4077     'description' => 1,
4078     'protection' => 1,
4079     'vmstatestorage' => 1,
4080 };
4081
4082 # hotplug changes in [PENDING]
4083 # $selection hash can be used to only apply specified options, for
4084 # example: { cores => 1 } (only apply changed 'cores')
4085 # $errors ref is used to return error messages
4086 sub vmconfig_hotplug_pending {
4087     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4088
4089     my $defaults = load_defaults();
4090
4091     # commit values which do not have any impact on running VM first
4092     # Note: those option cannot raise errors, we we do not care about
4093     # $selection and always apply them.
4094
4095     my $add_error = sub {
4096         my ($opt, $msg) = @_;
4097         $errors->{$opt} = "hotplug problem - $msg";
4098     };
4099
4100     my $changes = 0;
4101     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4102         if ($fast_plug_option->{$opt}) {
4103             $conf->{$opt} = $conf->{pending}->{$opt};
4104             delete $conf->{pending}->{$opt};
4105             $changes = 1;
4106         }
4107     }
4108
4109     if ($changes) {
4110         PVE::QemuConfig->write_config($vmid, $conf);
4111         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4112     }
4113
4114     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4115
4116     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4117     while (my ($opt, $force) = each %$pending_delete_hash) {
4118         next if $selection && !$selection->{$opt};
4119         eval {
4120             if ($opt eq 'hotplug') {
4121                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4122             } elsif ($opt eq 'tablet') {
4123                 die "skip\n" if !$hotplug_features->{usb};
4124                 if ($defaults->{tablet}) {
4125                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4126                 } else {
4127                     vm_deviceunplug($vmid, $conf, $opt);
4128                 }
4129             } elsif ($opt =~ m/^usb\d+/) {
4130                 die "skip\n";
4131                 # since we cannot reliably hot unplug usb devices
4132                 # we are disabling it
4133                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4134                 vm_deviceunplug($vmid, $conf, $opt);
4135             } elsif ($opt eq 'vcpus') {
4136                 die "skip\n" if !$hotplug_features->{cpu};
4137                 qemu_cpu_hotplug($vmid, $conf, undef);
4138             } elsif ($opt eq 'balloon') {
4139                 # enable balloon device is not hotpluggable
4140                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4141             } elsif ($fast_plug_option->{$opt}) {
4142                 # do nothing
4143             } elsif ($opt =~ m/^net(\d+)$/) {
4144                 die "skip\n" if !$hotplug_features->{network};
4145                 vm_deviceunplug($vmid, $conf, $opt);
4146             } elsif (is_valid_drivename($opt)) {
4147                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4148                 vm_deviceunplug($vmid, $conf, $opt);
4149                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4150             } elsif ($opt =~ m/^memory$/) {
4151                 die "skip\n" if !$hotplug_features->{memory};
4152                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4153             } elsif ($opt eq 'cpuunits') {
4154                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4155             } elsif ($opt eq 'cpulimit') {
4156                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4157             } else {
4158                 die "skip\n";
4159             }
4160         };
4161         if (my $err = $@) {
4162             &$add_error($opt, $err) if $err ne "skip\n";
4163         } else {
4164             # save new config if hotplug was successful
4165             delete $conf->{$opt};
4166             vmconfig_undelete_pending_option($conf, $opt);
4167             PVE::QemuConfig->write_config($vmid, $conf);
4168             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4169         }
4170     }
4171
4172     foreach my $opt (keys %{$conf->{pending}}) {
4173         next if $selection && !$selection->{$opt};
4174         my $value = $conf->{pending}->{$opt};
4175         eval {
4176             if ($opt eq 'hotplug') {
4177                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4178             } elsif ($opt eq 'tablet') {
4179                 die "skip\n" if !$hotplug_features->{usb};
4180                 if ($value == 1) {
4181                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4182                 } elsif ($value == 0) {
4183                     vm_deviceunplug($vmid, $conf, $opt);
4184                 }
4185             } elsif ($opt =~ m/^usb\d+$/) {
4186                 die "skip\n";
4187                 # since we cannot reliably hot unplug usb devices
4188                 # we are disabling it
4189                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4190                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4191                 die "skip\n" if !$d;
4192                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4193             } elsif ($opt eq 'vcpus') {
4194                 die "skip\n" if !$hotplug_features->{cpu};
4195                 qemu_cpu_hotplug($vmid, $conf, $value);
4196             } elsif ($opt eq 'balloon') {
4197                 # enable/disable balloning device is not hotpluggable
4198                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4199                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4200                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4201
4202                 # allow manual ballooning if shares is set to zero
4203                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4204                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4205                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4206                 }
4207             } elsif ($opt =~ m/^net(\d+)$/) {
4208                 # some changes can be done without hotplug
4209                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4210                                     $vmid, $opt, $value);
4211             } elsif (is_valid_drivename($opt)) {
4212                 # some changes can be done without hotplug
4213                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4214                                      $vmid, $opt, $value, 1);
4215             } elsif ($opt =~ m/^memory$/) { #dimms
4216                 die "skip\n" if !$hotplug_features->{memory};
4217                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4218             } elsif ($opt eq 'cpuunits') {
4219                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4220             } elsif ($opt eq 'cpulimit') {
4221                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4222                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4223             } else {
4224                 die "skip\n";  # skip non-hot-pluggable options
4225             }
4226         };
4227         if (my $err = $@) {
4228             &$add_error($opt, $err) if $err ne "skip\n";
4229         } else {
4230             # save new config if hotplug was successful
4231             $conf->{$opt} = $value;
4232             delete $conf->{pending}->{$opt};
4233             PVE::QemuConfig->write_config($vmid, $conf);
4234             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4235         }
4236     }
4237 }
4238
4239 sub try_deallocate_drive {
4240     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4241
4242     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4243         my $volid = $drive->{file};
4244         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4245             my $sid = PVE::Storage::parse_volume_id($volid);
4246             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4247
4248             # check if the disk is really unused
4249             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4250                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4251             PVE::Storage::vdisk_free($storecfg, $volid);
4252             return 1;
4253         } else {
4254             # If vm is not owner of this disk remove from config
4255             return 1;
4256         }
4257     }
4258
4259     return undef;
4260 }
4261
4262 sub vmconfig_delete_or_detach_drive {
4263     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4264
4265     my $drive = parse_drive($opt, $conf->{$opt});
4266
4267     my $rpcenv = PVE::RPCEnvironment::get();
4268     my $authuser = $rpcenv->get_user();
4269
4270     if ($force) {
4271         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4272         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4273     } else {
4274         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4275     }
4276 }
4277
4278 sub vmconfig_apply_pending {
4279     my ($vmid, $conf, $storecfg) = @_;
4280
4281     # cold plug
4282
4283     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4284     while (my ($opt, $force) = each %$pending_delete_hash) {
4285         die "internal error" if $opt =~ m/^unused/;
4286         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4287         if (!defined($conf->{$opt})) {
4288             vmconfig_undelete_pending_option($conf, $opt);
4289             PVE::QemuConfig->write_config($vmid, $conf);
4290         } elsif (is_valid_drivename($opt)) {
4291             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4292             vmconfig_undelete_pending_option($conf, $opt);
4293             delete $conf->{$opt};
4294             PVE::QemuConfig->write_config($vmid, $conf);
4295         } else {
4296             vmconfig_undelete_pending_option($conf, $opt);
4297             delete $conf->{$opt};
4298             PVE::QemuConfig->write_config($vmid, $conf);
4299         }
4300     }
4301
4302     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4303
4304     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4305         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4306
4307         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4308             # skip if nothing changed
4309         } elsif (is_valid_drivename($opt)) {
4310             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4311                 if defined($conf->{$opt});
4312             $conf->{$opt} = $conf->{pending}->{$opt};
4313         } else {
4314             $conf->{$opt} = $conf->{pending}->{$opt};
4315         }
4316
4317         delete $conf->{pending}->{$opt};
4318         PVE::QemuConfig->write_config($vmid, $conf);
4319     }
4320 }
4321
4322 my $safe_num_ne = sub {
4323     my ($a, $b) = @_;
4324
4325     return 0 if !defined($a) && !defined($b);
4326     return 1 if !defined($a);
4327     return 1 if !defined($b);
4328
4329     return $a != $b;
4330 };
4331
4332 my $safe_string_ne = sub {
4333     my ($a, $b) = @_;
4334
4335     return 0 if !defined($a) && !defined($b);
4336     return 1 if !defined($a);
4337     return 1 if !defined($b);
4338
4339     return $a ne $b;
4340 };
4341
4342 sub vmconfig_update_net {
4343     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4344
4345     my $newnet = parse_net($value);
4346
4347     if ($conf->{$opt}) {
4348         my $oldnet = parse_net($conf->{$opt});
4349
4350         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4351             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4352             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4353             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4354
4355             # for non online change, we try to hot-unplug
4356             die "skip\n" if !$hotplug;
4357             vm_deviceunplug($vmid, $conf, $opt);
4358         } else {
4359
4360             die "internal error" if $opt !~ m/net(\d+)/;
4361             my $iface = "tap${vmid}i$1";
4362
4363             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4364                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4365                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4366                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4367                 PVE::Network::tap_unplug($iface);
4368                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4369             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4370                 # Rate can be applied on its own but any change above needs to
4371                 # include the rate in tap_plug since OVS resets everything.
4372                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4373             }
4374
4375             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4376                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4377             }
4378
4379             return 1;
4380         }
4381     }
4382
4383     if ($hotplug) {
4384         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4385     } else {
4386         die "skip\n";
4387     }
4388 }
4389
4390 sub vmconfig_update_disk {
4391     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4392
4393     # fixme: do we need force?
4394
4395     my $drive = parse_drive($opt, $value);
4396
4397     if ($conf->{$opt}) {
4398
4399         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4400
4401             my $media = $drive->{media} || 'disk';
4402             my $oldmedia = $old_drive->{media} || 'disk';
4403             die "unable to change media type\n" if $media ne $oldmedia;
4404
4405             if (!drive_is_cdrom($old_drive)) {
4406
4407                 if ($drive->{file} ne $old_drive->{file}) {
4408
4409                     die "skip\n" if !$hotplug;
4410
4411                     # unplug and register as unused
4412                     vm_deviceunplug($vmid, $conf, $opt);
4413                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4414
4415                 } else {
4416                     # update existing disk
4417
4418                     # skip non hotpluggable value
4419                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4420                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4421                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4422                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4423                         die "skip\n";
4424                     }
4425
4426                     # apply throttle
4427                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4428                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4429                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4430                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4431                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4432                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4433                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4434                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4435                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4436                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4437                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4438                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4439                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4440                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4441                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4442                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4443                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4444                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4445
4446                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4447                                                    ($drive->{mbps} || 0)*1024*1024,
4448                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4449                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4450                                                    $drive->{iops} || 0,
4451                                                    $drive->{iops_rd} || 0,
4452                                                    $drive->{iops_wr} || 0,
4453                                                    ($drive->{mbps_max} || 0)*1024*1024,
4454                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4455                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4456                                                    $drive->{iops_max} || 0,
4457                                                    $drive->{iops_rd_max} || 0,
4458                                                    $drive->{iops_wr_max} || 0,
4459                                                    $drive->{bps_max_length} || 1,
4460                                                    $drive->{bps_rd_max_length} || 1,
4461                                                    $drive->{bps_wr_max_length} || 1,
4462                                                    $drive->{iops_max_length} || 1,
4463                                                    $drive->{iops_rd_max_length} || 1,
4464                                                    $drive->{iops_wr_max_length} || 1);
4465
4466                     }
4467
4468                     return 1;
4469                 }
4470
4471             } else { # cdrom
4472
4473                 if ($drive->{file} eq 'none') {
4474                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4475                 } else {
4476                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4477                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4478                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4479                 }
4480
4481                 return 1;
4482             }
4483         }
4484     }
4485
4486     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4487     # hotplug new disks
4488     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4489     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4490 }
4491
4492 sub vm_start {
4493     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4494         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4495
4496     PVE::QemuConfig->lock_config($vmid, sub {
4497         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4498
4499         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4500
4501         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4502
4503         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4504
4505         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4506             vmconfig_apply_pending($vmid, $conf, $storecfg);
4507             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4508         }
4509
4510         my $defaults = load_defaults();
4511
4512         # set environment variable useful inside network script
4513         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4514
4515         my $local_volumes = {};
4516
4517         if ($targetstorage) {
4518             foreach_drive($conf, sub {
4519                 my ($ds, $drive) = @_;
4520
4521                 return if drive_is_cdrom($drive);
4522
4523                 my $volid = $drive->{file};
4524
4525                 return if !$volid;
4526
4527                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4528
4529                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4530                 return if $scfg->{shared};
4531                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4532             });
4533
4534             my $format = undef;
4535
4536             foreach my $opt (sort keys %$local_volumes) {
4537
4538                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4539                 my $drive = parse_drive($opt, $conf->{$opt});
4540
4541                 #if remote storage is specified, use default format
4542                 if ($targetstorage && $targetstorage ne "1") {
4543                     $storeid = $targetstorage;
4544                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4545                     $format = $defFormat;
4546                 } else {
4547                     #else we use same format than original
4548                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4549                     $format = qemu_img_format($scfg, $volid);
4550                 }
4551
4552                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4553                 my $newdrive = $drive;
4554                 $newdrive->{format} = $format;
4555                 $newdrive->{file} = $newvolid;
4556                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4557                 $local_volumes->{$opt} = $drivestr;
4558                 #pass drive to conf for command line
4559                 $conf->{$opt} = $drivestr;
4560             }
4561         }
4562
4563         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4564
4565         my $migrate_port = 0;
4566         my $migrate_uri;
4567         if ($statefile) {
4568             if ($statefile eq 'tcp') {
4569                 my $localip = "localhost";
4570                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4571                 my $nodename = PVE::INotify::nodename();
4572
4573                 if (!defined($migration_type)) {
4574                     if (defined($datacenterconf->{migration}->{type})) {
4575                         $migration_type = $datacenterconf->{migration}->{type};
4576                     } else {
4577                         $migration_type = 'secure';
4578                     }
4579                 }
4580
4581                 if ($migration_type eq 'insecure') {
4582                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4583                     if ($migrate_network_addr) {
4584                         $localip = $migrate_network_addr;
4585                     } else {
4586                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4587                     }
4588
4589                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4590                 }
4591
4592                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4593                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4594                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4595                 push @$cmd, '-incoming', $migrate_uri;
4596                 push @$cmd, '-S';
4597
4598             } elsif ($statefile eq 'unix') {
4599                 # should be default for secure migrations as a ssh TCP forward
4600                 # tunnel is not deterministic reliable ready and fails regurarly
4601                 # to set up in time, so use UNIX socket forwards
4602                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4603                 unlink $socket_addr;
4604
4605                 $migrate_uri = "unix:$socket_addr";
4606
4607                 push @$cmd, '-incoming', $migrate_uri;
4608                 push @$cmd, '-S';
4609
4610             } else {
4611                 push @$cmd, '-loadstate', $statefile;
4612             }
4613         } elsif ($paused) {
4614             push @$cmd, '-S';
4615         }
4616
4617         # host pci devices
4618         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4619           my $d = parse_hostpci($conf->{"hostpci$i"});
4620           next if !$d;
4621           my $pcidevices = $d->{pciid};
4622           foreach my $pcidevice (@$pcidevices) {
4623                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4624
4625                 my $info = pci_device_info("0000:$pciid");
4626                 die "IOMMU not present\n" if !check_iommu_support();
4627                 die "no pci device info for device '$pciid'\n" if !$info;
4628                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4629                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4630           }
4631         }
4632
4633         PVE::Storage::activate_volumes($storecfg, $vollist);
4634
4635         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4636             my $cmd = [];
4637             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4638             eval  { run_command($cmd); };
4639         }
4640
4641         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4642                                                   : $defaults->{cpuunits};
4643
4644         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4645
4646         my %properties = (
4647             Slice => 'qemu.slice',
4648             KillMode => 'none',
4649             CPUShares => $cpuunits
4650         );
4651
4652         if (my $cpulimit = $conf->{cpulimit}) {
4653             $properties{CPUQuota} = int($cpulimit * 100);
4654         }
4655         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4656
4657         if ($conf->{hugepages}) {
4658
4659             my $code = sub {
4660                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4661                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4662
4663                 PVE::QemuServer::Memory::hugepages_mount();
4664                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4665
4666                 eval  {
4667                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4668                     run_command($cmd, %run_params);
4669                 };
4670
4671                 if (my $err = $@) {
4672                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4673                     die $err;
4674                 }
4675
4676                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4677             };
4678             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4679
4680         } else {
4681             eval  {
4682                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4683                 run_command($cmd, %run_params);
4684             };
4685         }
4686
4687         if (my $err = $@) {
4688             # deactivate volumes if start fails
4689             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4690             die "start failed: $err";
4691         }
4692
4693         print "migration listens on $migrate_uri\n" if $migrate_uri;
4694
4695         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4696             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4697             warn $@ if $@;
4698         }
4699
4700         #start nbd server for storage migration
4701         if ($targetstorage) {
4702             my $nodename = PVE::INotify::nodename();
4703             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4704             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4705             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4706             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4707
4708             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4709
4710             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4711
4712             foreach my $opt (sort keys %$local_volumes) {
4713                 my $volid = $local_volumes->{$opt};
4714                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4715                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4716                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4717             }
4718         }
4719
4720         if ($migratedfrom) {
4721             eval {
4722                 set_migration_caps($vmid);
4723             };
4724             warn $@ if $@;
4725
4726             if ($spice_port) {
4727                 print "spice listens on port $spice_port\n";
4728                 if ($spice_ticket) {
4729                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4730                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4731                 }
4732             }
4733
4734         } else {
4735             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4736                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4737                     if $conf->{balloon};
4738             }
4739
4740             foreach my $opt (keys %$conf) {
4741                 next if $opt !~  m/^net\d+$/;
4742                 my $nicconf = parse_net($conf->{$opt});
4743                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4744             }
4745         }
4746
4747         vm_mon_cmd_nocheck($vmid, 'qom-set',
4748                     path => "machine/peripheral/balloon0",
4749                     property => "guest-stats-polling-interval",
4750                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4751
4752     });
4753 }
4754
4755 sub vm_mon_cmd {
4756     my ($vmid, $execute, %params) = @_;
4757
4758     my $cmd = { execute => $execute, arguments => \%params };
4759     vm_qmp_command($vmid, $cmd);
4760 }
4761
4762 sub vm_mon_cmd_nocheck {
4763     my ($vmid, $execute, %params) = @_;
4764
4765     my $cmd = { execute => $execute, arguments => \%params };
4766     vm_qmp_command($vmid, $cmd, 1);
4767 }
4768
4769 sub vm_qmp_command {
4770     my ($vmid, $cmd, $nocheck) = @_;
4771
4772     my $res;
4773
4774     my $timeout;
4775     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4776         $timeout = $cmd->{arguments}->{timeout};
4777         delete $cmd->{arguments}->{timeout};
4778     }
4779
4780     eval {
4781         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4782         my $sname = qmp_socket($vmid);
4783         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4784             my $qmpclient = PVE::QMPClient->new();
4785
4786             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4787         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4788             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4789                 if scalar(%{$cmd->{arguments}});
4790             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4791         } else {
4792             die "unable to open monitor socket\n";
4793         }
4794     };
4795     if (my $err = $@) {
4796         syslog("err", "VM $vmid qmp command failed - $err");
4797         die $err;
4798     }
4799
4800     return $res;
4801 }
4802
4803 sub vm_human_monitor_command {
4804     my ($vmid, $cmdline) = @_;
4805
4806     my $res;
4807
4808     my $cmd = {
4809         execute => 'human-monitor-command',
4810         arguments => { 'command-line' => $cmdline},
4811     };
4812
4813     return vm_qmp_command($vmid, $cmd);
4814 }
4815
4816 sub vm_commandline {
4817     my ($storecfg, $vmid) = @_;
4818
4819     my $conf = PVE::QemuConfig->load_config($vmid);
4820
4821     my $defaults = load_defaults();
4822
4823     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4824
4825     return PVE::Tools::cmd2string($cmd);
4826 }
4827
4828 sub vm_reset {
4829     my ($vmid, $skiplock) = @_;
4830
4831     PVE::QemuConfig->lock_config($vmid, sub {
4832
4833         my $conf = PVE::QemuConfig->load_config($vmid);
4834
4835         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4836
4837         vm_mon_cmd($vmid, "system_reset");
4838     });
4839 }
4840
4841 sub get_vm_volumes {
4842     my ($conf) = @_;
4843
4844     my $vollist = [];
4845     foreach_volid($conf, sub {
4846         my ($volid, $attr) = @_;
4847
4848         return if $volid =~ m|^/|;
4849
4850         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4851         return if !$sid;
4852
4853         push @$vollist, $volid;
4854     });
4855
4856     return $vollist;
4857 }
4858
4859 sub vm_stop_cleanup {
4860     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4861
4862     eval {
4863
4864         if (!$keepActive) {
4865             my $vollist = get_vm_volumes($conf);
4866             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4867         }
4868
4869         foreach my $ext (qw(mon qmp pid vnc qga)) {
4870             unlink "/var/run/qemu-server/${vmid}.$ext";
4871         }
4872
4873         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4874     };
4875     warn $@ if $@; # avoid errors - just warn
4876 }
4877
4878 # Note: use $nockeck to skip tests if VM configuration file exists.
4879 # We need that when migration VMs to other nodes (files already moved)
4880 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4881 sub vm_stop {
4882     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4883
4884     $force = 1 if !defined($force) && !$shutdown;
4885
4886     if ($migratedfrom){
4887         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4888         kill 15, $pid if $pid;
4889         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4890         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4891         return;
4892     }
4893
4894     PVE::QemuConfig->lock_config($vmid, sub {
4895
4896         my $pid = check_running($vmid, $nocheck);
4897         return if !$pid;
4898
4899         my $conf;
4900         if (!$nocheck) {
4901             $conf = PVE::QemuConfig->load_config($vmid);
4902             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4903             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4904                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4905                 $timeout = $opts->{down} if $opts->{down};
4906             }
4907         }
4908
4909         $timeout = 60 if !defined($timeout);
4910
4911         eval {
4912             if ($shutdown) {
4913                 if (defined($conf) && $conf->{agent}) {
4914                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4915                 } else {
4916                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4917                 }
4918             } else {
4919                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4920             }
4921         };
4922         my $err = $@;
4923
4924         if (!$err) {
4925             my $count = 0;
4926             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4927                 $count++;
4928                 sleep 1;
4929             }
4930
4931             if ($count >= $timeout) {
4932                 if ($force) {
4933                     warn "VM still running - terminating now with SIGTERM\n";
4934                     kill 15, $pid;
4935                 } else {
4936                     die "VM quit/powerdown failed - got timeout\n";
4937                 }
4938             } else {
4939                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4940                 return;
4941             }
4942         } else {
4943             if ($force) {
4944                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4945                 kill 15, $pid;
4946             } else {
4947                 die "VM quit/powerdown failed\n";
4948             }
4949         }
4950
4951         # wait again
4952         $timeout = 10;
4953
4954         my $count = 0;
4955         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4956             $count++;
4957             sleep 1;
4958         }
4959
4960         if ($count >= $timeout) {
4961             warn "VM still running - terminating now with SIGKILL\n";
4962             kill 9, $pid;
4963             sleep 1;
4964         }
4965
4966         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4967    });
4968 }
4969
4970 sub vm_suspend {
4971     my ($vmid, $skiplock) = @_;
4972
4973     PVE::QemuConfig->lock_config($vmid, sub {
4974
4975         my $conf = PVE::QemuConfig->load_config($vmid);
4976
4977         PVE::QemuConfig->check_lock($conf)
4978             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4979
4980         vm_mon_cmd($vmid, "stop");
4981     });
4982 }
4983
4984 sub vm_resume {
4985     my ($vmid, $skiplock, $nocheck) = @_;
4986
4987     PVE::QemuConfig->lock_config($vmid, sub {
4988
4989         if (!$nocheck) {
4990
4991             my $conf = PVE::QemuConfig->load_config($vmid);
4992
4993             PVE::QemuConfig->check_lock($conf)
4994                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4995
4996             vm_mon_cmd($vmid, "cont");
4997
4998         } else {
4999             vm_mon_cmd_nocheck($vmid, "cont");
5000         }
5001     });
5002 }
5003
5004 sub vm_sendkey {
5005     my ($vmid, $skiplock, $key) = @_;
5006
5007     PVE::QemuConfig->lock_config($vmid, sub {
5008
5009         my $conf = PVE::QemuConfig->load_config($vmid);
5010
5011         # there is no qmp command, so we use the human monitor command
5012         vm_human_monitor_command($vmid, "sendkey $key");
5013     });
5014 }
5015
5016 sub vm_destroy {
5017     my ($storecfg, $vmid, $skiplock) = @_;
5018
5019     PVE::QemuConfig->lock_config($vmid, sub {
5020
5021         my $conf = PVE::QemuConfig->load_config($vmid);
5022
5023         if (!check_running($vmid)) {
5024             destroy_vm($storecfg, $vmid, undef, $skiplock);
5025         } else {
5026             die "VM $vmid is running - destroy failed\n";
5027         }
5028     });
5029 }
5030
5031 # pci helpers
5032
5033 sub file_write {
5034     my ($filename, $buf) = @_;
5035
5036     my $fh = IO::File->new($filename, "w");
5037     return undef if !$fh;
5038
5039     my $res = print $fh $buf;
5040
5041     $fh->close();
5042
5043     return $res;
5044 }
5045
5046 sub pci_device_info {
5047     my ($name) = @_;
5048
5049     my $res;
5050
5051     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5052     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5053
5054     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5055     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5056
5057     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5058     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5059
5060     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5061     return undef if !defined($product) || $product !~ s/^0x//;
5062
5063     $res = {
5064         name => $name,
5065         vendor => $vendor,
5066         product => $product,
5067         domain => $domain,
5068         bus => $bus,
5069         slot => $slot,
5070         func => $func,
5071         irq => $irq,
5072         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5073     };
5074
5075     return $res;
5076 }
5077
5078 sub pci_dev_reset {
5079     my ($dev) = @_;
5080
5081     my $name = $dev->{name};
5082
5083     my $fn = "$pcisysfs/devices/$name/reset";
5084
5085     return file_write($fn, "1");
5086 }
5087
5088 sub pci_dev_bind_to_vfio {
5089     my ($dev) = @_;
5090
5091     my $name = $dev->{name};
5092
5093     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5094
5095     if (!-d $vfio_basedir) {
5096         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5097     }
5098     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5099
5100     my $testdir = "$vfio_basedir/$name";
5101     return 1 if -d $testdir;
5102
5103     my $data = "$dev->{vendor} $dev->{product}";
5104     return undef if !file_write("$vfio_basedir/new_id", $data);
5105
5106     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5107     if (!file_write($fn, $name)) {
5108         return undef if -f $fn;
5109     }
5110
5111     $fn = "$vfio_basedir/bind";
5112     if (! -d $testdir) {
5113         return undef if !file_write($fn, $name);
5114     }
5115
5116     return -d $testdir;
5117 }
5118
5119 sub pci_dev_group_bind_to_vfio {
5120     my ($pciid) = @_;
5121
5122     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5123
5124     if (!-d $vfio_basedir) {
5125         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5126     }
5127     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5128
5129     # get IOMMU group devices
5130     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5131       my @devs = grep /^0000:/, readdir($D);
5132     closedir($D);
5133
5134     foreach my $pciid (@devs) {
5135         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5136
5137         # pci bridges, switches or root ports are not supported
5138         # they have a pci_bus subdirectory so skip them
5139         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5140
5141         my $info = pci_device_info($1);
5142         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5143     }
5144
5145     return 1;
5146 }
5147
5148 # vzdump restore implementaion
5149
5150 sub tar_archive_read_firstfile {
5151     my $archive = shift;
5152
5153     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5154
5155     # try to detect archive type first
5156     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5157         die "unable to open file '$archive'\n";
5158     my $firstfile = <$fh>;
5159     kill 15, $pid;
5160     close $fh;
5161
5162     die "ERROR: archive contaions no data\n" if !$firstfile;
5163     chomp $firstfile;
5164
5165     return $firstfile;
5166 }
5167
5168 sub tar_restore_cleanup {
5169     my ($storecfg, $statfile) = @_;
5170
5171     print STDERR "starting cleanup\n";
5172
5173     if (my $fd = IO::File->new($statfile, "r")) {
5174         while (defined(my $line = <$fd>)) {
5175             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5176                 my $volid = $2;
5177                 eval {
5178                     if ($volid =~ m|^/|) {
5179                         unlink $volid || die 'unlink failed\n';
5180                     } else {
5181                         PVE::Storage::vdisk_free($storecfg, $volid);
5182                     }
5183                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5184                 };
5185                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5186             } else {
5187                 print STDERR "unable to parse line in statfile - $line";
5188             }
5189         }
5190         $fd->close();
5191     }
5192 }
5193
5194 sub restore_archive {
5195     my ($archive, $vmid, $user, $opts) = @_;
5196
5197     my $format = $opts->{format};
5198     my $comp;
5199
5200     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5201         $format = 'tar' if !$format;
5202         $comp = 'gzip';
5203     } elsif ($archive =~ m/\.tar$/) {
5204         $format = 'tar' if !$format;
5205     } elsif ($archive =~ m/.tar.lzo$/) {
5206         $format = 'tar' if !$format;
5207         $comp = 'lzop';
5208     } elsif ($archive =~ m/\.vma$/) {
5209         $format = 'vma' if !$format;
5210     } elsif ($archive =~ m/\.vma\.gz$/) {
5211         $format = 'vma' if !$format;
5212         $comp = 'gzip';
5213     } elsif ($archive =~ m/\.vma\.lzo$/) {
5214         $format = 'vma' if !$format;
5215         $comp = 'lzop';
5216     } else {
5217         $format = 'vma' if !$format; # default
5218     }
5219
5220     # try to detect archive format
5221     if ($format eq 'tar') {
5222         return restore_tar_archive($archive, $vmid, $user, $opts);
5223     } else {
5224         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5225     }
5226 }
5227
5228 sub restore_update_config_line {
5229     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5230
5231     return if $line =~ m/^\#qmdump\#/;
5232     return if $line =~ m/^\#vzdump\#/;
5233     return if $line =~ m/^lock:/;
5234     return if $line =~ m/^unused\d+:/;
5235     return if $line =~ m/^parent:/;
5236     return if $line =~ m/^template:/; # restored VM is never a template
5237
5238     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5239     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5240         # try to convert old 1.X settings
5241         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5242         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5243             my ($model, $macaddr) = split(/\=/, $devconfig);
5244             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5245             my $net = {
5246                 model => $model,
5247                 bridge => "vmbr$ind",
5248                 macaddr => $macaddr,
5249             };
5250             my $netstr = print_net($net);
5251
5252             print $outfd "net$cookie->{netcount}: $netstr\n";
5253             $cookie->{netcount}++;
5254         }
5255     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5256         my ($id, $netstr) = ($1, $2);
5257         my $net = parse_net($netstr);
5258         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5259         $netstr = print_net($net);
5260         print $outfd "$id: $netstr\n";
5261     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5262         my $virtdev = $1;
5263         my $value = $3;
5264         my $di = parse_drive($virtdev, $value);
5265         if (defined($di->{backup}) && !$di->{backup}) {
5266             print $outfd "#$line";
5267         } elsif ($map->{$virtdev}) {
5268             delete $di->{format}; # format can change on restore
5269             $di->{file} = $map->{$virtdev};
5270             $value = print_drive($vmid, $di);
5271             print $outfd "$virtdev: $value\n";
5272         } else {
5273             print $outfd $line;
5274         }
5275     } else {
5276         print $outfd $line;
5277     }
5278 }
5279
5280 sub scan_volids {
5281     my ($cfg, $vmid) = @_;
5282
5283     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5284
5285     my $volid_hash = {};
5286     foreach my $storeid (keys %$info) {
5287         foreach my $item (@{$info->{$storeid}}) {
5288             next if !($item->{volid} && $item->{size});
5289             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5290             $volid_hash->{$item->{volid}} = $item;
5291         }
5292     }
5293
5294     return $volid_hash;
5295 }
5296
5297 sub is_volume_in_use {
5298     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5299
5300     my $path = PVE::Storage::path($storecfg, $volid);
5301
5302     my $scan_config = sub {
5303         my ($cref, $snapname) = @_;
5304
5305         foreach my $key (keys %$cref) {
5306             my $value = $cref->{$key};
5307             if (is_valid_drivename($key)) {
5308                 next if $skip_drive && $key eq $skip_drive;
5309                 my $drive = parse_drive($key, $value);
5310                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5311                 return 1 if $volid eq $drive->{file};
5312                 if ($drive->{file} =~ m!^/!) {
5313                     return 1 if $drive->{file} eq $path;
5314                 } else {
5315                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5316                     next if !$storeid;
5317                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5318                     next if !$scfg;
5319                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5320                 }
5321             }
5322         }
5323
5324         return 0;
5325     };
5326
5327     return 1 if &$scan_config($conf);
5328
5329     undef $skip_drive;
5330
5331     foreach my $snapname (keys %{$conf->{snapshots}}) {
5332         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5333     }
5334
5335     return 0;
5336 }
5337
5338 sub update_disksize {
5339     my ($vmid, $conf, $volid_hash) = @_;
5340
5341     my $changes;
5342
5343     my $used = {};
5344
5345     # Note: it is allowed to define multiple storages with same path (alias), so
5346     # we need to check both 'volid' and real 'path' (two different volid can point
5347     # to the same path).
5348
5349     my $usedpath = {};
5350
5351     # update size info
5352     foreach my $opt (keys %$conf) {
5353         if (is_valid_drivename($opt)) {
5354             my $drive = parse_drive($opt, $conf->{$opt});
5355             my $volid = $drive->{file};
5356             next if !$volid;
5357
5358             $used->{$volid} = 1;
5359             if ($volid_hash->{$volid} &&
5360                 (my $path = $volid_hash->{$volid}->{path})) {
5361                 $usedpath->{$path} = 1;
5362             }
5363
5364             next if drive_is_cdrom($drive);
5365             next if !$volid_hash->{$volid};
5366
5367             $drive->{size} = $volid_hash->{$volid}->{size};
5368             my $new = print_drive($vmid, $drive);
5369             if ($new ne $conf->{$opt}) {
5370                 $changes = 1;
5371                 $conf->{$opt} = $new;
5372             }
5373         }
5374     }
5375
5376     # remove 'unusedX' entry if volume is used
5377     foreach my $opt (keys %$conf) {
5378         next if $opt !~ m/^unused\d+$/;
5379         my $volid = $conf->{$opt};
5380         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5381         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5382             $changes = 1;
5383             delete $conf->{$opt};
5384         }
5385     }
5386
5387     foreach my $volid (sort keys %$volid_hash) {
5388         next if $volid =~ m/vm-$vmid-state-/;
5389         next if $used->{$volid};
5390         my $path = $volid_hash->{$volid}->{path};
5391         next if !$path; # just to be sure
5392         next if $usedpath->{$path};
5393         $changes = 1;
5394         PVE::QemuConfig->add_unused_volume($conf, $volid);
5395         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5396     }
5397
5398     return $changes;
5399 }
5400
5401 sub rescan {
5402     my ($vmid, $nolock) = @_;
5403
5404     my $cfg = PVE::Storage::config();
5405
5406     my $volid_hash = scan_volids($cfg, $vmid);
5407
5408     my $updatefn =  sub {
5409         my ($vmid) = @_;
5410
5411         my $conf = PVE::QemuConfig->load_config($vmid);
5412
5413         PVE::QemuConfig->check_lock($conf);
5414
5415         my $vm_volids = {};
5416         foreach my $volid (keys %$volid_hash) {
5417             my $info = $volid_hash->{$volid};
5418             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5419         }
5420
5421         my $changes = update_disksize($vmid, $conf, $vm_volids);
5422
5423         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5424     };
5425
5426     if (defined($vmid)) {
5427         if ($nolock) {
5428             &$updatefn($vmid);
5429         } else {
5430             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5431         }
5432     } else {
5433         my $vmlist = config_list();
5434         foreach my $vmid (keys %$vmlist) {
5435             if ($nolock) {
5436                 &$updatefn($vmid);
5437             } else {
5438                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5439             }
5440         }
5441     }
5442 }
5443
5444 sub restore_vma_archive {
5445     my ($archive, $vmid, $user, $opts, $comp) = @_;
5446
5447     my $input = $archive eq '-' ? "<&STDIN" : undef;
5448     my $readfrom = $archive;
5449
5450     my $uncomp = '';
5451     if ($comp) {
5452         $readfrom = '-';
5453         my $qarchive = PVE::Tools::shellquote($archive);
5454         if ($comp eq 'gzip') {
5455             $uncomp = "zcat $qarchive|";
5456         } elsif ($comp eq 'lzop') {
5457             $uncomp = "lzop -d -c $qarchive|";
5458         } else {
5459             die "unknown compression method '$comp'\n";
5460         }
5461
5462     }
5463
5464     my $tmpdir = "/var/tmp/vzdumptmp$$";
5465     rmtree $tmpdir;
5466
5467     # disable interrupts (always do cleanups)
5468     local $SIG{INT} =
5469         local $SIG{TERM} =
5470         local $SIG{QUIT} =
5471         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5472
5473     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5474     POSIX::mkfifo($mapfifo, 0600);
5475     my $fifofh;
5476
5477     my $openfifo = sub {
5478         open($fifofh, '>', $mapfifo) || die $!;
5479     };
5480
5481     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5482
5483     my $oldtimeout;
5484     my $timeout = 5;
5485
5486     my $devinfo = {};
5487
5488     my $rpcenv = PVE::RPCEnvironment::get();
5489
5490     my $conffile = PVE::QemuConfig->config_file($vmid);
5491     my $tmpfn = "$conffile.$$.tmp";
5492
5493     # Note: $oldconf is undef if VM does not exists
5494     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5495     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5496
5497     my $print_devmap = sub {
5498         my $virtdev_hash = {};
5499
5500         my $cfgfn = "$tmpdir/qemu-server.conf";
5501
5502         # we can read the config - that is already extracted
5503         my $fh = IO::File->new($cfgfn, "r") ||
5504             "unable to read qemu-server.conf - $!\n";
5505
5506         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5507         if (-f $fwcfgfn) {
5508             my $pve_firewall_dir = '/etc/pve/firewall';
5509             mkdir $pve_firewall_dir; # make sure the dir exists
5510             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5511         }
5512
5513         while (defined(my $line = <$fh>)) {
5514             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5515                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5516                 die "archive does not contain data for drive '$virtdev'\n"
5517                     if !$devinfo->{$devname};
5518                 if (defined($opts->{storage})) {
5519                     $storeid = $opts->{storage} || 'local';
5520                 } elsif (!$storeid) {
5521                     $storeid = 'local';
5522                 }
5523                 $format = 'raw' if !$format;
5524                 $devinfo->{$devname}->{devname} = $devname;
5525                 $devinfo->{$devname}->{virtdev} = $virtdev;
5526                 $devinfo->{$devname}->{format} = $format;
5527                 $devinfo->{$devname}->{storeid} = $storeid;
5528
5529                 # check permission on storage
5530                 my $pool = $opts->{pool}; # todo: do we need that?
5531                 if ($user ne 'root@pam') {
5532                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5533                 }
5534
5535                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5536             }
5537         }
5538
5539         foreach my $devname (keys %$devinfo) {
5540             die "found no device mapping information for device '$devname'\n"
5541                 if !$devinfo->{$devname}->{virtdev};
5542         }
5543
5544         my $cfg = PVE::Storage::config();
5545
5546         # create empty/temp config
5547         if ($oldconf) {
5548             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5549             foreach_drive($oldconf, sub {
5550                 my ($ds, $drive) = @_;
5551
5552                 return if drive_is_cdrom($drive);
5553
5554                 my $volid = $drive->{file};
5555
5556                 return if !$volid || $volid =~ m|^/|;
5557
5558                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5559                 return if !$path || !$owner || ($owner != $vmid);
5560
5561                 # Note: only delete disk we want to restore
5562                 # other volumes will become unused
5563                 if ($virtdev_hash->{$ds}) {
5564                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5565                     if (my $err = $@) {
5566                         warn $err;
5567                     }
5568                 }
5569             });
5570
5571             # delete vmstate files
5572             # since after the restore we have no snapshots anymore
5573             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5574                 my $snap = $oldconf->{snapshots}->{$snapname};
5575                 if ($snap->{vmstate}) {
5576                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5577                     if (my $err = $@) {
5578                         warn $err;
5579                     }
5580                 }
5581             }
5582         }
5583
5584         my $map = {};
5585         foreach my $virtdev (sort keys %$virtdev_hash) {
5586             my $d = $virtdev_hash->{$virtdev};
5587             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5588             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5589
5590             # test if requested format is supported
5591             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5592             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5593             $d->{format} = $defFormat if !$supported;
5594
5595             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5596                                                   $d->{format}, undef, $alloc_size);
5597             print STDERR "new volume ID is '$volid'\n";
5598             $d->{volid} = $volid;
5599             my $path = PVE::Storage::path($cfg, $volid);
5600
5601             PVE::Storage::activate_volumes($cfg,[$volid]);
5602
5603             my $write_zeros = 1;
5604             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5605                 $write_zeros = 0;
5606             }
5607
5608             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5609
5610             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5611             $map->{$virtdev} = $volid;
5612         }
5613
5614         $fh->seek(0, 0) || die "seek failed - $!\n";
5615
5616         my $outfd = new IO::File ($tmpfn, "w") ||
5617             die "unable to write config for VM $vmid\n";
5618
5619         my $cookie = { netcount => 0 };
5620         while (defined(my $line = <$fh>)) {
5621             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5622         }
5623
5624         $fh->close();
5625         $outfd->close();
5626     };
5627
5628     eval {
5629         # enable interrupts
5630         local $SIG{INT} =
5631             local $SIG{TERM} =
5632             local $SIG{QUIT} =
5633             local $SIG{HUP} =
5634             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5635         local $SIG{ALRM} = sub { die "got timeout\n"; };
5636
5637         $oldtimeout = alarm($timeout);
5638
5639         my $parser = sub {
5640             my $line = shift;
5641
5642             print "$line\n";
5643
5644             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5645                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5646                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5647             } elsif ($line =~ m/^CTIME: /) {
5648                 # we correctly received the vma config, so we can disable
5649                 # the timeout now for disk allocation (set to 10 minutes, so
5650                 # that we always timeout if something goes wrong)
5651                 alarm(600);
5652                 &$print_devmap();
5653                 print $fifofh "done\n";
5654                 my $tmp = $oldtimeout || 0;
5655                 $oldtimeout = undef;
5656                 alarm($tmp);
5657                 close($fifofh);
5658             }
5659         };
5660
5661         print "restore vma archive: $cmd\n";
5662         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5663     };
5664     my $err = $@;
5665
5666     alarm($oldtimeout) if $oldtimeout;
5667
5668     my $vollist = [];
5669     foreach my $devname (keys %$devinfo) {
5670         my $volid = $devinfo->{$devname}->{volid};
5671         push @$vollist, $volid if $volid;
5672     }
5673
5674     my $cfg = PVE::Storage::config();
5675     PVE::Storage::deactivate_volumes($cfg, $vollist);
5676
5677     unlink $mapfifo;
5678
5679     if ($err) {
5680         rmtree $tmpdir;
5681         unlink $tmpfn;
5682
5683         foreach my $devname (keys %$devinfo) {
5684             my $volid = $devinfo->{$devname}->{volid};
5685             next if !$volid;
5686             eval {
5687                 if ($volid =~ m|^/|) {
5688                     unlink $volid || die 'unlink failed\n';
5689                 } else {
5690                     PVE::Storage::vdisk_free($cfg, $volid);
5691                 }
5692                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5693             };
5694             print STDERR "unable to cleanup '$volid' - $@" if $@;
5695         }
5696         die $err;
5697     }
5698
5699     rmtree $tmpdir;
5700
5701     rename($tmpfn, $conffile) ||
5702         die "unable to commit configuration file '$conffile'\n";
5703
5704     PVE::Cluster::cfs_update(); # make sure we read new file
5705
5706     eval { rescan($vmid, 1); };
5707     warn $@ if $@;
5708 }
5709
5710 sub restore_tar_archive {
5711     my ($archive, $vmid, $user, $opts) = @_;
5712
5713     if ($archive ne '-') {
5714         my $firstfile = tar_archive_read_firstfile($archive);
5715         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5716             if $firstfile ne 'qemu-server.conf';
5717     }
5718
5719     my $storecfg = PVE::Storage::config();
5720
5721     # destroy existing data - keep empty config
5722     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5723     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5724
5725     my $tocmd = "/usr/lib/qemu-server/qmextract";
5726
5727     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5728     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5729     $tocmd .= ' --prealloc' if $opts->{prealloc};
5730     $tocmd .= ' --info' if $opts->{info};
5731
5732     # tar option "xf" does not autodetect compression when read from STDIN,
5733     # so we pipe to zcat
5734     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5735         PVE::Tools::shellquote("--to-command=$tocmd");
5736
5737     my $tmpdir = "/var/tmp/vzdumptmp$$";
5738     mkpath $tmpdir;
5739
5740     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5741     local $ENV{VZDUMP_VMID} = $vmid;
5742     local $ENV{VZDUMP_USER} = $user;
5743
5744     my $conffile = PVE::QemuConfig->config_file($vmid);
5745     my $tmpfn = "$conffile.$$.tmp";
5746
5747     # disable interrupts (always do cleanups)
5748     local $SIG{INT} =
5749         local $SIG{TERM} =
5750         local $SIG{QUIT} =
5751         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5752
5753     eval {
5754         # enable interrupts
5755         local $SIG{INT} =
5756             local $SIG{TERM} =
5757             local $SIG{QUIT} =
5758             local $SIG{HUP} =
5759             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5760
5761         if ($archive eq '-') {
5762             print "extracting archive from STDIN\n";
5763             run_command($cmd, input => "<&STDIN");
5764         } else {
5765             print "extracting archive '$archive'\n";
5766             run_command($cmd);
5767         }
5768
5769         return if $opts->{info};
5770
5771         # read new mapping
5772         my $map = {};
5773         my $statfile = "$tmpdir/qmrestore.stat";
5774         if (my $fd = IO::File->new($statfile, "r")) {
5775             while (defined (my $line = <$fd>)) {
5776                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5777                     $map->{$1} = $2 if $1;
5778                 } else {
5779                     print STDERR "unable to parse line in statfile - $line\n";
5780                 }
5781             }
5782             $fd->close();
5783         }
5784
5785         my $confsrc = "$tmpdir/qemu-server.conf";
5786
5787         my $srcfd = new IO::File($confsrc, "r") ||
5788             die "unable to open file '$confsrc'\n";
5789
5790         my $outfd = new IO::File ($tmpfn, "w") ||
5791             die "unable to write config for VM $vmid\n";
5792
5793         my $cookie = { netcount => 0 };
5794         while (defined (my $line = <$srcfd>)) {
5795             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5796         }
5797
5798         $srcfd->close();
5799         $outfd->close();
5800     };
5801     my $err = $@;
5802
5803     if ($err) {
5804
5805         unlink $tmpfn;
5806
5807         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5808
5809         die $err;
5810     }
5811
5812     rmtree $tmpdir;
5813
5814     rename $tmpfn, $conffile ||
5815         die "unable to commit configuration file '$conffile'\n";
5816
5817     PVE::Cluster::cfs_update(); # make sure we read new file
5818
5819     eval { rescan($vmid, 1); };
5820     warn $@ if $@;
5821 };
5822
5823 sub foreach_storage_used_by_vm {
5824     my ($conf, $func) = @_;
5825
5826     my $sidhash = {};
5827
5828     foreach_drive($conf, sub {
5829         my ($ds, $drive) = @_;
5830         return if drive_is_cdrom($drive);
5831
5832         my $volid = $drive->{file};
5833
5834         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5835         $sidhash->{$sid} = $sid if $sid;
5836     });
5837
5838     foreach my $sid (sort keys %$sidhash) {
5839         &$func($sid);
5840     }
5841 }
5842
5843 sub do_snapshots_with_qemu {
5844     my ($storecfg, $volid) = @_;
5845
5846     my $storage_name = PVE::Storage::parse_volume_id($volid);
5847
5848     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5849         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5850         return 1;
5851     }
5852
5853     if ($volid =~ m/\.(qcow2|qed)$/){
5854         return 1;
5855     }
5856
5857     return undef;
5858 }
5859
5860 sub qga_check_running {
5861     my ($vmid) = @_;
5862
5863     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5864     if ($@) {
5865         warn "Qemu Guest Agent are not running - $@";
5866         return 0;
5867     }
5868     return 1;
5869 }
5870
5871 sub template_create {
5872     my ($vmid, $conf, $disk) = @_;
5873
5874     my $storecfg = PVE::Storage::config();
5875
5876     foreach_drive($conf, sub {
5877         my ($ds, $drive) = @_;
5878
5879         return if drive_is_cdrom($drive);
5880         return if $disk && $ds ne $disk;
5881
5882         my $volid = $drive->{file};
5883         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5884
5885         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5886         $drive->{file} = $voliddst;
5887         $conf->{$ds} = print_drive($vmid, $drive);
5888         PVE::QemuConfig->write_config($vmid, $conf);
5889     });
5890 }
5891
5892 sub qemu_img_convert {
5893     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5894
5895     my $storecfg = PVE::Storage::config();
5896     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5897     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5898
5899     if ($src_storeid && $dst_storeid) {
5900
5901         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5902
5903         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5904         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5905
5906         my $src_format = qemu_img_format($src_scfg, $src_volname);
5907         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5908
5909         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5910         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5911
5912         my $cmd = [];
5913         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5914         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5915         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5916         if ($is_zero_initialized) {
5917             push @$cmd, "zeroinit:$dst_path";
5918         } else {
5919             push @$cmd, $dst_path;
5920         }
5921
5922         my $parser = sub {
5923             my $line = shift;
5924             if($line =~ m/\((\S+)\/100\%\)/){
5925                 my $percent = $1;
5926                 my $transferred = int($size * $percent / 100);
5927                 my $remaining = $size - $transferred;
5928
5929                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5930             }
5931
5932         };
5933
5934         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5935         my $err = $@;
5936         die "copy failed: $err" if $err;
5937     }
5938 }
5939
5940 sub qemu_img_format {
5941     my ($scfg, $volname) = @_;
5942
5943     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5944         return $1;
5945     } else {
5946         return "raw";
5947     }
5948 }
5949
5950 sub qemu_drive_mirror {
5951     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5952
5953     $jobs = {} if !$jobs;
5954
5955     my $qemu_target;
5956     my $format;
5957     $jobs->{"drive-$drive"} = {};
5958
5959     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5960         my $server = $1;
5961         my $port = $2;
5962         my $exportname = $3;
5963
5964         $format = "nbd";
5965         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5966         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5967         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5968
5969         my $pid = fork();
5970         if (!defined($pid)) {
5971             die "forking socat tunnel failed\n";
5972         } elsif ($pid == 0) {
5973             exec(@$cmd);
5974             warn "exec failed: $!\n";
5975             POSIX::_exit(-1);
5976         }
5977         $jobs->{"drive-$drive"}->{pid} = $pid;
5978
5979         my $timeout = 0;
5980         while (!-S $unixsocket) {
5981             die "nbd connection helper timed out\n"
5982                 if $timeout++ > 5;
5983             sleep 1;
5984         }
5985     } else {
5986         my $storecfg = PVE::Storage::config();
5987         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5988
5989         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5990
5991         $format = qemu_img_format($dst_scfg, $dst_volname);
5992
5993         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5994
5995         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5996     }
5997
5998     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5999     $opts->{format} = $format if $format;
6000
6001     print "drive mirror is starting for drive-$drive\n";
6002
6003     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6004
6005     if (my $err = $@) {
6006         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6007         die "mirroring error: $err";
6008     }
6009
6010     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6011 }
6012
6013 sub qemu_drive_mirror_monitor {
6014     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6015
6016     eval {
6017         my $err_complete = 0;
6018
6019         while (1) {
6020             die "storage migration timed out\n" if $err_complete > 300;
6021
6022             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6023
6024             my $running_mirror_jobs = {};
6025             foreach my $stat (@$stats) {
6026                 next if $stat->{type} ne 'mirror';
6027                 $running_mirror_jobs->{$stat->{device}} = $stat;
6028             }
6029
6030             my $readycounter = 0;
6031
6032             foreach my $job (keys %$jobs) {
6033
6034                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6035                     print "$job : finished\n";
6036                     delete $jobs->{$job};
6037                     next;
6038                 }
6039
6040                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6041
6042                 my $busy = $running_mirror_jobs->{$job}->{busy};
6043                 my $ready = $running_mirror_jobs->{$job}->{ready};
6044                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6045                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6046                     my $remaining = $total - $transferred;
6047                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6048
6049                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6050                 }
6051
6052                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6053             }
6054
6055             last if scalar(keys %$jobs) == 0;
6056
6057             if ($readycounter == scalar(keys %$jobs)) {
6058                 print "all mirroring jobs are ready \n";
6059                 last if $skipcomplete; #do the complete later
6060
6061                 if ($vmiddst && $vmiddst != $vmid) {
6062                     if ($qga) {
6063                         print "freeze filesystem\n";
6064                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6065                     } else {
6066                         print "suspend vm\n";
6067                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6068                     }
6069
6070                     # if we clone a disk for a new target vm, we don't switch the disk
6071                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6072
6073                     if ($qga) {
6074                         print "unfreeze filesystem\n";
6075                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6076                     } else {
6077                         print "resume vm\n";
6078                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6079                     }
6080
6081                     last;
6082                 } else {
6083
6084                     foreach my $job (keys %$jobs) {
6085                         # try to switch the disk if source and destination are on the same guest
6086                         print "$job: Completing block job...\n";
6087
6088                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6089                         if ($@ =~ m/cannot be completed/) {
6090                             print "$job: Block job cannot be completed, try again.\n";
6091                             $err_complete++;
6092                         }else {
6093                             print "$job: Completed successfully.\n";
6094                             $jobs->{$job}->{complete} = 1;
6095                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6096                         }
6097                     }
6098                 }
6099             }
6100             sleep 1;
6101         }
6102     };
6103     my $err = $@;
6104
6105     if ($err) {
6106         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6107         die "mirroring error: $err";
6108     }
6109
6110 }
6111
6112 sub qemu_blockjobs_cancel {
6113     my ($vmid, $jobs) = @_;
6114
6115     foreach my $job (keys %$jobs) {
6116         print "$job: Cancelling block job\n";
6117         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6118         $jobs->{$job}->{cancel} = 1;
6119     }
6120
6121     while (1) {
6122         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6123
6124         my $running_jobs = {};
6125         foreach my $stat (@$stats) {
6126             $running_jobs->{$stat->{device}} = $stat;
6127         }
6128
6129         foreach my $job (keys %$jobs) {
6130
6131             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6132                 print "$job: Done.\n";
6133                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6134                 delete $jobs->{$job};
6135             }
6136         }
6137
6138         last if scalar(keys %$jobs) == 0;
6139
6140         sleep 1;
6141     }
6142 }
6143
6144 sub qemu_blockjobs_finish_tunnel {
6145    my ($vmid, $job, $cpid) = @_;
6146
6147    return if !$cpid;
6148
6149    for (my $i = 1; $i < 20; $i++) {
6150         my $waitpid = waitpid($cpid, WNOHANG);
6151         last if (defined($waitpid) && ($waitpid == $cpid));
6152
6153         if ($i == 10) {
6154             kill(15, $cpid);
6155         } elsif ($i >= 15) {
6156             kill(9, $cpid);
6157         }
6158         sleep (1);
6159     }
6160     unlink "/run/qemu-server/$vmid.mirror-$job";
6161 }
6162
6163 sub clone_disk {
6164     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6165         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6166
6167     my $newvolid;
6168
6169     if (!$full) {
6170         print "create linked clone of drive $drivename ($drive->{file})\n";
6171         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6172         push @$newvollist, $newvolid;
6173     } else {
6174
6175         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6176         $storeid = $storage if $storage;
6177
6178         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6179         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6180
6181         print "create full clone of drive $drivename ($drive->{file})\n";
6182         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6183         push @$newvollist, $newvolid;
6184
6185         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6186
6187         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6188         if (!$running || $snapname) {
6189             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6190         } else {
6191
6192             my $kvmver = get_running_qemu_version ($vmid);
6193             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6194                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6195                     if $drive->{iothread};
6196             }
6197
6198             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6199         }
6200     }
6201
6202     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6203
6204     my $disk = $drive;
6205     $disk->{format} = undef;
6206     $disk->{file} = $newvolid;
6207     $disk->{size} = $size;
6208
6209     return $disk;
6210 }
6211
6212 # this only works if VM is running
6213 sub get_current_qemu_machine {
6214     my ($vmid) = @_;
6215
6216     my $cmd = { execute => 'query-machines', arguments => {} };
6217     my $res = vm_qmp_command($vmid, $cmd);
6218
6219     my ($current, $default);
6220     foreach my $e (@$res) {
6221         $default = $e->{name} if $e->{'is-default'};
6222         $current = $e->{name} if $e->{'is-current'};
6223     }
6224
6225     # fallback to the default machine if current is not supported by qemu
6226     return $current || $default || 'pc';
6227 }
6228
6229 sub get_running_qemu_version {
6230     my ($vmid) = @_;
6231     my $cmd = { execute => 'query-version', arguments => {} };
6232     my $res = vm_qmp_command($vmid, $cmd);
6233     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6234 }
6235
6236 sub qemu_machine_feature_enabled {
6237     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6238
6239     my $current_major;
6240     my $current_minor;
6241
6242     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6243
6244         $current_major = $3;
6245         $current_minor = $4;
6246
6247     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6248
6249         $current_major = $1;
6250         $current_minor = $2;
6251     }
6252
6253     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6254
6255
6256 }
6257
6258 sub qemu_machine_pxe {
6259     my ($vmid, $conf, $machine) = @_;
6260
6261     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6262
6263     foreach my $opt (keys %$conf) {
6264         next if $opt !~ m/^net(\d+)$/;
6265         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6266         next if !$net;
6267         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6268         return $machine.".pxe" if $romfile =~ m/pxe/;
6269         last;
6270     }
6271
6272     return $machine;
6273 }
6274
6275 sub qemu_use_old_bios_files {
6276     my ($machine_type) = @_;
6277
6278     return if !$machine_type;
6279
6280     my $use_old_bios_files = undef;
6281
6282     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6283         $machine_type = $1;
6284         $use_old_bios_files = 1;
6285     } else {
6286         my $kvmver = kvm_user_version();
6287         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6288         # load new efi bios files on migration. So this hack is required to allow
6289         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6290         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6291         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6292     }
6293
6294     return ($use_old_bios_files, $machine_type);
6295 }
6296
6297 sub create_efidisk {
6298     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6299
6300     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6301
6302     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6303     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6304     PVE::Storage::activate_volumes($storecfg, [$volid]);
6305
6306     my $path = PVE::Storage::path($storecfg, $volid);
6307     eval {
6308         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6309     };
6310     die "Copying EFI vars image failed: $@" if $@;
6311
6312     return ($volid, $vars_size);
6313 }
6314
6315 sub lspci {
6316
6317     my $devices = {};
6318
6319     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6320             my (undef, $id, $function) = @_;
6321             my $res = { id => $id, function => $function};
6322             push @{$devices->{$id}}, $res;
6323     });
6324
6325     # Entries should be sorted by functions.
6326     foreach my $id (keys %$devices) {
6327         my $dev = $devices->{$id};
6328         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6329     }
6330
6331     return $devices;
6332 }
6333
6334 sub vm_iothreads_list {
6335     my ($vmid) = @_;
6336
6337     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6338
6339     my $iothreads = {};
6340     foreach my $iothread (@$res) {
6341         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6342     }
6343
6344     return $iothreads;
6345 }
6346
6347 sub scsihw_infos {
6348     my ($conf, $drive) = @_;
6349
6350     my $maxdev = 0;
6351
6352     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6353         $maxdev = 7;
6354     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6355         $maxdev = 1;
6356     } else {
6357         $maxdev = 256;
6358     }
6359
6360     my $controller = int($drive->{index} / $maxdev);
6361     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6362
6363     return ($maxdev, $controller, $controller_prefix);
6364 }
6365
6366 sub add_hyperv_enlightenments {
6367     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6368
6369     return if $winversion < 6;
6370     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6371
6372     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6373
6374     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6375         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6376         push @$cpuFlags , 'hv_vapic';
6377         push @$cpuFlags , 'hv_time';
6378     } else {
6379         push @$cpuFlags , 'hv_spinlocks=0xffff';
6380     }
6381
6382     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6383         push @$cpuFlags , 'hv_reset';
6384         push @$cpuFlags , 'hv_vpindex';
6385         push @$cpuFlags , 'hv_runtime';
6386     }
6387
6388     if ($winversion >= 7) {
6389         push @$cpuFlags , 'hv_relaxed';
6390     }
6391 }
6392
6393 sub windows_version {
6394     my ($ostype) = @_;
6395
6396     return 0 if !$ostype;
6397
6398     my $winversion = 0;
6399
6400     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6401         $winversion = 5;
6402     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6403         $winversion = 6;
6404     } elsif ($ostype =~ m/^win(\d+)$/) {
6405         $winversion = $1;
6406     }
6407
6408     return $winversion;
6409 }
6410
6411 sub resolve_dst_disk_format {
6412         my ($storecfg, $storeid, $src_volname, $format) = @_;
6413         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6414
6415         if (!$format) {
6416             # if no target format is specified, use the source disk format as hint
6417             if ($src_volname) {
6418                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6419                 $format = qemu_img_format($scfg, $src_volname);
6420             } else {
6421                 return $defFormat;
6422             }
6423         }
6424
6425         # test if requested format is supported - else use default
6426         my $supported = grep { $_ eq $format } @$validFormats;
6427         $format = $defFormat if !$supported;
6428         return $format;
6429 }
6430
6431 # bash completion helper
6432
6433 sub complete_backup_archives {
6434     my ($cmdname, $pname, $cvalue) = @_;
6435
6436     my $cfg = PVE::Storage::config();
6437
6438     my $storeid;
6439
6440     if ($cvalue =~ m/^([^:]+):/) {
6441         $storeid = $1;
6442     }
6443
6444     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6445
6446     my $res = [];
6447     foreach my $id (keys %$data) {
6448         foreach my $item (@{$data->{$id}}) {
6449             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6450             push @$res, $item->{volid} if defined($item->{volid});
6451         }
6452     }
6453
6454     return $res;
6455 }
6456
6457 my $complete_vmid_full = sub {
6458     my ($running) = @_;
6459
6460     my $idlist = vmstatus();
6461
6462     my $res = [];
6463
6464     foreach my $id (keys %$idlist) {
6465         my $d = $idlist->{$id};
6466         if (defined($running)) {
6467             next if $d->{template};
6468             next if $running && $d->{status} ne 'running';
6469             next if !$running && $d->{status} eq 'running';
6470         }
6471         push @$res, $id;
6472
6473     }
6474     return $res;
6475 };
6476
6477 sub complete_vmid {
6478     return &$complete_vmid_full();
6479 }
6480
6481 sub complete_vmid_stopped {
6482     return &$complete_vmid_full(0);
6483 }
6484
6485 sub complete_vmid_running {
6486     return &$complete_vmid_full(1);
6487 }
6488
6489 sub complete_storage {
6490
6491     my $cfg = PVE::Storage::config();
6492     my $ids = $cfg->{ids};
6493
6494     my $res = [];
6495     foreach my $sid (keys %$ids) {
6496         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6497         next if !$ids->{$sid}->{content}->{images};
6498         push @$res, $sid;
6499     }
6500
6501     return $res;
6502 }
6503
6504 sub nbd_stop {
6505     my ($vmid) = @_;
6506
6507     vm_mon_cmd($vmid, 'nbd-server-stop');
6508 }
6509
6510 1;