PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120     'Skylake-Client' => 'GenuineIntel',
 121
 122     # AMD CPUs
 123     athlon => 'AuthenticAMD',
 124     phenom  => 'AuthenticAMD',
 125     Opteron_G1  => 'AuthenticAMD',
 126     Opteron_G2  => 'AuthenticAMD',
 127     Opteron_G3  => 'AuthenticAMD',
 128     Opteron_G4  => 'AuthenticAMD',
 129     Opteron_G5  => 'AuthenticAMD',
 130
 131     # generic types, use vendor from host node
 132     host => 'default',
 133     kvm32 => 'default',
 134     kvm64 => 'default',
 135     qemu32 => 'default',
 136     qemu64 => 'default',
 137 };
 138
 139 my $cpu_fmt = {
 140     cputype => {
 141         description => "Emulated CPU type.",
 142         type => 'string',
 143         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 218         minimum => 0,
 219         maximum => 500000,
 220         default => 1024,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 247         enum => PVE::Tools::kvmkeymaplist(),
 248         default => 'en-us',
 249     },
 250     name => {
 251         optional => 1,
 252         type => 'string', format => 'dns-name',
 253         description => "Set a name for the VM. Only used on the configuration web interface.",
 254     },
 255     scsihw => {
 256         optional => 1,
 257         type => 'string',
 258         description => "SCSI controller model",
 259         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 260         default => 'lsi',
 261     },
 262     description => {
 263         optional => 1,
 264         type => 'string',
 265         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 266     },
 267     ostype => {
 268         optional => 1,
 269         type => 'string',
 270         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 271         description => "Specify guest operating system.",
 272         verbose_description => <<EODESC,
 273 Specify guest operating system. This is used to enable special
 274 optimization/features for specific operating systems:
 275
 276 [horizontal]
 277 other;; unspecified OS
 278 wxp;; Microsoft Windows XP
 279 w2k;; Microsoft Windows 2000
 280 w2k3;; Microsoft Windows 2003
 281 w2k8;; Microsoft Windows 2008
 282 wvista;; Microsoft Windows Vista
 283 win7;; Microsoft Windows 7
 284 win8;; Microsoft Windows 8/2012
 285 l24;; Linux 2.4 Kernel
 286 l26;; Linux 2.6/3.X Kernel
 287 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 288 EODESC
 289     },
 290     boot => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 294         pattern => '[acdn]{1,4}',
 295         default => 'cdn',
 296     },
 297     bootdisk => {
 298         optional => 1,
 299         type => 'string', format => 'pve-qm-bootdisk',
 300         description => "Enable booting from specified disk.",
 301         pattern => '(ide|sata|scsi|virtio)\d+',
 302     },
 303     smp => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "The number of CPUs. Please use option -sockets instead.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     sockets => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "The number of CPU sockets.",
 314         minimum => 1,
 315         default => 1,
 316     },
 317     cores => {
 318         optional => 1,
 319         type => 'integer',
 320         description => "The number of cores per socket.",
 321         minimum => 1,
 322         default => 1,
 323     },
 324     numa => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Enable/disable NUMA.",
 328         default => 0,
 329     },
 330     hugepages => {
 331         optional => 1,
 332         type => 'string',
 333         description => "Enable/disable hugepages memory.",
 334         enum => [qw(any 2 1024)],
 335     },
 336     vcpus => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Number of hotplugged vcpus.",
 340         minimum => 1,
 341         default => 0,
 342     },
 343     acpi => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable ACPI.",
 347         default => 1,
 348     },
 349     agent => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable Qemu GuestAgent.",
 353         default => 0,
 354     },
 355     kvm => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable KVM hardware virtualization.",
 359         default => 1,
 360     },
 361     tdf => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable time drift fix.",
 365         default => 0,
 366     },
 367     localtime => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 371     },
 372     freeze => {
 373         optional => 1,
 374         type => 'boolean',
 375         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 376     },
 377     vga => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Select the VGA type.",
 381         verbose_description => "Select the VGA type. If you want to use high resolution" .
 382             " modes (>= 1280x1024x16) then you should use the options " .
 383             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 384             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 385             "display sever. For win* OS you can select how many independent " .
 386             "displays you want, Linux guests can add displays them self. " .
 387             "You can also run without any graphic card, using a serial device" .
 388             " as terminal.",
 389         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 390     },
 391     watchdog => {
 392         optional => 1,
 393         type => 'string', format => 'pve-qm-watchdog',
 394         description => "Create a virtual hardware watchdog device.",
 395         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 396             " (by a guest action), the watchdog must be periodically polled " .
 397             "by an agent inside the guest or else the watchdog will reset " .
 398             "the guest (or execute the respective action specified)",
 399     },
 400     startdate => {
 401         optional => 1,
 402         type => 'string',
 403         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 404         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 405         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 406         default => 'now',
 407     },
 408     startup =>  get_standard_option('pve-startup-order'),
 409     template => {
 410         optional => 1,
 411         type => 'boolean',
 412         description => "Enable/disable Template.",
 413         default => 0,
 414     },
 415     args => {
 416         optional => 1,
 417         type => 'string',
 418         description => "Arbitrary arguments passed to kvm.",
 419         verbose_description => <<EODESCR,
 420 Arbitrary arguments passed to kvm, for example:
 421
 422 args: -no-reboot -no-hpet
 423
 424 NOTE: this option is for experts only.
 425 EODESCR
 426     },
 427     tablet => {
 428         optional => 1,
 429         type => 'boolean',
 430         default => 1,
 431         description => "Enable/disable the USB tablet device.",
 432         verbose_description => "Enable/disable the USB tablet device. This device is " .
 433             "usually needed to allow absolute mouse positioning with VNC. " .
 434             "Else the mouse runs out of sync with normal VNC clients. " .
 435             "If you're running lots of console-only guests on one host, " .
 436             "you may consider disabling this to save some context switches. " .
 437             "This is turned off by default if you use spice (-vga=qxl).",
 438     },
 439     migrate_speed => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 443         minimum => 0,
 444         default => 0,
 445     },
 446     migrate_downtime => {
 447         optional => 1,
 448         type => 'number',
 449         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 450         minimum => 0,
 451         default => 0.1,
 452     },
 453     cdrom => {
 454         optional => 1,
 455         type => 'string', format => 'pve-qm-ide',
 456         typetext => '<volume>',
 457         description => "This is an alias for option -ide2",
 458     },
 459     cpu => {
 460         optional => 1,
 461         description => "Emulated CPU type.",
 462         type => 'string',
 463         format => $cpu_fmt,
 464     },
 465     parent => get_standard_option('pve-snapshot-name', {
 466         optional => 1,
 467         description => "Parent snapshot name. This is used internally, and should not be modified.",
 468     }),
 469     snaptime => {
 470         optional => 1,
 471         description => "Timestamp for snapshots.",
 472         type => 'integer',
 473         minimum => 0,
 474     },
 475     vmstate => {
 476         optional => 1,
 477         type => 'string', format => 'pve-volume-id',
 478         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 479     },
 480     vmstatestorage => get_standard_option('pve-storage-id', {
 481         description => "Default storage for VM state volumes/files.",
 482         optional => 1,
 483     }),
 484     machine => {
 485         description => "Specific the Qemu machine type.",
 486         type => 'string',
 487         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 488         maxLength => 40,
 489         optional => 1,
 490     },
 491     smbios1 => {
 492         description => "Specify SMBIOS type 1 fields.",
 493         type => 'string', format => 'pve-qm-smbios1',
 494         maxLength => 256,
 495         optional => 1,
 496     },
 497     protection => {
 498         optional => 1,
 499         type => 'boolean',
 500         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 501         default => 0,
 502     },
 503     bios => {
 504         optional => 1,
 505         type => 'string',
 506         enum => [ qw(seabios ovmf) ],
 507         description => "Select BIOS implementation.",
 508         default => 'seabios',
 509     },
 510 };
 511
 512 # what about other qemu settings ?
 513 #cpu => 'string',
 514 #machine => 'string',
 515 #fda => 'file',
 516 #fdb => 'file',
 517 #mtdblock => 'file',
 518 #sd => 'file',
 519 #pflash => 'file',
 520 #snapshot => 'bool',
 521 #bootp => 'file',
 522 ##tftp => 'dir',
 523 ##smb => 'dir',
 524 #kernel => 'file',
 525 #append => 'string',
 526 #initrd => 'file',
 527 ##soundhw => 'string',
 528
 529 while (my ($k, $v) = each %$confdesc) {
 530     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 531 }
 532
 533 my $MAX_IDE_DISKS = 4;
 534 my $MAX_SCSI_DISKS = 14;
 535 my $MAX_VIRTIO_DISKS = 16;
 536 my $MAX_SATA_DISKS = 6;
 537 my $MAX_USB_DEVICES = 5;
 538 my $MAX_NETS = 32;
 539 my $MAX_UNUSED_DISKS = 8;
 540 my $MAX_HOSTPCI_DEVICES = 4;
 541 my $MAX_SERIAL_PORTS = 4;
 542 my $MAX_PARALLEL_PORTS = 3;
 543 my $MAX_NUMA = 8;
 544
 545 my $numa_fmt = {
 546     cpus => {
 547         type => "string",
 548         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 549         description => "CPUs accessing this NUMA node.",
 550         format_description => "id[-id];...",
 551     },
 552     memory => {
 553         type => "number",
 554         description => "Amount of memory this NUMA node provides.",
 555         optional => 1,
 556     },
 557     hostnodes => {
 558         type => "string",
 559         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 560         description => "Host NUMA nodes to use.",
 561         format_description => "id[-id];...",
 562         optional => 1,
 563     },
 564     policy => {
 565         type => 'string',
 566         enum => [qw(preferred bind interleave)],
 567         description => "NUMA allocation policy.",
 568         optional => 1,
 569     },
 570 };
 571 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 572 my $numadesc = {
 573     optional => 1,
 574     type => 'string', format => $numa_fmt,
 575     description => "NUMA topology.",
 576 };
 577 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 578
 579 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 580     $confdesc->{"numa$i"} = $numadesc;
 581 }
 582
 583 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 584                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 585                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 586 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 587
 588 my $net_fmt_bridge_descr = <<__EOD__;
 589 Bridge to attach the network device to. The Proxmox VE standard bridge
 590 is called 'vmbr0'.
 591
 592 If you do not specify a bridge, we create a kvm user (NATed) network
 593 device, which provides DHCP and DNS services. The following addresses
 594 are used:
 595
 596  10.0.2.2   Gateway
 597  10.0.2.3   DNS Server
 598  10.0.2.4   SMB Server
 599
 600 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 601 __EOD__
 602
 603 my $net_fmt = {
 604     macaddr => {
 605         type => 'string',
 606         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 607         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 608         format_description => "XX:XX:XX:XX:XX:XX",
 609         optional => 1,
 610     },
 611     model => {
 612         type => 'string',
 613         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 614         enum => $nic_model_list,
 615         default_key => 1,
 616     },
 617     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 618     bridge => {
 619         type => 'string',
 620         description => $net_fmt_bridge_descr,
 621         format_description => 'bridge',
 622         optional => 1,
 623     },
 624     queues => {
 625         type => 'integer',
 626         minimum => 0, maximum => 16,
 627         description => 'Number of packet queues to be used on the device.',
 628         optional => 1,
 629     },
 630     rate => {
 631         type => 'number',
 632         minimum => 0,
 633         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 634         optional => 1,
 635     },
 636     tag => {
 637         type => 'integer',
 638         minimum => 1, maximum => 4094,
 639         description => 'VLAN tag to apply to packets on this interface.',
 640         optional => 1,
 641     },
 642     trunks => {
 643         type => 'string',
 644         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 645         description => 'VLAN trunks to pass through this interface.',
 646         format_description => 'vlanid[;vlanid...]',
 647         optional => 1,
 648     },
 649     firewall => {
 650         type => 'boolean',
 651         description => 'Whether this interface should be protected by the firewall.',
 652         optional => 1,
 653     },
 654     link_down => {
 655         type => 'boolean',
 656         description => 'Whether this interface should be disconnected (like pulling the plug).',
 657         optional => 1,
 658     },
 659 };
 660
 661 my $netdesc = {
 662     optional => 1,
 663     type => 'string', format => $net_fmt,
 664     description => "Specify network devices.",
 665 };
 666
 667 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 668
 669 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 670     $confdesc->{"net$i"} = $netdesc;
 671 }
 672
 673 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 674 sub verify_volume_id_or_qm_path {
 675     my ($volid, $noerr) = @_;
 676
 677     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 678         return $volid;
 679     }
 680
 681     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 682     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 683     if ($@) {
 684         return undef if $noerr;
 685         die $@;
 686     }
 687     return $volid;
 688 }
 689
 690 my $drivename_hash;
 691
 692 my %drivedesc_base = (
 693     volume => { alias => 'file' },
 694     file => {
 695         type => 'string',
 696         format => 'pve-volume-id-or-qm-path',
 697         default_key => 1,
 698         format_description => 'volume',
 699         description => "The drive's backing volume.",
 700     },
 701     media => {
 702         type => 'string',
 703         enum => [qw(cdrom disk)],
 704         description => "The drive's media type.",
 705         default => 'disk',
 706         optional => 1
 707     },
 708     cyls => {
 709         type => 'integer',
 710         description => "Force the drive's physical geometry to have a specific cylinder count.",
 711         optional => 1
 712     },
 713     heads => {
 714         type => 'integer',
 715         description => "Force the drive's physical geometry to have a specific head count.",
 716         optional => 1
 717     },
 718     secs => {
 719         type => 'integer',
 720         description => "Force the drive's physical geometry to have a specific sector count.",
 721         optional => 1
 722     },
 723     trans => {
 724         type => 'string',
 725         enum => [qw(none lba auto)],
 726         description => "Force disk geometry bios translation mode.",
 727         optional => 1,
 728     },
 729     snapshot => {
 730         type => 'boolean',
 731         description => "Whether the drive should be included when making snapshots.",
 732         optional => 1,
 733     },
 734     cache => {
 735         type => 'string',
 736         enum => [qw(none writethrough writeback unsafe directsync)],
 737         description => "The drive's cache mode",
 738         optional => 1,
 739     },
 740     format => get_standard_option('pve-qm-image-format'),
 741     size => {
 742         type => 'string',
 743         format => 'disk-size',
 744         format_description => 'DiskSize',
 745         description => "Disk size. This is purely informational and has no effect.",
 746         optional => 1,
 747     },
 748     backup => {
 749         type => 'boolean',
 750         description => "Whether the drive should be included when making backups.",
 751         optional => 1,
 752     },
 753     replicate => {
 754         type => 'boolean',
 755         description => 'Whether the drive should considered for replication jobs.',
 756         optional => 1,
 757         default => 1,
 758     },
 759     rerror => {
 760         type => 'string',
 761         enum => [qw(ignore report stop)],
 762         description => 'Read error action.',
 763         optional => 1,
 764     },
 765     werror => {
 766         type => 'string',
 767         enum => [qw(enospc ignore report stop)],
 768         description => 'Write error action.',
 769         optional => 1,
 770     },
 771     aio => {
 772         type => 'string',
 773         enum => [qw(native threads)],
 774         description => 'AIO type to use.',
 775         optional => 1,
 776     },
 777     discard => {
 778         type => 'string',
 779         enum => [qw(ignore on)],
 780         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 781         optional => 1,
 782     },
 783     detect_zeroes => {
 784         type => 'boolean',
 785         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 786         optional => 1,
 787     },
 788     serial => {
 789         type => 'string',
 790         format => 'urlencoded',
 791         format_description => 'serial',
 792         maxLength => 20*3, # *3 since it's %xx url enoded
 793         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 794         optional => 1,
 795     }
 796 );
 797
 798 my %iothread_fmt = ( iothread => {
 799         type => 'boolean',
 800         description => "Whether to use iothreads for this drive",
 801         optional => 1,
 802 });
 803
 804 my %model_fmt = (
 805     model => {
 806         type => 'string',
 807         format => 'urlencoded',
 808         format_description => 'model',
 809         maxLength => 40*3, # *3 since it's %xx url enoded
 810         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 811         optional => 1,
 812     },
 813 );
 814
 815 my %queues_fmt = (
 816     queues => {
 817         type => 'integer',
 818         description => "Number of queues.",
 819         minimum => 2,
 820         optional => 1
 821     }
 822 );
 823
 824 my %scsiblock_fmt = (
 825     scsiblock => {
 826         type => 'boolean',
 827         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 828         optional => 1,
 829         default => 0,
 830     },
 831 );
 832
 833 my $add_throttle_desc = sub {
 834     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 835     my $d = {
 836         type => $type,
 837         format_description => $unit,
 838         description => "Maximum $what in $longunit.",
 839         optional => 1,
 840     };
 841     $d->{minimum} = $minimum if defined($minimum);
 842     $drivedesc_base{$key} = $d;
 843 };
 844 # throughput: (leaky bucket)
 845 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 846 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 847 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 848 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 849 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 850 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 851 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 852 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 853 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 854
 855 # pools: (pool of IO before throttling starts taking effect)
 856 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 857 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 858 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 859 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 860 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 861 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 862
 863 # burst lengths
 864 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 865 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 866 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 867 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 868 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 869 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 870
 871 # legacy support
 872 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 873 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 874 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 875 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 876
 877 my $ide_fmt = {
 878     %drivedesc_base,
 879     %model_fmt,
 880 };
 881 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 882
 883 my $idedesc = {
 884     optional => 1,
 885     type => 'string', format => $ide_fmt,
 886     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 887 };
 888 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 889
 890 my $scsi_fmt = {
 891     %drivedesc_base,
 892     %iothread_fmt,
 893     %queues_fmt,
 894     %scsiblock_fmt,
 895 };
 896 my $scsidesc = {
 897     optional => 1,
 898     type => 'string', format => $scsi_fmt,
 899     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 900 };
 901 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 902
 903 my $sata_fmt = {
 904     %drivedesc_base,
 905 };
 906 my $satadesc = {
 907     optional => 1,
 908     type => 'string', format => $sata_fmt,
 909     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 910 };
 911 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 912
 913 my $virtio_fmt = {
 914     %drivedesc_base,
 915     %iothread_fmt,
 916 };
 917 my $virtiodesc = {
 918     optional => 1,
 919     type => 'string', format => $virtio_fmt,
 920     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 921 };
 922 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 923
 924 my $alldrive_fmt = {
 925     %drivedesc_base,
 926     %iothread_fmt,
 927     %model_fmt,
 928     %queues_fmt,
 929     %scsiblock_fmt,
 930 };
 931
 932 my $efidisk_fmt = {
 933     volume => { alias => 'file' },
 934     file => {
 935         type => 'string',
 936         format => 'pve-volume-id-or-qm-path',
 937         default_key => 1,
 938         format_description => 'volume',
 939         description => "The drive's backing volume.",
 940     },
 941     format => get_standard_option('pve-qm-image-format'),
 942     size => {
 943         type => 'string',
 944         format => 'disk-size',
 945         format_description => 'DiskSize',
 946         description => "Disk size. This is purely informational and has no effect.",
 947         optional => 1,
 948     },
 949 };
 950
 951 my $efidisk_desc = {
 952     optional => 1,
 953     type => 'string', format => $efidisk_fmt,
 954     description => "Configure a Disk for storing EFI vars",
 955 };
 956
 957 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 958
 959 my $usb_fmt = {
 960     host => {
 961         default_key => 1,
 962         type => 'string', format => 'pve-qm-usb-device',
 963         format_description => 'HOSTUSBDEVICE|spice',
 964         description => <<EODESCR,
 965 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 966
 967  'bus-port(.port)*' (decimal numbers) or
 968  'vendor_id:product_id' (hexadeciaml numbers) or
 969  'spice'
 970
 971 You can use the 'lsusb -t' command to list existing usb devices.
 972
 973 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 974
 975 The value 'spice' can be used to add a usb redirection devices for spice.
 976 EODESCR
 977     },
 978     usb3 => {
 979         optional => 1,
 980         type => 'boolean',
 981         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 982         default => 0,
 983     },
 984 };
 985
 986 my $usbdesc = {
 987     optional => 1,
 988     type => 'string', format => $usb_fmt,
 989     description => "Configure an USB device (n is 0 to 4).",
 990 };
 991 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 992
 993 # NOTE: the match-groups of this regex are used in parse_hostpci
 994 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 995 my $hostpci_fmt = {
 996     host => {
 997         default_key => 1,
 998         type => 'string',
 999         pattern => qr/$PCIRE(;$PCIRE)*/,
1000         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1001         description => <<EODESCR,
1002 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1003 of PCI virtual functions of the host. HOSTPCIID syntax is:
1004
1005 'bus:dev.func' (hexadecimal numbers)
1006
1007 You can us the 'lspci' command to list existing PCI devices.
1008 EODESCR
1009     },
1010     rombar => {
1011         type => 'boolean',
1012         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1013         optional => 1,
1014         default => 1,
1015     },
1016     romfile => {
1017         type => 'string',
1018         pattern => '[^,;]+',
1019         format_description => 'string',
1020         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1021         optional => 1,
1022     },
1023     pcie => {
1024         type => 'boolean',
1025         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1026         optional => 1,
1027         default => 0,
1028     },
1029     'x-vga' => {
1030         type => 'boolean',
1031         description =>  "Enable vfio-vga device support.",
1032         optional => 1,
1033         default => 0,
1034     },
1035 };
1036 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1037
1038 my $hostpcidesc = {
1039         optional => 1,
1040         type => 'string', format => 'pve-qm-hostpci',
1041         description => "Map host PCI devices into guest.",
1042         verbose_description =>  <<EODESCR,
1043 Map host PCI devices into guest.
1044
1045 NOTE: This option allows direct access to host hardware. So it is no longer
1046 possible to migrate such machines - use with special care.
1047
1048 CAUTION: Experimental! User reported problems with this option.
1049 EODESCR
1050 };
1051 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1052
1053 my $serialdesc = {
1054         optional => 1,
1055         type => 'string',
1056         pattern => '(/dev/.+|socket)',
1057         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1058         verbose_description =>  <<EODESCR,
1059 Create a serial device inside the VM (n is 0 to 3), and pass through a
1060 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1061 host side (use 'qm terminal' to open a terminal connection).
1062
1063 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1064
1065 CAUTION: Experimental! User reported problems with this option.
1066 EODESCR
1067 };
1068
1069 my $paralleldesc= {
1070         optional => 1,
1071         type => 'string',
1072         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1073         description =>  "Map host parallel devices (n is 0 to 2).",
1074         verbose_description =>  <<EODESCR,
1075 Map host parallel devices (n is 0 to 2).
1076
1077 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1078
1079 CAUTION: Experimental! User reported problems with this option.
1080 EODESCR
1081 };
1082
1083 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1084     $confdesc->{"parallel$i"} = $paralleldesc;
1085 }
1086
1087 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1088     $confdesc->{"serial$i"} = $serialdesc;
1089 }
1090
1091 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1092     $confdesc->{"hostpci$i"} = $hostpcidesc;
1093 }
1094
1095 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1096     $drivename_hash->{"ide$i"} = 1;
1097     $confdesc->{"ide$i"} = $idedesc;
1098 }
1099
1100 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1101     $drivename_hash->{"sata$i"} = 1;
1102     $confdesc->{"sata$i"} = $satadesc;
1103 }
1104
1105 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1106     $drivename_hash->{"scsi$i"} = 1;
1107     $confdesc->{"scsi$i"} = $scsidesc ;
1108 }
1109
1110 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1111     $drivename_hash->{"virtio$i"} = 1;
1112     $confdesc->{"virtio$i"} = $virtiodesc;
1113 }
1114
1115 $drivename_hash->{efidisk0} = 1;
1116 $confdesc->{efidisk0} = $efidisk_desc;
1117
1118 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1119     $confdesc->{"usb$i"} = $usbdesc;
1120 }
1121
1122 my $unuseddesc = {
1123     optional => 1,
1124     type => 'string', format => 'pve-volume-id',
1125     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1126 };
1127
1128 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1129     $confdesc->{"unused$i"} = $unuseddesc;
1130 }
1131
1132 my $kvm_api_version = 0;
1133
1134 sub kvm_version {
1135
1136     return $kvm_api_version if $kvm_api_version;
1137
1138     my $fh = IO::File->new("</dev/kvm") ||
1139         return 0;
1140
1141     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1142         $kvm_api_version = $v;
1143     }
1144
1145     $fh->close();
1146
1147     return  $kvm_api_version;
1148 }
1149
1150 my $kvm_user_version;
1151
1152 sub kvm_user_version {
1153
1154     return $kvm_user_version if $kvm_user_version;
1155
1156     $kvm_user_version = 'unknown';
1157
1158     my $code = sub {
1159         my $line = shift;
1160         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1161             $kvm_user_version = $2;
1162         }
1163     };
1164
1165     eval { run_command("kvm -version", outfunc => $code); };
1166     warn $@ if $@;
1167
1168     return $kvm_user_version;
1169
1170 }
1171
1172 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1173
1174 sub valid_drive_names {
1175     # order is important - used to autoselect boot disk
1176     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1177             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1178             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1179             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1180             'efidisk0');
1181 }
1182
1183 sub is_valid_drivename {
1184     my $dev = shift;
1185
1186     return defined($drivename_hash->{$dev});
1187 }
1188
1189 sub option_exists {
1190     my $key = shift;
1191     return defined($confdesc->{$key});
1192 }
1193
1194 sub nic_models {
1195     return $nic_model_list;
1196 }
1197
1198 sub os_list_description {
1199
1200     return {
1201         other => 'Other',
1202         wxp => 'Windows XP',
1203         w2k => 'Windows 2000',
1204         w2k3 =>, 'Windows 2003',
1205         w2k8 => 'Windows 2008',
1206         wvista => 'Windows Vista',
1207         win7 => 'Windows 7',
1208         win8 => 'Windows 8/2012',
1209         win10 => 'Windows 10/2016',
1210         l24 => 'Linux 2.4',
1211         l26 => 'Linux 2.6',
1212     };
1213 }
1214
1215 my $cdrom_path;
1216
1217 sub get_cdrom_path {
1218
1219     return  $cdrom_path if $cdrom_path;
1220
1221     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1222     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1223     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1224 }
1225
1226 sub get_iso_path {
1227     my ($storecfg, $vmid, $cdrom) = @_;
1228
1229     if ($cdrom eq 'cdrom') {
1230         return get_cdrom_path();
1231     } elsif ($cdrom eq 'none') {
1232         return '';
1233     } elsif ($cdrom =~ m|^/|) {
1234         return $cdrom;
1235     } else {
1236         return PVE::Storage::path($storecfg, $cdrom);
1237     }
1238 }
1239
1240 # try to convert old style file names to volume IDs
1241 sub filename_to_volume_id {
1242     my ($vmid, $file, $media) = @_;
1243
1244     if (!($file eq 'none' || $file eq 'cdrom' ||
1245           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1246
1247         return undef if $file =~ m|/|;
1248
1249         if ($media && $media eq 'cdrom') {
1250             $file = "local:iso/$file";
1251         } else {
1252             $file = "local:$vmid/$file";
1253         }
1254     }
1255
1256     return $file;
1257 }
1258
1259 sub verify_media_type {
1260     my ($opt, $vtype, $media) = @_;
1261
1262     return if !$media;
1263
1264     my $etype;
1265     if ($media eq 'disk') {
1266         $etype = 'images';
1267     } elsif ($media eq 'cdrom') {
1268         $etype = 'iso';
1269     } else {
1270         die "internal error";
1271     }
1272
1273     return if ($vtype eq $etype);
1274
1275     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1276 }
1277
1278 sub cleanup_drive_path {
1279     my ($opt, $storecfg, $drive) = @_;
1280
1281     # try to convert filesystem paths to volume IDs
1282
1283     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1284         ($drive->{file} !~ m|^/dev/.+|) &&
1285         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1286         ($drive->{file} !~ m/^\d+$/)) {
1287         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1288         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1289         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1290         verify_media_type($opt, $vtype, $drive->{media});
1291         $drive->{file} = $volid;
1292     }
1293
1294     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1295 }
1296
1297 sub parse_hotplug_features {
1298     my ($data) = @_;
1299
1300     my $res = {};
1301
1302     return $res if $data eq '0';
1303
1304     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1305
1306     foreach my $feature (PVE::Tools::split_list($data)) {
1307         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1308             $res->{$1} = 1;
1309         } else {
1310             die "invalid hotplug feature '$feature'\n";
1311         }
1312     }
1313     return $res;
1314 }
1315
1316 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1317 sub pve_verify_hotplug_features {
1318     my ($value, $noerr) = @_;
1319
1320     return $value if parse_hotplug_features($value);
1321
1322     return undef if $noerr;
1323
1324     die "unable to parse hotplug option\n";
1325 }
1326
1327 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1328 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1329 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1330 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1331 #        [,iothread=on][,serial=serial][,model=model]
1332
1333 sub parse_drive {
1334     my ($key, $data) = @_;
1335
1336     my ($interface, $index);
1337
1338     if ($key =~ m/^([^\d]+)(\d+)$/) {
1339         $interface = $1;
1340         $index = $2;
1341     } else {
1342         return undef;
1343     }
1344
1345     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1346                                      : $confdesc->{$key}->{format};
1347     if (!$desc) {
1348         warn "invalid drive key: $key\n";
1349         return undef;
1350     }
1351     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1352     return undef if !$res;
1353     $res->{interface} = $interface;
1354     $res->{index} = $index;
1355
1356     my $error = 0;
1357     foreach my $opt (qw(bps bps_rd bps_wr)) {
1358         if (my $bps = defined(delete $res->{$opt})) {
1359             if (defined($res->{"m$opt"})) {
1360                 warn "both $opt and m$opt specified\n";
1361                 ++$error;
1362                 next;
1363             }
1364             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1365         }
1366     }
1367
1368     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1369     for my $requirement (
1370         [mbps_max => 'mbps'],
1371         [mbps_rd_max => 'mbps_rd'],
1372         [mbps_wr_max => 'mbps_wr'],
1373         [miops_max => 'miops'],
1374         [miops_rd_max => 'miops_rd'],
1375         [miops_wr_max => 'miops_wr'],
1376         [bps_max_length => 'mbps_max'],
1377         [bps_rd_max_length => 'mbps_rd_max'],
1378         [bps_wr_max_length => 'mbps_wr_max'],
1379         [iops_max_length => 'iops_max'],
1380         [iops_rd_max_length => 'iops_rd_max'],
1381         [iops_wr_max_length => 'iops_wr_max']) {
1382         my ($option, $requires) = @$requirement;
1383         if ($res->{$option} && !$res->{$requires}) {
1384             warn "$option requires $requires\n";
1385             ++$error;
1386         }
1387     }
1388
1389     return undef if $error;
1390
1391     return undef if $res->{mbps_rd} && $res->{mbps};
1392     return undef if $res->{mbps_wr} && $res->{mbps};
1393     return undef if $res->{iops_rd} && $res->{iops};
1394     return undef if $res->{iops_wr} && $res->{iops};
1395
1396     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1397         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1398         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1399         return undef if $res->{interface} eq 'virtio';
1400     }
1401
1402     if (my $size = $res->{size}) {
1403         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1404     }
1405
1406     return $res;
1407 }
1408
1409 sub print_drive {
1410     my ($vmid, $drive) = @_;
1411     my $data = { %$drive };
1412     delete $data->{$_} for qw(index interface);
1413     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1414 }
1415
1416 sub scsi_inquiry {
1417     my($fh, $noerr) = @_;
1418
1419     my $SG_IO = 0x2285;
1420     my $SG_GET_VERSION_NUM = 0x2282;
1421
1422     my $versionbuf = "\x00" x 8;
1423     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1424     if (!$ret) {
1425         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1426         return undef;
1427     }
1428     my $version = unpack("I", $versionbuf);
1429     if ($version < 30000) {
1430         die "scsi generic interface too old\n"  if !$noerr;
1431         return undef;
1432     }
1433
1434     my $buf = "\x00" x 36;
1435     my $sensebuf = "\x00" x 8;
1436     my $cmd = pack("C x3 C x1", 0x12, 36);
1437
1438     # see /usr/include/scsi/sg.h
1439     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1440
1441     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1442                       length($sensebuf), 0, length($buf), $buf,
1443                       $cmd, $sensebuf, 6000);
1444
1445     $ret = ioctl($fh, $SG_IO, $packet);
1446     if (!$ret) {
1447         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1448         return undef;
1449     }
1450
1451     my @res = unpack($sg_io_hdr_t, $packet);
1452     if ($res[17] || $res[18]) {
1453         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1454         return undef;
1455     }
1456
1457     my $res = {};
1458     (my $byte0, my $byte1, $res->{vendor},
1459      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1460
1461     $res->{removable} = $byte1 & 128 ? 1 : 0;
1462     $res->{type} = $byte0 & 31;
1463
1464     return $res;
1465 }
1466
1467 sub path_is_scsi {
1468     my ($path) = @_;
1469
1470     my $fh = IO::File->new("+<$path") || return undef;
1471     my $res = scsi_inquiry($fh, 1);
1472     close($fh);
1473
1474     return $res;
1475 }
1476
1477 sub machine_type_is_q35 {
1478     my ($conf) = @_;
1479
1480     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1481 }
1482
1483 sub print_tabletdevice_full {
1484     my ($conf) = @_;
1485
1486     my $q35 = machine_type_is_q35($conf);
1487
1488     # we use uhci for old VMs because tablet driver was buggy in older qemu
1489     my $usbbus = $q35 ? "ehci" : "uhci";
1490
1491     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1492 }
1493
1494 sub print_drivedevice_full {
1495     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1496
1497     my $device = '';
1498     my $maxdev = 0;
1499
1500     if ($drive->{interface} eq 'virtio') {
1501         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1502         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1503         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1504     } elsif ($drive->{interface} eq 'scsi') {
1505
1506         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1507         my $unit = $drive->{index} % $maxdev;
1508         my $devicetype = 'hd';
1509         my $path = '';
1510         if (drive_is_cdrom($drive)) {
1511             $devicetype = 'cd';
1512         } else {
1513             if ($drive->{file} =~ m|^/|) {
1514                 $path = $drive->{file};
1515                 if (my $info = path_is_scsi($path)) {
1516                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1517                         $devicetype = 'block';
1518                     } elsif ($info->{type} == 1) { # tape
1519                         $devicetype = 'generic';
1520                     }
1521                 }
1522             } else {
1523                  $path = PVE::Storage::path($storecfg, $drive->{file});
1524             }
1525
1526             if($path =~ m/^iscsi\:\/\//){
1527                 $devicetype = 'generic';
1528             }
1529         }
1530
1531         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1532             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1533         } else {
1534             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1535         }
1536
1537     } elsif ($drive->{interface} eq 'ide'){
1538         $maxdev = 2;
1539         my $controller = int($drive->{index} / $maxdev);
1540         my $unit = $drive->{index} % $maxdev;
1541         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1542
1543         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1544         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1545             $model = URI::Escape::uri_unescape($model);
1546             $device .= ",model=$model";
1547         }
1548     } elsif ($drive->{interface} eq 'sata'){
1549         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1550         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1551         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1552     } elsif ($drive->{interface} eq 'usb') {
1553         die "implement me";
1554         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1555     } else {
1556         die "unsupported interface type";
1557     }
1558
1559     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1560
1561     return $device;
1562 }
1563
1564 sub get_initiator_name {
1565     my $initiator;
1566
1567     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1568     while (defined(my $line = <$fh>)) {
1569         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1570         $initiator = $1;
1571         last;
1572     }
1573     $fh->close();
1574
1575     return $initiator;
1576 }
1577
1578 sub print_drive_full {
1579     my ($storecfg, $vmid, $drive) = @_;
1580
1581     my $path;
1582     my $volid = $drive->{file};
1583     my $format;
1584
1585     if (drive_is_cdrom($drive)) {
1586         $path = get_iso_path($storecfg, $vmid, $volid);
1587     } else {
1588         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1589         if ($storeid) {
1590             $path = PVE::Storage::path($storecfg, $volid);
1591             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1592             $format = qemu_img_format($scfg, $volname);
1593         } else {
1594             $path = $volid;
1595             $format = "raw";
1596         }
1597    }
1598
1599     my $opts = '';
1600     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1601     foreach my $o (@qemu_drive_options) {
1602         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1603     }
1604     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1605         my ($dir, $qmpname) = @$type;
1606         if (my $v = $drive->{"mbps$dir"}) {
1607             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1608         }
1609         if (my $v = $drive->{"mbps${dir}_max"}) {
1610             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1611         }
1612         if (my $v = $drive->{"bps${dir}_max_length"}) {
1613             $opts .= ",throttling.bps$qmpname-max-length=$v";
1614         }
1615         if (my $v = $drive->{"iops${dir}"}) {
1616             $opts .= ",throttling.iops$qmpname=$v";
1617         }
1618         if (my $v = $drive->{"iops${dir}_max"}) {
1619             $opts .= ",throttling.iops$qmpname-max=$v";
1620         }
1621         if (my $v = $drive->{"iops${dir}_max_length"}) {
1622             $opts .= ",throttling.iops$qmpname-max-length=$v";
1623         }
1624     }
1625
1626     if (my $serial = $drive->{serial}) {
1627         $serial = URI::Escape::uri_unescape($serial);
1628         $opts .= ",serial=$serial";
1629     }
1630
1631     $opts .= ",format=$format" if $format && !$drive->{format};
1632
1633     my $cache_direct = 0;
1634
1635     if (my $cache = $drive->{cache}) {
1636         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1637     } elsif (!drive_is_cdrom($drive)) {
1638         $opts .= ",cache=none";
1639         $cache_direct = 1;
1640     }
1641
1642     # aio native works only with O_DIRECT
1643     if (!$drive->{aio}) {
1644         if($cache_direct) {
1645             $opts .= ",aio=native";
1646         } else {
1647             $opts .= ",aio=threads";
1648         }
1649     }
1650
1651     if (!drive_is_cdrom($drive)) {
1652         my $detectzeroes;
1653         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1654             $detectzeroes = 'off';
1655         } elsif ($drive->{discard}) {
1656             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1657         } else {
1658             # This used to be our default with discard not being specified:
1659             $detectzeroes = 'on';
1660         }
1661         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1662     }
1663
1664     my $pathinfo = $path ? "file=$path," : '';
1665
1666     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1667 }
1668
1669 sub print_netdevice_full {
1670     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1671
1672     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1673
1674     my $device = $net->{model};
1675     if ($net->{model} eq 'virtio') {
1676          $device = 'virtio-net-pci';
1677      };
1678
1679     my $pciaddr = print_pci_addr("$netid", $bridges);
1680     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1681     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1682         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1683         my $vectors = $net->{queues} * 2 + 2;
1684         $tmpstr .= ",vectors=$vectors,mq=on";
1685     }
1686     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1687
1688     if ($use_old_bios_files) {
1689         my $romfile;
1690         if ($device eq 'virtio-net-pci') {
1691             $romfile = 'pxe-virtio.rom';
1692         } elsif ($device eq 'e1000') {
1693             $romfile = 'pxe-e1000.rom';
1694         } elsif ($device eq 'ne2k') {
1695             $romfile = 'pxe-ne2k_pci.rom';
1696         } elsif ($device eq 'pcnet') {
1697             $romfile = 'pxe-pcnet.rom';
1698         } elsif ($device eq 'rtl8139') {
1699             $romfile = 'pxe-rtl8139.rom';
1700         }
1701         $tmpstr .= ",romfile=$romfile" if $romfile;
1702     }
1703
1704     return $tmpstr;
1705 }
1706
1707 sub print_netdev_full {
1708     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1709
1710     my $i = '';
1711     if ($netid =~ m/^net(\d+)$/) {
1712         $i = int($1);
1713     }
1714
1715     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1716
1717     my $ifname = "tap${vmid}i$i";
1718
1719     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1720     die "interface name '$ifname' is too long (max 15 character)\n"
1721         if length($ifname) >= 16;
1722
1723     my $vhostparam = '';
1724     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1725
1726     my $vmname = $conf->{name} || "vm$vmid";
1727
1728     my $netdev = "";
1729     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1730
1731     if ($net->{bridge}) {
1732         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1733     } else {
1734         $netdev = "type=user,id=$netid,hostname=$vmname";
1735     }
1736
1737     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1738
1739     return $netdev;
1740 }
1741
1742
1743 sub print_cpu_device {
1744     my ($conf, $id) = @_;
1745
1746     my $kvm = $conf->{kvm} // 1;
1747     my $cpu = $kvm ? "kvm64" : "qemu64";
1748     if (my $cputype = $conf->{cpu}) {
1749         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1750             or die "Cannot parse cpu description: $cputype\n";
1751         $cpu = $cpuconf->{cputype};
1752     }
1753
1754     my $cores = $conf->{cores} || 1;
1755
1756     my $current_core = ($id - 1) % $cores;
1757     my $current_socket = int(($id - 1 - $current_core)/$cores);
1758
1759     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1760 }
1761
1762 sub drive_is_cdrom {
1763     my ($drive) = @_;
1764
1765     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1766
1767 }
1768
1769 sub parse_number_sets {
1770     my ($set) = @_;
1771     my $res = [];
1772     foreach my $part (split(/;/, $set)) {
1773         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1774             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1775             push @$res, [ $1, $2 ];
1776         } else {
1777             die "invalid range: $part\n";
1778         }
1779     }
1780     return $res;
1781 }
1782
1783 sub parse_numa {
1784     my ($data) = @_;
1785
1786     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1787     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1788     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1789     return $res;
1790 }
1791
1792 sub parse_hostpci {
1793     my ($value) = @_;
1794
1795     return undef if !$value;
1796
1797     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1798
1799     my @idlist = split(/;/, $res->{host});
1800     delete $res->{host};
1801     foreach my $id (@idlist) {
1802         if ($id =~ /^$PCIRE$/) {
1803             if (defined($2)) {
1804                 push @{$res->{pciid}}, { id => $1, function => $2 };
1805             } else {
1806                 my $pcidevices = lspci($1);
1807                 $res->{pciid} = $pcidevices->{$1};
1808             }
1809         } else {
1810             # should have been caught by parse_property_string already
1811             die "failed to parse PCI id: $id\n";
1812         }
1813     }
1814     return $res;
1815 }
1816
1817 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1818 sub parse_net {
1819     my ($data) = @_;
1820
1821     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1822     if ($@) {
1823         warn $@;
1824         return undef;
1825     }
1826     if (!defined($res->{macaddr})) {
1827         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1828         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1829     }
1830     return $res;
1831 }
1832
1833 sub print_net {
1834     my $net = shift;
1835
1836     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1837 }
1838
1839 sub add_random_macs {
1840     my ($settings) = @_;
1841
1842     foreach my $opt (keys %$settings) {
1843         next if $opt !~ m/^net(\d+)$/;
1844         my $net = parse_net($settings->{$opt});
1845         next if !$net;
1846         $settings->{$opt} = print_net($net);
1847     }
1848 }
1849
1850 sub vm_is_volid_owner {
1851     my ($storecfg, $vmid, $volid) = @_;
1852
1853     if ($volid !~  m|^/|) {
1854         my ($path, $owner);
1855         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1856         if ($owner && ($owner == $vmid)) {
1857             return 1;
1858         }
1859     }
1860
1861     return undef;
1862 }
1863
1864 sub split_flagged_list {
1865     my $text = shift || '';
1866     $text =~ s/[,;]/ /g;
1867     $text =~ s/^\s+//;
1868     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1869 }
1870
1871 sub join_flagged_list {
1872     my ($how, $lst) = @_;
1873     join $how, map { $lst->{$_} . $_ } keys %$lst;
1874 }
1875
1876 sub vmconfig_delete_pending_option {
1877     my ($conf, $key, $force) = @_;
1878
1879     delete $conf->{pending}->{$key};
1880     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1881     $pending_delete_hash->{$key} = $force ? '!' : '';
1882     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1883 }
1884
1885 sub vmconfig_undelete_pending_option {
1886     my ($conf, $key) = @_;
1887
1888     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1889     delete $pending_delete_hash->{$key};
1890
1891     if (%$pending_delete_hash) {
1892         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1893     } else {
1894         delete $conf->{pending}->{delete};
1895     }
1896 }
1897
1898 sub vmconfig_register_unused_drive {
1899     my ($storecfg, $vmid, $conf, $drive) = @_;
1900
1901     if (!drive_is_cdrom($drive)) {
1902         my $volid = $drive->{file};
1903         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1904             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1905         }
1906     }
1907 }
1908
1909 sub vmconfig_cleanup_pending {
1910     my ($conf) = @_;
1911
1912     # remove pending changes when nothing changed
1913     my $changes;
1914     foreach my $opt (keys %{$conf->{pending}}) {
1915         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1916             $changes = 1;
1917             delete $conf->{pending}->{$opt};
1918         }
1919     }
1920
1921     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1922     my $pending_delete_hash = {};
1923     while (my ($opt, $force) = each %$current_delete_hash) {
1924         if (defined($conf->{$opt})) {
1925             $pending_delete_hash->{$opt} = $force;
1926         } else {
1927             $changes = 1;
1928         }
1929     }
1930
1931     if (%$pending_delete_hash) {
1932         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1933     } else {
1934         delete $conf->{pending}->{delete};
1935     }
1936
1937     return $changes;
1938 }
1939
1940 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1941 my $smbios1_fmt = {
1942     uuid => {
1943         type => 'string',
1944         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1945         format_description => 'UUID',
1946         description => "Set SMBIOS1 UUID.",
1947         optional => 1,
1948     },
1949     version => {
1950         type => 'string',
1951         pattern => '\S+',
1952         format_description => 'string',
1953         description => "Set SMBIOS1 version.",
1954         optional => 1,
1955     },
1956     serial => {
1957         type => 'string',
1958         pattern => '\S+',
1959         format_description => 'string',
1960         description => "Set SMBIOS1 serial number.",
1961         optional => 1,
1962     },
1963     manufacturer => {
1964         type => 'string',
1965         pattern => '\S+',
1966         format_description => 'string',
1967         description => "Set SMBIOS1 manufacturer.",
1968         optional => 1,
1969     },
1970     product => {
1971         type => 'string',
1972         pattern => '\S+',
1973         format_description => 'string',
1974         description => "Set SMBIOS1 product ID.",
1975         optional => 1,
1976     },
1977     sku => {
1978         type => 'string',
1979         pattern => '\S+',
1980         format_description => 'string',
1981         description => "Set SMBIOS1 SKU string.",
1982         optional => 1,
1983     },
1984     family => {
1985         type => 'string',
1986         pattern => '\S+',
1987         format_description => 'string',
1988         description => "Set SMBIOS1 family string.",
1989         optional => 1,
1990     },
1991 };
1992
1993 sub parse_smbios1 {
1994     my ($data) = @_;
1995
1996     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1997     warn $@ if $@;
1998     return $res;
1999 }
2000
2001 sub print_smbios1 {
2002     my ($smbios1) = @_;
2003     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2004 }
2005
2006 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2007
2008 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2009 sub verify_bootdisk {
2010     my ($value, $noerr) = @_;
2011
2012     return $value if is_valid_drivename($value);
2013
2014     return undef if $noerr;
2015
2016     die "invalid boot disk '$value'\n";
2017 }
2018
2019 sub parse_watchdog {
2020     my ($value) = @_;
2021
2022     return undef if !$value;
2023
2024     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2025     warn $@ if $@;
2026     return $res;
2027 }
2028
2029 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2030 sub verify_usb_device {
2031     my ($value, $noerr) = @_;
2032
2033     return $value if parse_usb_device($value);
2034
2035     return undef if $noerr;
2036
2037     die "unable to parse usb device\n";
2038 }
2039
2040 # add JSON properties for create and set function
2041 sub json_config_properties {
2042     my $prop = shift;
2043
2044     foreach my $opt (keys %$confdesc) {
2045         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2046         $prop->{$opt} = $confdesc->{$opt};
2047     }
2048
2049     return $prop;
2050 }
2051
2052 sub check_type {
2053     my ($key, $value) = @_;
2054
2055     die "unknown setting '$key'\n" if !$confdesc->{$key};
2056
2057     my $type = $confdesc->{$key}->{type};
2058
2059     if (!defined($value)) {
2060         die "got undefined value\n";
2061     }
2062
2063     if ($value =~ m/[\n\r]/) {
2064         die "property contains a line feed\n";
2065     }
2066
2067     if ($type eq 'boolean') {
2068         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2069         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2070         die "type check ('boolean') failed - got '$value'\n";
2071     } elsif ($type eq 'integer') {
2072         return int($1) if $value =~ m/^(\d+)$/;
2073         die "type check ('integer') failed - got '$value'\n";
2074     } elsif ($type eq 'number') {
2075         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2076         die "type check ('number') failed - got '$value'\n";
2077     } elsif ($type eq 'string') {
2078         if (my $fmt = $confdesc->{$key}->{format}) {
2079             PVE::JSONSchema::check_format($fmt, $value);
2080             return $value;
2081         }
2082         $value =~ s/^\"(.*)\"$/$1/;
2083         return $value;
2084     } else {
2085         die "internal error"
2086     }
2087 }
2088
2089 sub check_iommu_support{
2090     #fixme : need to check IOMMU support
2091     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2092
2093     my $iommu=1;
2094     return $iommu;
2095
2096 }
2097
2098 sub touch_config {
2099     my ($vmid) = @_;
2100
2101     my $conf = PVE::QemuConfig->config_file($vmid);
2102     utime undef, undef, $conf;
2103 }
2104
2105 sub destroy_vm {
2106     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2107
2108     my $conffile = PVE::QemuConfig->config_file($vmid);
2109
2110     my $conf = PVE::QemuConfig->load_config($vmid);
2111
2112     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2113
2114     # only remove disks owned by this VM
2115     foreach_drive($conf, sub {
2116         my ($ds, $drive) = @_;
2117
2118         return if drive_is_cdrom($drive);
2119
2120         my $volid = $drive->{file};
2121
2122         return if !$volid || $volid =~ m|^/|;
2123
2124         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2125         return if !$path || !$owner || ($owner != $vmid);
2126
2127         eval {
2128             PVE::Storage::vdisk_free($storecfg, $volid);
2129         };
2130         warn "Could not remove disk '$volid', check manually: $@" if $@;
2131
2132     });
2133
2134     if ($keep_empty_config) {
2135         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2136     } else {
2137         unlink $conffile;
2138     }
2139
2140     # also remove unused disk
2141     eval {
2142         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2143
2144         eval {
2145             PVE::Storage::foreach_volid($dl, sub {
2146                 my ($volid, $sid, $volname, $d) = @_;
2147                 PVE::Storage::vdisk_free($storecfg, $volid);
2148             });
2149         };
2150         warn $@ if $@;
2151
2152     };
2153     warn $@ if $@;
2154 }
2155
2156 sub parse_vm_config {
2157     my ($filename, $raw) = @_;
2158
2159     return undef if !defined($raw);
2160
2161     my $res = {
2162         digest => Digest::SHA::sha1_hex($raw),
2163         snapshots => {},
2164         pending => {},
2165     };
2166
2167     $filename =~ m|/qemu-server/(\d+)\.conf$|
2168         || die "got strange filename '$filename'";
2169
2170     my $vmid = $1;
2171
2172     my $conf = $res;
2173     my $descr;
2174     my $section = '';
2175
2176     my @lines = split(/\n/, $raw);
2177     foreach my $line (@lines) {
2178         next if $line =~ m/^\s*$/;
2179
2180         if ($line =~ m/^\[PENDING\]\s*$/i) {
2181             $section = 'pending';
2182             if (defined($descr)) {
2183                 $descr =~ s/\s+$//;
2184                 $conf->{description} = $descr;
2185             }
2186             $descr = undef;
2187             $conf = $res->{$section} = {};
2188             next;
2189
2190         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2191             $section = $1;
2192             if (defined($descr)) {
2193                 $descr =~ s/\s+$//;
2194                 $conf->{description} = $descr;
2195             }
2196             $descr = undef;
2197             $conf = $res->{snapshots}->{$section} = {};
2198             next;
2199         }
2200
2201         if ($line =~ m/^\#(.*)\s*$/) {
2202             $descr = '' if !defined($descr);
2203             $descr .= PVE::Tools::decode_text($1) . "\n";
2204             next;
2205         }
2206
2207         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2208             $descr = '' if !defined($descr);
2209             $descr .= PVE::Tools::decode_text($2);
2210         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2211             $conf->{snapstate} = $1;
2212         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2213             my $key = $1;
2214             my $value = $2;
2215             $conf->{$key} = $value;
2216         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2217             my $value = $1;
2218             if ($section eq 'pending') {
2219                 $conf->{delete} = $value; # we parse this later
2220             } else {
2221                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2222             }
2223         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2224             my $key = $1;
2225             my $value = $2;
2226             eval { $value = check_type($key, $value); };
2227             if ($@) {
2228                 warn "vm $vmid - unable to parse value of '$key' - $@";
2229             } else {
2230                 $key = 'ide2' if $key eq 'cdrom';
2231                 my $fmt = $confdesc->{$key}->{format};
2232                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2233                     my $v = parse_drive($key, $value);
2234                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2235                         $v->{file} = $volid;
2236                         $value = print_drive($vmid, $v);
2237                     } else {
2238                         warn "vm $vmid - unable to parse value of '$key'\n";
2239                         next;
2240                     }
2241                 }
2242
2243                 $conf->{$key} = $value;
2244             }
2245         }
2246     }
2247
2248     if (defined($descr)) {
2249         $descr =~ s/\s+$//;
2250         $conf->{description} = $descr;
2251     }
2252     delete $res->{snapstate}; # just to be sure
2253
2254     return $res;
2255 }
2256
2257 sub write_vm_config {
2258     my ($filename, $conf) = @_;
2259
2260     delete $conf->{snapstate}; # just to be sure
2261
2262     if ($conf->{cdrom}) {
2263         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2264         $conf->{ide2} = $conf->{cdrom};
2265         delete $conf->{cdrom};
2266     }
2267
2268     # we do not use 'smp' any longer
2269     if ($conf->{sockets}) {
2270         delete $conf->{smp};
2271     } elsif ($conf->{smp}) {
2272         $conf->{sockets} = $conf->{smp};
2273         delete $conf->{cores};
2274         delete $conf->{smp};
2275     }
2276
2277     my $used_volids = {};
2278
2279     my $cleanup_config = sub {
2280         my ($cref, $pending, $snapname) = @_;
2281
2282         foreach my $key (keys %$cref) {
2283             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2284                 $key eq 'snapstate' || $key eq 'pending';
2285             my $value = $cref->{$key};
2286             if ($key eq 'delete') {
2287                 die "propertry 'delete' is only allowed in [PENDING]\n"
2288                     if !$pending;
2289                 # fixme: check syntax?
2290                 next;
2291             }
2292             eval { $value = check_type($key, $value); };
2293             die "unable to parse value of '$key' - $@" if $@;
2294
2295             $cref->{$key} = $value;
2296
2297             if (!$snapname && is_valid_drivename($key)) {
2298                 my $drive = parse_drive($key, $value);
2299                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2300             }
2301         }
2302     };
2303
2304     &$cleanup_config($conf);
2305
2306     &$cleanup_config($conf->{pending}, 1);
2307
2308     foreach my $snapname (keys %{$conf->{snapshots}}) {
2309         die "internal error" if $snapname eq 'pending';
2310         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2311     }
2312
2313     # remove 'unusedX' settings if we re-add a volume
2314     foreach my $key (keys %$conf) {
2315         my $value = $conf->{$key};
2316         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2317             delete $conf->{$key};
2318         }
2319     }
2320
2321     my $generate_raw_config = sub {
2322         my ($conf, $pending) = @_;
2323
2324         my $raw = '';
2325
2326         # add description as comment to top of file
2327         if (defined(my $descr = $conf->{description})) {
2328             if ($descr) {
2329                 foreach my $cl (split(/\n/, $descr)) {
2330                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2331                 }
2332             } else {
2333                 $raw .= "#\n" if $pending;
2334             }
2335         }
2336
2337         foreach my $key (sort keys %$conf) {
2338             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2339             $raw .= "$key: $conf->{$key}\n";
2340         }
2341         return $raw;
2342     };
2343
2344     my $raw = &$generate_raw_config($conf);
2345
2346     if (scalar(keys %{$conf->{pending}})){
2347         $raw .= "\n[PENDING]\n";
2348         $raw .= &$generate_raw_config($conf->{pending}, 1);
2349     }
2350
2351     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2352         $raw .= "\n[$snapname]\n";
2353         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2354     }
2355
2356     return $raw;
2357 }
2358
2359 sub load_defaults {
2360
2361     my $res = {};
2362
2363     # we use static defaults from our JSON schema configuration
2364     foreach my $key (keys %$confdesc) {
2365         if (defined(my $default = $confdesc->{$key}->{default})) {
2366             $res->{$key} = $default;
2367         }
2368     }
2369
2370     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2371     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2372
2373     return $res;
2374 }
2375
2376 sub config_list {
2377     my $vmlist = PVE::Cluster::get_vmlist();
2378     my $res = {};
2379     return $res if !$vmlist || !$vmlist->{ids};
2380     my $ids = $vmlist->{ids};
2381
2382     foreach my $vmid (keys %$ids) {
2383         my $d = $ids->{$vmid};
2384         next if !$d->{node} || $d->{node} ne $nodename;
2385         next if !$d->{type} || $d->{type} ne 'qemu';
2386         $res->{$vmid}->{exists} = 1;
2387     }
2388     return $res;
2389 }
2390
2391 # test if VM uses local resources (to prevent migration)
2392 sub check_local_resources {
2393     my ($conf, $noerr) = @_;
2394
2395     my $loc_res = 0;
2396
2397     $loc_res = 1 if $conf->{hostusb}; # old syntax
2398     $loc_res = 1 if $conf->{hostpci}; # old syntax
2399
2400     foreach my $k (keys %$conf) {
2401         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2402         # sockets are safe: they will recreated be on the target side post-migrate
2403         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2404         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2405     }
2406
2407     die "VM uses local resources\n" if $loc_res && !$noerr;
2408
2409     return $loc_res;
2410 }
2411
2412 # check if used storages are available on all nodes (use by migrate)
2413 sub check_storage_availability {
2414     my ($storecfg, $conf, $node) = @_;
2415
2416     foreach_drive($conf, sub {
2417         my ($ds, $drive) = @_;
2418
2419         my $volid = $drive->{file};
2420         return if !$volid;
2421
2422         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2423         return if !$sid;
2424
2425         # check if storage is available on both nodes
2426         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2427         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2428    });
2429 }
2430
2431 # list nodes where all VM images are available (used by has_feature API)
2432 sub shared_nodes {
2433     my ($conf, $storecfg) = @_;
2434
2435     my $nodelist = PVE::Cluster::get_nodelist();
2436     my $nodehash = { map { $_ => 1 } @$nodelist };
2437     my $nodename = PVE::INotify::nodename();
2438
2439     foreach_drive($conf, sub {
2440         my ($ds, $drive) = @_;
2441
2442         my $volid = $drive->{file};
2443         return if !$volid;
2444
2445         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2446         if ($storeid) {
2447             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2448             if ($scfg->{disable}) {
2449                 $nodehash = {};
2450             } elsif (my $avail = $scfg->{nodes}) {
2451                 foreach my $node (keys %$nodehash) {
2452                     delete $nodehash->{$node} if !$avail->{$node};
2453                 }
2454             } elsif (!$scfg->{shared}) {
2455                 foreach my $node (keys %$nodehash) {
2456                     delete $nodehash->{$node} if $node ne $nodename
2457                 }
2458             }
2459         }
2460     });
2461
2462     return $nodehash
2463 }
2464
2465 sub check_cmdline {
2466     my ($pidfile, $pid) = @_;
2467
2468     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2469     if (defined($fh)) {
2470         my $line = <$fh>;
2471         $fh->close;
2472         return undef if !$line;
2473         my @param = split(/\0/, $line);
2474
2475         my $cmd = $param[0];
2476         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2477
2478         for (my $i = 0; $i < scalar (@param); $i++) {
2479             my $p = $param[$i];
2480             next if !$p;
2481             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2482                 my $p = $param[$i+1];
2483                 return 1 if $p && ($p eq $pidfile);
2484                 return undef;
2485             }
2486         }
2487     }
2488     return undef;
2489 }
2490
2491 sub check_running {
2492     my ($vmid, $nocheck, $node) = @_;
2493
2494     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2495
2496     die "unable to find configuration file for VM $vmid - no such machine\n"
2497         if !$nocheck && ! -f $filename;
2498
2499     my $pidfile = pidfile_name($vmid);
2500
2501     if (my $fd = IO::File->new("<$pidfile")) {
2502         my $st = stat($fd);
2503         my $line = <$fd>;
2504         close($fd);
2505
2506         my $mtime = $st->mtime;
2507         if ($mtime > time()) {
2508             warn "file '$filename' modified in future\n";
2509         }
2510
2511         if ($line =~ m/^(\d+)$/) {
2512             my $pid = $1;
2513             if (check_cmdline($pidfile, $pid)) {
2514                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2515                     return $pid;
2516                 }
2517             }
2518         }
2519     }
2520
2521     return undef;
2522 }
2523
2524 sub vzlist {
2525
2526     my $vzlist = config_list();
2527
2528     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2529
2530     while (defined(my $de = $fd->read)) {
2531         next if $de !~ m/^(\d+)\.pid$/;
2532         my $vmid = $1;
2533         next if !defined($vzlist->{$vmid});
2534         if (my $pid = check_running($vmid)) {
2535             $vzlist->{$vmid}->{pid} = $pid;
2536         }
2537     }
2538
2539     return $vzlist;
2540 }
2541
2542 sub disksize {
2543     my ($storecfg, $conf) = @_;
2544
2545     my $bootdisk = $conf->{bootdisk};
2546     return undef if !$bootdisk;
2547     return undef if !is_valid_drivename($bootdisk);
2548
2549     return undef if !$conf->{$bootdisk};
2550
2551     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2552     return undef if !defined($drive);
2553
2554     return undef if drive_is_cdrom($drive);
2555
2556     my $volid = $drive->{file};
2557     return undef if !$volid;
2558
2559     return $drive->{size};
2560 }
2561
2562 my $last_proc_pid_stat;
2563
2564 # get VM status information
2565 # This must be fast and should not block ($full == false)
2566 # We only query KVM using QMP if $full == true (this can be slow)
2567 sub vmstatus {
2568     my ($opt_vmid, $full) = @_;
2569
2570     my $res = {};
2571
2572     my $storecfg = PVE::Storage::config();
2573
2574     my $list = vzlist();
2575     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2576
2577     my $cpucount = $cpuinfo->{cpus} || 1;
2578
2579     foreach my $vmid (keys %$list) {
2580         next if $opt_vmid && ($vmid ne $opt_vmid);
2581
2582         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2583         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2584
2585         my $d = {};
2586         $d->{pid} = $list->{$vmid}->{pid};
2587
2588         # fixme: better status?
2589         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2590
2591         my $size = disksize($storecfg, $conf);
2592         if (defined($size)) {
2593             $d->{disk} = 0; # no info available
2594             $d->{maxdisk} = $size;
2595         } else {
2596             $d->{disk} = 0;
2597             $d->{maxdisk} = 0;
2598         }
2599
2600         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2601         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2602         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2603
2604         $d->{name} = $conf->{name} || "VM $vmid";
2605         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2606
2607         if ($conf->{balloon}) {
2608             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2609             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2610         }
2611
2612         $d->{uptime} = 0;
2613         $d->{cpu} = 0;
2614         $d->{mem} = 0;
2615
2616         $d->{netout} = 0;
2617         $d->{netin} = 0;
2618
2619         $d->{diskread} = 0;
2620         $d->{diskwrite} = 0;
2621
2622         $d->{template} = PVE::QemuConfig->is_template($conf);
2623
2624         $res->{$vmid} = $d;
2625     }
2626
2627     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2628     foreach my $dev (keys %$netdev) {
2629         next if $dev !~ m/^tap([1-9]\d*)i/;
2630         my $vmid = $1;
2631         my $d = $res->{$vmid};
2632         next if !$d;
2633
2634         $d->{netout} += $netdev->{$dev}->{receive};
2635         $d->{netin} += $netdev->{$dev}->{transmit};
2636
2637         if ($full) {
2638             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2639             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2640         }
2641
2642     }
2643
2644     my $ctime = gettimeofday;
2645
2646     foreach my $vmid (keys %$list) {
2647
2648         my $d = $res->{$vmid};
2649         my $pid = $d->{pid};
2650         next if !$pid;
2651
2652         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2653         next if !$pstat; # not running
2654
2655         my $used = $pstat->{utime} + $pstat->{stime};
2656
2657         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2658
2659         if ($pstat->{vsize}) {
2660             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2661         }
2662
2663         my $old = $last_proc_pid_stat->{$pid};
2664         if (!$old) {
2665             $last_proc_pid_stat->{$pid} = {
2666                 time => $ctime,
2667                 used => $used,
2668                 cpu => 0,
2669             };
2670             next;
2671         }
2672
2673         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2674
2675         if ($dtime > 1000) {
2676             my $dutime = $used -  $old->{used};
2677
2678             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2679             $last_proc_pid_stat->{$pid} = {
2680                 time => $ctime,
2681                 used => $used,
2682                 cpu => $d->{cpu},
2683             };
2684         } else {
2685             $d->{cpu} = $old->{cpu};
2686         }
2687     }
2688
2689     return $res if !$full;
2690
2691     my $qmpclient = PVE::QMPClient->new();
2692
2693     my $ballooncb = sub {
2694         my ($vmid, $resp) = @_;
2695
2696         my $info = $resp->{'return'};
2697         return if !$info->{max_mem};
2698
2699         my $d = $res->{$vmid};
2700
2701         # use memory assigned to VM
2702         $d->{maxmem} = $info->{max_mem};
2703         $d->{balloon} = $info->{actual};
2704
2705         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2706             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2707             $d->{freemem} = $info->{free_mem};
2708         }
2709
2710         $d->{ballooninfo} = $info;
2711     };
2712
2713     my $blockstatscb = sub {
2714         my ($vmid, $resp) = @_;
2715         my $data = $resp->{'return'} || [];
2716         my $totalrdbytes = 0;
2717         my $totalwrbytes = 0;
2718
2719         for my $blockstat (@$data) {
2720             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2721             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2722
2723             $blockstat->{device} =~ s/drive-//;
2724             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2725         }
2726         $res->{$vmid}->{diskread} = $totalrdbytes;
2727         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2728     };
2729
2730     my $statuscb = sub {
2731         my ($vmid, $resp) = @_;
2732
2733         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2734         # this fails if ballon driver is not loaded, so this must be
2735         # the last commnand (following command are aborted if this fails).
2736         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2737
2738         my $status = 'unknown';
2739         if (!defined($status = $resp->{'return'}->{status})) {
2740             warn "unable to get VM status\n";
2741             return;
2742         }
2743
2744         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2745     };
2746
2747     foreach my $vmid (keys %$list) {
2748         next if $opt_vmid && ($vmid ne $opt_vmid);
2749         next if !$res->{$vmid}->{pid}; # not running
2750         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2751     }
2752
2753     $qmpclient->queue_execute(undef, 2);
2754
2755     foreach my $vmid (keys %$list) {
2756         next if $opt_vmid && ($vmid ne $opt_vmid);
2757         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2758     }
2759
2760     return $res;
2761 }
2762
2763 sub foreach_drive {
2764     my ($conf, $func, @param) = @_;
2765
2766     foreach my $ds (valid_drive_names()) {
2767         next if !defined($conf->{$ds});
2768
2769         my $drive = parse_drive($ds, $conf->{$ds});
2770         next if !$drive;
2771
2772         &$func($ds, $drive, @param);
2773     }
2774 }
2775
2776 sub foreach_volid {
2777     my ($conf, $func, @param) = @_;
2778
2779     my $volhash = {};
2780
2781     my $test_volid = sub {
2782         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2783
2784         return if !$volid;
2785
2786         $volhash->{$volid}->{cdrom} //= 1;
2787         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2788
2789         $volhash->{$volid}->{replicate} //= 0;
2790         $volhash->{$volid}->{replicate} = 1 if $replicate;
2791
2792         $volhash->{$volid}->{referenced_in_config} //= 0;
2793         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2794
2795         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2796             if defined($snapname);
2797     };
2798
2799     foreach_drive($conf, sub {
2800         my ($ds, $drive) = @_;
2801         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2802     });
2803
2804     foreach my $snapname (keys %{$conf->{snapshots}}) {
2805         my $snap = $conf->{snapshots}->{$snapname};
2806         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2807         foreach_drive($snap, sub {
2808             my ($ds, $drive) = @_;
2809             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2810         });
2811     }
2812
2813     foreach my $volid (keys %$volhash) {
2814         &$func($volid, $volhash->{$volid}, @param);
2815     }
2816 }
2817
2818 sub vga_conf_has_spice {
2819     my ($vga) = @_;
2820
2821     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2822
2823     return $1 || 1;
2824 }
2825
2826 sub config_to_command {
2827     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2828
2829     my $cmd = [];
2830     my $globalFlags = [];
2831     my $machineFlags = [];
2832     my $rtcFlags = [];
2833     my $cpuFlags = [];
2834     my $devices = [];
2835     my $pciaddr = '';
2836     my $bridges = {};
2837     my $kvmver = kvm_user_version();
2838     my $vernum = 0; # unknown
2839     my $ostype = $conf->{ostype};
2840     my $winversion = windows_version($ostype);
2841     my $kvm = $conf->{kvm} // 1;
2842
2843     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2844
2845     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2846         $vernum = $1*1000000+$2*1000;
2847     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2848         $vernum = $1*1000000+$2*1000+$3;
2849     }
2850
2851     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2852
2853     my $have_ovz = -f '/proc/vz/vestat';
2854
2855     my $q35 = machine_type_is_q35($conf);
2856     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2857     my $machine_type = $forcemachine || $conf->{machine};
2858     my $use_old_bios_files = undef;
2859     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2860
2861     my $cpuunits = defined($conf->{cpuunits}) ?
2862             $conf->{cpuunits} : $defaults->{cpuunits};
2863
2864     push @$cmd, '/usr/bin/kvm';
2865
2866     push @$cmd, '-id', $vmid;
2867
2868     my $use_virtio = 0;
2869
2870     my $qmpsocket = qmp_socket($vmid);
2871     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2872     push @$cmd, '-mon', "chardev=qmp,mode=control";
2873
2874
2875     push @$cmd, '-pidfile' , pidfile_name($vmid);
2876
2877     push @$cmd, '-daemonize';
2878
2879     if ($conf->{smbios1}) {
2880         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2881     }
2882
2883     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2884         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2885
2886         my $path;
2887         if (my $efidisk = $conf->{efidisk0}) {
2888             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2889             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2890             if ($storeid) {
2891                 $path = PVE::Storage::path($storecfg, $d->{file});
2892             } else {
2893                 $path = $d->{file};
2894             }
2895         } else {
2896             warn "no efidisk configured! Using temporary efivars disk.\n";
2897             $path = "/tmp/$vmid-ovmf.fd";
2898             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2899         }
2900
2901         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2902         push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,file=$path";
2903     }
2904
2905
2906     # add usb controllers
2907     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2908     push @$devices, @usbcontrollers if @usbcontrollers;
2909     my $vga = $conf->{vga};
2910
2911     my $qxlnum = vga_conf_has_spice($vga);
2912     $vga = 'qxl' if $qxlnum;
2913
2914     if (!$vga) {
2915         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2916             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2917         } else {
2918             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2919         }
2920     }
2921
2922     # enable absolute mouse coordinates (needed by vnc)
2923     my $tablet;
2924     if (defined($conf->{tablet})) {
2925         $tablet = $conf->{tablet};
2926     } else {
2927         $tablet = $defaults->{tablet};
2928         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2929         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2930     }
2931
2932     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2933
2934     my $kvm_off = 0;
2935     my $gpu_passthrough;
2936
2937     # host pci devices
2938     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2939         my $d = parse_hostpci($conf->{"hostpci$i"});
2940         next if !$d;
2941
2942         my $pcie = $d->{pcie};
2943         if($pcie){
2944             die "q35 machine model is not enabled" if !$q35;
2945             $pciaddr = print_pcie_addr("hostpci$i");
2946         }else{
2947             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2948         }
2949
2950         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2951         my $romfile = $d->{romfile};
2952
2953         my $xvga = '';
2954         if ($d->{'x-vga'}) {
2955             $xvga = ',x-vga=on';
2956             $kvm_off = 1;
2957             $vga = 'none';
2958             $gpu_passthrough = 1;
2959
2960             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2961                 $xvga = "";
2962             }
2963         }
2964         my $pcidevices = $d->{pciid};
2965         my $multifunction = 1 if @$pcidevices > 1;
2966
2967         my $j=0;
2968         foreach my $pcidevice (@$pcidevices) {
2969
2970             my $id = "hostpci$i";
2971             $id .= ".$j" if $multifunction;
2972             my $addr = $pciaddr;
2973             $addr .= ".$j" if $multifunction;
2974             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2975
2976             if($j == 0){
2977                 $devicestr .= "$rombar$xvga";
2978                 $devicestr .= ",multifunction=on" if $multifunction;
2979                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2980             }
2981
2982             push @$devices, '-device', $devicestr;
2983             $j++;
2984         }
2985     }
2986
2987     # usb devices
2988     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2989     push @$devices, @usbdevices if @usbdevices;
2990     # serial devices
2991     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2992         if (my $path = $conf->{"serial$i"}) {
2993             if ($path eq 'socket') {
2994                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2995                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2996                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2997             } else {
2998                 die "no such serial device\n" if ! -c $path;
2999                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3000                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3001             }
3002         }
3003     }
3004
3005     # parallel devices
3006     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3007         if (my $path = $conf->{"parallel$i"}) {
3008             die "no such parallel device\n" if ! -c $path;
3009             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3010             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3011             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3012         }
3013     }
3014
3015     my $vmname = $conf->{name} || "vm$vmid";
3016
3017     push @$cmd, '-name', $vmname;
3018
3019     my $sockets = 1;
3020     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3021     $sockets = $conf->{sockets} if  $conf->{sockets};
3022
3023     my $cores = $conf->{cores} || 1;
3024
3025     my $maxcpus = $sockets * $cores;
3026
3027     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3028
3029     my $allowed_vcpus = $cpuinfo->{cpus};
3030
3031     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3032         if ($allowed_vcpus < $maxcpus);
3033
3034     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3035
3036         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3037         for (my $i = 2; $i <= $vcpus; $i++)  {
3038             my $cpustr = print_cpu_device($conf,$i);
3039             push @$cmd, '-device', $cpustr;
3040         }
3041
3042     } else {
3043
3044         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3045     }
3046     push @$cmd, '-nodefaults';
3047
3048     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3049
3050     my $bootindex_hash = {};
3051     my $i = 1;
3052     foreach my $o (split(//, $bootorder)) {
3053         $bootindex_hash->{$o} = $i*100;
3054         $i++;
3055     }
3056
3057     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3058
3059     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3060
3061     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3062
3063     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3064
3065     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3066         my $socket = vnc_socket($vmid);
3067         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3068     } else {
3069         push @$cmd, '-nographic';
3070     }
3071
3072     # time drift fix
3073     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3074
3075     my $useLocaltime = $conf->{localtime};
3076
3077     if ($winversion >= 5) { # windows
3078         $useLocaltime = 1 if !defined($conf->{localtime});
3079
3080         # use time drift fix when acpi is enabled
3081         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3082             $tdf = 1 if !defined($conf->{tdf});
3083         }
3084     }
3085
3086     if ($winversion >= 6) {
3087         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3088         push @$cmd, '-no-hpet';
3089     }
3090
3091     push @$rtcFlags, 'driftfix=slew' if $tdf;
3092
3093     if (!$kvm) {
3094         push @$machineFlags, 'accel=tcg';
3095     }
3096
3097     if ($machine_type) {
3098         push @$machineFlags, "type=${machine_type}";
3099     }
3100
3101     if ($conf->{startdate}) {
3102         push @$rtcFlags, "base=$conf->{startdate}";
3103     } elsif ($useLocaltime) {
3104         push @$rtcFlags, 'base=localtime';
3105     }
3106
3107     my $cpu = $kvm ? "kvm64" : "qemu64";
3108     if (my $cputype = $conf->{cpu}) {
3109         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3110             or die "Cannot parse cpu description: $cputype\n";
3111         $cpu = $cpuconf->{cputype};
3112         $kvm_off = 1 if $cpuconf->{hidden};
3113     }
3114
3115     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3116
3117     push @$cpuFlags , '-x2apic'
3118         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3119
3120     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3121
3122     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3123
3124     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3125
3126         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3127         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3128     }
3129
3130     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3131
3132     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3133
3134     push @$cpuFlags, 'kvm=off' if $kvm_off;
3135
3136     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3137         die "internal error"; # should not happen
3138
3139     push @$cpuFlags, "vendor=${cpu_vendor}"
3140         if $cpu_vendor ne 'default';
3141
3142     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3143
3144     push @$cmd, '-cpu', $cpu;
3145
3146     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3147
3148     push @$cmd, '-S' if $conf->{freeze};
3149
3150     # set keyboard layout
3151     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3152     push @$cmd, '-k', $kb if $kb;
3153
3154     # enable sound
3155     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3156     #push @$cmd, '-soundhw', 'es1370';
3157     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3158
3159     if($conf->{agent}) {
3160         my $qgasocket = qmp_socket($vmid, 1);
3161         my $pciaddr = print_pci_addr("qga0", $bridges);
3162         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3163         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3164         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3165     }
3166
3167     my $spice_port;
3168
3169     if ($qxlnum) {
3170         if ($qxlnum > 1) {
3171             if ($winversion){
3172                 for(my $i = 1; $i < $qxlnum; $i++){
3173                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3174                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3175                 }
3176             } else {
3177                 # assume other OS works like Linux
3178                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3179                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3180             }
3181         }
3182
3183         my $pciaddr = print_pci_addr("spice", $bridges);
3184
3185         my $nodename = PVE::INotify::nodename();
3186         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3187         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3188         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3189         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3190         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3191
3192         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3193
3194         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3195         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3196         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3197     }
3198
3199     # enable balloon by default, unless explicitly disabled
3200     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3201         $pciaddr = print_pci_addr("balloon0", $bridges);
3202         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3203     }
3204
3205     if ($conf->{watchdog}) {
3206         my $wdopts = parse_watchdog($conf->{watchdog});
3207         $pciaddr = print_pci_addr("watchdog", $bridges);
3208         my $watchdog = $wdopts->{model} || 'i6300esb';
3209         push @$devices, '-device', "$watchdog$pciaddr";
3210         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3211     }
3212
3213     my $vollist = [];
3214     my $scsicontroller = {};
3215     my $ahcicontroller = {};
3216     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3217
3218     # Add iscsi initiator name if available
3219     if (my $initiator = get_initiator_name()) {
3220         push @$devices, '-iscsi', "initiator-name=$initiator";
3221     }
3222
3223     foreach_drive($conf, sub {
3224         my ($ds, $drive) = @_;
3225
3226         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3227             push @$vollist, $drive->{file};
3228         }
3229
3230         # ignore efidisk here, already added in bios/fw handling code above
3231         return if $drive->{interface} eq 'efidisk';
3232
3233         $use_virtio = 1 if $ds =~ m/^virtio/;
3234
3235         if (drive_is_cdrom ($drive)) {
3236             if ($bootindex_hash->{d}) {
3237                 $drive->{bootindex} = $bootindex_hash->{d};
3238                 $bootindex_hash->{d} += 1;
3239             }
3240         } else {
3241             if ($bootindex_hash->{c}) {
3242                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3243                 $bootindex_hash->{c} += 1;
3244             }
3245         }
3246
3247         if($drive->{interface} eq 'virtio'){
3248            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3249         }
3250
3251         if ($drive->{interface} eq 'scsi') {
3252
3253             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3254
3255             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3256             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3257
3258             my $iothread = '';
3259             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3260                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3261                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3262             } elsif ($drive->{iothread}) {
3263                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3264             }
3265
3266             my $queues = '';
3267             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3268                 $queues = ",num_queues=$drive->{queues}";
3269             }
3270
3271             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3272             $scsicontroller->{$controller}=1;
3273         }
3274
3275         if ($drive->{interface} eq 'sata') {
3276            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3277            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3278            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3279            $ahcicontroller->{$controller}=1;
3280         }
3281
3282         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3283         push @$devices, '-drive',$drive_cmd;
3284         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3285     });
3286
3287     for (my $i = 0; $i < $MAX_NETS; $i++) {
3288          next if !$conf->{"net$i"};
3289          my $d = parse_net($conf->{"net$i"});
3290          next if !$d;
3291
3292          $use_virtio = 1 if $d->{model} eq 'virtio';
3293
3294          if ($bootindex_hash->{n}) {
3295             $d->{bootindex} = $bootindex_hash->{n};
3296             $bootindex_hash->{n} += 1;
3297          }
3298
3299          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3300          push @$devices, '-netdev', $netdevfull;
3301
3302          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3303          push @$devices, '-device', $netdevicefull;
3304     }
3305
3306     if (!$q35) {
3307         # add pci bridges
3308         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3309            $bridges->{1} = 1;
3310            $bridges->{2} = 1;
3311         }
3312
3313         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3314
3315         while (my ($k, $v) = each %$bridges) {
3316             $pciaddr = print_pci_addr("pci.$k");
3317             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3318         }
3319     }
3320
3321     # add custom args
3322     if ($conf->{args}) {
3323         my $aa = PVE::Tools::split_args($conf->{args});
3324         push @$cmd, @$aa;
3325     }
3326
3327     push @$cmd, @$devices;
3328     push @$cmd, '-rtc', join(',', @$rtcFlags)
3329         if scalar(@$rtcFlags);
3330     push @$cmd, '-machine', join(',', @$machineFlags)
3331         if scalar(@$machineFlags);
3332     push @$cmd, '-global', join(',', @$globalFlags)
3333         if scalar(@$globalFlags);
3334
3335     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3336 }
3337
3338 sub vnc_socket {
3339     my ($vmid) = @_;
3340     return "${var_run_tmpdir}/$vmid.vnc";
3341 }
3342
3343 sub spice_port {
3344     my ($vmid) = @_;
3345
3346     my $res = vm_mon_cmd($vmid, 'query-spice');
3347
3348     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3349 }
3350
3351 sub qmp_socket {
3352     my ($vmid, $qga) = @_;
3353     my $sockettype = $qga ? 'qga' : 'qmp';
3354     return "${var_run_tmpdir}/$vmid.$sockettype";
3355 }
3356
3357 sub pidfile_name {
3358     my ($vmid) = @_;
3359     return "${var_run_tmpdir}/$vmid.pid";
3360 }
3361
3362 sub vm_devices_list {
3363     my ($vmid) = @_;
3364
3365     my $res = vm_mon_cmd($vmid, 'query-pci');
3366     my $devices = {};
3367     foreach my $pcibus (@$res) {
3368         foreach my $device (@{$pcibus->{devices}}) {
3369             next if !$device->{'qdev_id'};
3370             if ($device->{'pci_bridge'}) {
3371                 $devices->{$device->{'qdev_id'}} = 1;
3372                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3373                     next if !$bridge_device->{'qdev_id'};
3374                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3375                     $devices->{$device->{'qdev_id'}}++;
3376                 }
3377             } else {
3378                 $devices->{$device->{'qdev_id'}} = 1;
3379             }
3380         }
3381     }
3382
3383     my $resblock = vm_mon_cmd($vmid, 'query-block');
3384     foreach my $block (@$resblock) {
3385         if($block->{device} =~ m/^drive-(\S+)/){
3386                 $devices->{$1} = 1;
3387         }
3388     }
3389
3390     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3391     foreach my $mice (@$resmice) {
3392         if ($mice->{name} eq 'QEMU HID Tablet') {
3393             $devices->{tablet} = 1;
3394             last;
3395         }
3396     }
3397
3398     # for usb devices there is no query-usb
3399     # but we can iterate over the entries in
3400     # qom-list path=/machine/peripheral
3401     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3402     foreach my $per (@$resperipheral) {
3403         if ($per->{name} =~ m/^usb\d+$/) {
3404             $devices->{$per->{name}} = 1;
3405         }
3406     }
3407
3408     return $devices;
3409 }
3410
3411 sub vm_deviceplug {
3412     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3413
3414     my $q35 = machine_type_is_q35($conf);
3415
3416     my $devices_list = vm_devices_list($vmid);
3417     return 1 if defined($devices_list->{$deviceid});
3418
3419     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3420
3421     if ($deviceid eq 'tablet') {
3422
3423         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3424
3425     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3426
3427         die "usb hotplug currently not reliable\n";
3428         # since we can't reliably hot unplug all added usb devices
3429         # and usb passthrough disables live migration
3430         # we disable usb hotplugging for now
3431         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3432
3433     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3434
3435         qemu_iothread_add($vmid, $deviceid, $device);
3436
3437         qemu_driveadd($storecfg, $vmid, $device);
3438         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3439
3440         qemu_deviceadd($vmid, $devicefull);
3441         eval { qemu_deviceaddverify($vmid, $deviceid); };
3442         if (my $err = $@) {
3443             eval { qemu_drivedel($vmid, $deviceid); };
3444             warn $@ if $@;
3445             die $err;
3446         }
3447
3448     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3449
3450
3451         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3452         my $pciaddr = print_pci_addr($deviceid);
3453         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3454
3455         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3456
3457         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3458             qemu_iothread_add($vmid, $deviceid, $device);
3459             $devicefull .= ",iothread=iothread-$deviceid";
3460         }
3461
3462         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3463             $devicefull .= ",num_queues=$device->{queues}";
3464         }
3465
3466         qemu_deviceadd($vmid, $devicefull);
3467         qemu_deviceaddverify($vmid, $deviceid);
3468
3469     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3470
3471         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3472         qemu_driveadd($storecfg, $vmid, $device);
3473
3474         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3475         eval { qemu_deviceadd($vmid, $devicefull); };
3476         if (my $err = $@) {
3477             eval { qemu_drivedel($vmid, $deviceid); };
3478             warn $@ if $@;
3479             die $err;
3480         }
3481
3482     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3483
3484         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3485
3486         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3487         my $use_old_bios_files = undef;
3488         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3489
3490         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3491         qemu_deviceadd($vmid, $netdevicefull);
3492         eval { qemu_deviceaddverify($vmid, $deviceid); };
3493         if (my $err = $@) {
3494             eval { qemu_netdevdel($vmid, $deviceid); };
3495             warn $@ if $@;
3496             die $err;
3497         }
3498
3499     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3500
3501         my $bridgeid = $2;
3502         my $pciaddr = print_pci_addr($deviceid);
3503         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3504
3505         qemu_deviceadd($vmid, $devicefull);
3506         qemu_deviceaddverify($vmid, $deviceid);
3507
3508     } else {
3509         die "can't hotplug device '$deviceid'\n";
3510     }
3511
3512     return 1;
3513 }
3514
3515 # fixme: this should raise exceptions on error!
3516 sub vm_deviceunplug {
3517     my ($vmid, $conf, $deviceid) = @_;
3518
3519     my $devices_list = vm_devices_list($vmid);
3520     return 1 if !defined($devices_list->{$deviceid});
3521
3522     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3523
3524     if ($deviceid eq 'tablet') {
3525
3526         qemu_devicedel($vmid, $deviceid);
3527
3528     } elsif ($deviceid =~ m/^usb\d+$/) {
3529
3530         die "usb hotplug currently not reliable\n";
3531         # when unplugging usb devices this way,
3532         # there may be remaining usb controllers/hubs
3533         # so we disable it for now
3534         qemu_devicedel($vmid, $deviceid);
3535         qemu_devicedelverify($vmid, $deviceid);
3536
3537     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3538
3539         qemu_devicedel($vmid, $deviceid);
3540         qemu_devicedelverify($vmid, $deviceid);
3541         qemu_drivedel($vmid, $deviceid);
3542         qemu_iothread_del($conf, $vmid, $deviceid);
3543
3544     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3545
3546         qemu_devicedel($vmid, $deviceid);
3547         qemu_devicedelverify($vmid, $deviceid);
3548         qemu_iothread_del($conf, $vmid, $deviceid);
3549
3550     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3551
3552         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3553         my $device = parse_drive($deviceid, $conf->{$deviceid});
3554         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3555
3556         qemu_devicedel($vmid, $deviceid);
3557         qemu_drivedel($vmid, $deviceid);
3558         qemu_deletescsihw($conf, $vmid, $deviceid);
3559
3560     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3561
3562         qemu_devicedel($vmid, $deviceid);
3563         qemu_devicedelverify($vmid, $deviceid);
3564         qemu_netdevdel($vmid, $deviceid);
3565
3566     } else {
3567         die "can't unplug device '$deviceid'\n";
3568     }
3569
3570     return 1;
3571 }
3572
3573 sub qemu_deviceadd {
3574     my ($vmid, $devicefull) = @_;
3575
3576     $devicefull = "driver=".$devicefull;
3577     my %options =  split(/[=,]/, $devicefull);
3578
3579     vm_mon_cmd($vmid, "device_add" , %options);
3580 }
3581
3582 sub qemu_devicedel {
3583     my ($vmid, $deviceid) = @_;
3584
3585     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3586 }
3587
3588 sub qemu_iothread_add {
3589     my($vmid, $deviceid, $device) = @_;
3590
3591     if ($device->{iothread}) {
3592         my $iothreads = vm_iothreads_list($vmid);
3593         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3594     }
3595 }
3596
3597 sub qemu_iothread_del {
3598     my($conf, $vmid, $deviceid) = @_;
3599
3600     my $device = parse_drive($deviceid, $conf->{$deviceid});
3601     if ($device->{iothread}) {
3602         my $iothreads = vm_iothreads_list($vmid);
3603         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3604     }
3605 }
3606
3607 sub qemu_objectadd {
3608     my($vmid, $objectid, $qomtype) = @_;
3609
3610     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3611
3612     return 1;
3613 }
3614
3615 sub qemu_objectdel {
3616     my($vmid, $objectid) = @_;
3617
3618     vm_mon_cmd($vmid, "object-del", id => $objectid);
3619
3620     return 1;
3621 }
3622
3623 sub qemu_driveadd {
3624     my ($storecfg, $vmid, $device) = @_;
3625
3626     my $drive = print_drive_full($storecfg, $vmid, $device);
3627     $drive =~ s/\\/\\\\/g;
3628     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3629
3630     # If the command succeeds qemu prints: "OK"
3631     return 1 if $ret =~ m/OK/s;
3632
3633     die "adding drive failed: $ret\n";
3634 }
3635
3636 sub qemu_drivedel {
3637     my($vmid, $deviceid) = @_;
3638
3639     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3640     $ret =~ s/^\s+//;
3641
3642     return 1 if $ret eq "";
3643
3644     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3645     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3646
3647     die "deleting drive $deviceid failed : $ret\n";
3648 }
3649
3650 sub qemu_deviceaddverify {
3651     my ($vmid, $deviceid) = @_;
3652
3653     for (my $i = 0; $i <= 5; $i++) {
3654          my $devices_list = vm_devices_list($vmid);
3655          return 1 if defined($devices_list->{$deviceid});
3656          sleep 1;
3657     }
3658
3659     die "error on hotplug device '$deviceid'\n";
3660 }
3661
3662
3663 sub qemu_devicedelverify {
3664     my ($vmid, $deviceid) = @_;
3665
3666     # need to verify that the device is correctly removed as device_del
3667     # is async and empty return is not reliable
3668
3669     for (my $i = 0; $i <= 5; $i++) {
3670          my $devices_list = vm_devices_list($vmid);
3671          return 1 if !defined($devices_list->{$deviceid});
3672          sleep 1;
3673     }
3674
3675     die "error on hot-unplugging device '$deviceid'\n";
3676 }
3677
3678 sub qemu_findorcreatescsihw {
3679     my ($storecfg, $conf, $vmid, $device) = @_;
3680
3681     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3682
3683     my $scsihwid="$controller_prefix$controller";
3684     my $devices_list = vm_devices_list($vmid);
3685
3686     if(!defined($devices_list->{$scsihwid})) {
3687         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3688     }
3689
3690     return 1;
3691 }
3692
3693 sub qemu_deletescsihw {
3694     my ($conf, $vmid, $opt) = @_;
3695
3696     my $device = parse_drive($opt, $conf->{$opt});
3697
3698     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3699         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3700         return 1;
3701     }
3702
3703     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3704
3705     my $devices_list = vm_devices_list($vmid);
3706     foreach my $opt (keys %{$devices_list}) {
3707         if (PVE::QemuServer::is_valid_drivename($opt)) {
3708             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3709             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3710                 return 1;
3711             }
3712         }
3713     }
3714
3715     my $scsihwid="scsihw$controller";
3716
3717     vm_deviceunplug($vmid, $conf, $scsihwid);
3718
3719     return 1;
3720 }
3721
3722 sub qemu_add_pci_bridge {
3723     my ($storecfg, $conf, $vmid, $device) = @_;
3724
3725     my $bridges = {};
3726
3727     my $bridgeid;
3728
3729     print_pci_addr($device, $bridges);
3730
3731     while (my ($k, $v) = each %$bridges) {
3732         $bridgeid = $k;
3733     }
3734     return 1 if !defined($bridgeid) || $bridgeid < 1;
3735
3736     my $bridge = "pci.$bridgeid";
3737     my $devices_list = vm_devices_list($vmid);
3738
3739     if (!defined($devices_list->{$bridge})) {
3740         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3741     }
3742
3743     return 1;
3744 }
3745
3746 sub qemu_set_link_status {
3747     my ($vmid, $device, $up) = @_;
3748
3749     vm_mon_cmd($vmid, "set_link", name => $device,
3750                up => $up ? JSON::true : JSON::false);
3751 }
3752
3753 sub qemu_netdevadd {
3754     my ($vmid, $conf, $device, $deviceid) = @_;
3755
3756     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3757     my %options =  split(/[=,]/, $netdev);
3758
3759     vm_mon_cmd($vmid, "netdev_add",  %options);
3760     return 1;
3761 }
3762
3763 sub qemu_netdevdel {
3764     my ($vmid, $deviceid) = @_;
3765
3766     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3767 }
3768
3769 sub qemu_usb_hotplug {
3770     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3771
3772     return if !$device;
3773
3774     # remove the old one first
3775     vm_deviceunplug($vmid, $conf, $deviceid);
3776
3777     # check if xhci controller is necessary and available
3778     if ($device->{usb3}) {
3779
3780         my $devicelist = vm_devices_list($vmid);
3781
3782         if (!$devicelist->{xhci}) {
3783             my $pciaddr = print_pci_addr("xhci");
3784             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3785         }
3786     }
3787     my $d = parse_usb_device($device->{host});
3788     $d->{usb3} = $device->{usb3};
3789
3790     # add the new one
3791     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3792 }
3793
3794 sub qemu_cpu_hotplug {
3795     my ($vmid, $conf, $vcpus) = @_;
3796
3797     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3798
3799     my $sockets = 1;
3800     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3801     $sockets = $conf->{sockets} if  $conf->{sockets};
3802     my $cores = $conf->{cores} || 1;
3803     my $maxcpus = $sockets * $cores;
3804
3805     $vcpus = $maxcpus if !$vcpus;
3806
3807     die "you can't add more vcpus than maxcpus\n"
3808         if $vcpus > $maxcpus;
3809
3810     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3811
3812     if ($vcpus < $currentvcpus) {
3813
3814         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3815
3816             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3817                 qemu_devicedel($vmid, "cpu$i");
3818                 my $retry = 0;
3819                 my $currentrunningvcpus = undef;
3820                 while (1) {
3821                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3822                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3823                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3824                     $retry++;
3825                     sleep 1;
3826                 }
3827                 #update conf after each succesfull cpu unplug
3828                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3829                 PVE::QemuConfig->write_config($vmid, $conf);
3830             }
3831         } else {
3832             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3833         }
3834
3835         return;
3836     }
3837
3838     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3839     die "vcpus in running vm does not match its configuration\n"
3840         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3841
3842     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3843
3844         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3845             my $cpustr = print_cpu_device($conf, $i);
3846             qemu_deviceadd($vmid, $cpustr);
3847
3848             my $retry = 0;
3849             my $currentrunningvcpus = undef;
3850             while (1) {
3851                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3852                 last if scalar(@{$currentrunningvcpus}) == $i;
3853                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3854                 sleep 1;
3855                 $retry++;
3856             }
3857             #update conf after each succesfull cpu hotplug
3858             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3859             PVE::QemuConfig->write_config($vmid, $conf);
3860         }
3861     } else {
3862
3863         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3864             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3865         }
3866     }
3867 }
3868
3869 sub qemu_block_set_io_throttle {
3870     my ($vmid, $deviceid,
3871         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3872         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3873         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3874         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3875
3876     return if !check_running($vmid) ;
3877
3878     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3879         bps => int($bps),
3880         bps_rd => int($bps_rd),
3881         bps_wr => int($bps_wr),
3882         iops => int($iops),
3883         iops_rd => int($iops_rd),
3884         iops_wr => int($iops_wr),
3885         bps_max => int($bps_max),
3886         bps_rd_max => int($bps_rd_max),
3887         bps_wr_max => int($bps_wr_max),
3888         iops_max => int($iops_max),
3889         iops_rd_max => int($iops_rd_max),
3890         iops_wr_max => int($iops_wr_max),
3891         bps_max_length => int($bps_max_length),
3892         bps_rd_max_length => int($bps_rd_max_length),
3893         bps_wr_max_length => int($bps_wr_max_length),
3894         iops_max_length => int($iops_max_length),
3895         iops_rd_max_length => int($iops_rd_max_length),
3896         iops_wr_max_length => int($iops_wr_max_length),
3897     );
3898
3899 }
3900
3901 # old code, only used to shutdown old VM after update
3902 sub __read_avail {
3903     my ($fh, $timeout) = @_;
3904
3905     my $sel = new IO::Select;
3906     $sel->add($fh);
3907
3908     my $res = '';
3909     my $buf;
3910
3911     my @ready;
3912     while (scalar (@ready = $sel->can_read($timeout))) {
3913         my $count;
3914         if ($count = $fh->sysread($buf, 8192)) {
3915             if ($buf =~ /^(.*)\(qemu\) $/s) {
3916                 $res .= $1;
3917                 last;
3918             } else {
3919                 $res .= $buf;
3920             }
3921         } else {
3922             if (!defined($count)) {
3923                 die "$!\n";
3924             }
3925             last;
3926         }
3927     }
3928
3929     die "monitor read timeout\n" if !scalar(@ready);
3930
3931     return $res;
3932 }
3933
3934 # old code, only used to shutdown old VM after update
3935 sub vm_monitor_command {
3936     my ($vmid, $cmdstr, $nocheck) = @_;
3937
3938     my $res;
3939
3940     eval {
3941         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3942
3943         my $sname = "${var_run_tmpdir}/$vmid.mon";
3944
3945         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3946             die "unable to connect to VM $vmid socket - $!\n";
3947
3948         my $timeout = 3;
3949
3950         # hack: migrate sometime blocks the monitor (when migrate_downtime
3951         # is set)
3952         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3953             $timeout = 60*60; # 1 hour
3954         }
3955
3956         # read banner;
3957         my $data = __read_avail($sock, $timeout);
3958
3959         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3960             die "got unexpected qemu monitor banner\n";
3961         }
3962
3963         my $sel = new IO::Select;
3964         $sel->add($sock);
3965
3966         if (!scalar(my @ready = $sel->can_write($timeout))) {
3967             die "monitor write error - timeout";
3968         }
3969
3970         my $fullcmd = "$cmdstr\r";
3971
3972         # syslog('info', "VM $vmid monitor command: $cmdstr");
3973
3974         my $b;
3975         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3976             die "monitor write error - $!";
3977         }
3978
3979         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3980
3981         $timeout = 20;
3982
3983         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3984             $timeout = 60*60; # 1 hour
3985         } elsif ($cmdstr =~ m/^(eject|change)/) {
3986             $timeout = 60; # note: cdrom mount command is slow
3987         }
3988         if ($res = __read_avail($sock, $timeout)) {
3989
3990             my @lines = split("\r?\n", $res);
3991
3992             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3993
3994             $res = join("\n", @lines);
3995             $res .= "\n";
3996         }
3997     };
3998
3999     my $err = $@;
4000
4001     if ($err) {
4002         syslog("err", "VM $vmid monitor command failed - $err");
4003         die $err;
4004     }
4005
4006     return $res;
4007 }
4008
4009 sub qemu_block_resize {
4010     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4011
4012     my $running = check_running($vmid);
4013
4014     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4015
4016     return if !$running;
4017
4018     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4019
4020 }
4021
4022 sub qemu_volume_snapshot {
4023     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4024
4025     my $running = check_running($vmid);
4026
4027     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4028         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4029     } else {
4030         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4031     }
4032 }
4033
4034 sub qemu_volume_snapshot_delete {
4035     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4036
4037     my $running = check_running($vmid);
4038
4039     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4040         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4041     } else {
4042         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4043     }
4044 }
4045
4046 sub set_migration_caps {
4047     my ($vmid) = @_;
4048
4049     my $cap_ref = [];
4050
4051     my $enabled_cap = {
4052         "auto-converge" => 1,
4053         "xbzrle" => 1,
4054         "x-rdma-pin-all" => 0,
4055         "zero-blocks" => 0,
4056         "compress" => 0
4057     };
4058
4059     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4060
4061     for my $supported_capability (@$supported_capabilities) {
4062         push @$cap_ref, {
4063             capability => $supported_capability->{capability},
4064             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4065         };
4066     }
4067
4068     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4069 }
4070
4071 my $fast_plug_option = {
4072     'lock' => 1,
4073     'name' => 1,
4074     'onboot' => 1,
4075     'shares' => 1,
4076     'startup' => 1,
4077     'description' => 1,
4078     'protection' => 1,
4079 };
4080
4081 # hotplug changes in [PENDING]
4082 # $selection hash can be used to only apply specified options, for
4083 # example: { cores => 1 } (only apply changed 'cores')
4084 # $errors ref is used to return error messages
4085 sub vmconfig_hotplug_pending {
4086     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4087
4088     my $defaults = load_defaults();
4089
4090     # commit values which do not have any impact on running VM first
4091     # Note: those option cannot raise errors, we we do not care about
4092     # $selection and always apply them.
4093
4094     my $add_error = sub {
4095         my ($opt, $msg) = @_;
4096         $errors->{$opt} = "hotplug problem - $msg";
4097     };
4098
4099     my $changes = 0;
4100     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4101         if ($fast_plug_option->{$opt}) {
4102             $conf->{$opt} = $conf->{pending}->{$opt};
4103             delete $conf->{pending}->{$opt};
4104             $changes = 1;
4105         }
4106     }
4107
4108     if ($changes) {
4109         PVE::QemuConfig->write_config($vmid, $conf);
4110         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4111     }
4112
4113     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4114
4115     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4116     while (my ($opt, $force) = each %$pending_delete_hash) {
4117         next if $selection && !$selection->{$opt};
4118         eval {
4119             if ($opt eq 'hotplug') {
4120                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4121             } elsif ($opt eq 'tablet') {
4122                 die "skip\n" if !$hotplug_features->{usb};
4123                 if ($defaults->{tablet}) {
4124                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4125                 } else {
4126                     vm_deviceunplug($vmid, $conf, $opt);
4127                 }
4128             } elsif ($opt =~ m/^usb\d+/) {
4129                 die "skip\n";
4130                 # since we cannot reliably hot unplug usb devices
4131                 # we are disabling it
4132                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4133                 vm_deviceunplug($vmid, $conf, $opt);
4134             } elsif ($opt eq 'vcpus') {
4135                 die "skip\n" if !$hotplug_features->{cpu};
4136                 qemu_cpu_hotplug($vmid, $conf, undef);
4137             } elsif ($opt eq 'balloon') {
4138                 # enable balloon device is not hotpluggable
4139                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4140             } elsif ($fast_plug_option->{$opt}) {
4141                 # do nothing
4142             } elsif ($opt =~ m/^net(\d+)$/) {
4143                 die "skip\n" if !$hotplug_features->{network};
4144                 vm_deviceunplug($vmid, $conf, $opt);
4145             } elsif (is_valid_drivename($opt)) {
4146                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4147                 vm_deviceunplug($vmid, $conf, $opt);
4148                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4149             } elsif ($opt =~ m/^memory$/) {
4150                 die "skip\n" if !$hotplug_features->{memory};
4151                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4152             } elsif ($opt eq 'cpuunits') {
4153                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4154             } elsif ($opt eq 'cpulimit') {
4155                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4156             } else {
4157                 die "skip\n";
4158             }
4159         };
4160         if (my $err = $@) {
4161             &$add_error($opt, $err) if $err ne "skip\n";
4162         } else {
4163             # save new config if hotplug was successful
4164             delete $conf->{$opt};
4165             vmconfig_undelete_pending_option($conf, $opt);
4166             PVE::QemuConfig->write_config($vmid, $conf);
4167             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4168         }
4169     }
4170
4171     foreach my $opt (keys %{$conf->{pending}}) {
4172         next if $selection && !$selection->{$opt};
4173         my $value = $conf->{pending}->{$opt};
4174         eval {
4175             if ($opt eq 'hotplug') {
4176                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4177             } elsif ($opt eq 'tablet') {
4178                 die "skip\n" if !$hotplug_features->{usb};
4179                 if ($value == 1) {
4180                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4181                 } elsif ($value == 0) {
4182                     vm_deviceunplug($vmid, $conf, $opt);
4183                 }
4184             } elsif ($opt =~ m/^usb\d+$/) {
4185                 die "skip\n";
4186                 # since we cannot reliably hot unplug usb devices
4187                 # we are disabling it
4188                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4189                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4190                 die "skip\n" if !$d;
4191                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4192             } elsif ($opt eq 'vcpus') {
4193                 die "skip\n" if !$hotplug_features->{cpu};
4194                 qemu_cpu_hotplug($vmid, $conf, $value);
4195             } elsif ($opt eq 'balloon') {
4196                 # enable/disable balloning device is not hotpluggable
4197                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4198                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4199                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4200
4201                 # allow manual ballooning if shares is set to zero
4202                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4203                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4204                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4205                 }
4206             } elsif ($opt =~ m/^net(\d+)$/) {
4207                 # some changes can be done without hotplug
4208                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4209                                     $vmid, $opt, $value);
4210             } elsif (is_valid_drivename($opt)) {
4211                 # some changes can be done without hotplug
4212                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4213                                      $vmid, $opt, $value, 1);
4214             } elsif ($opt =~ m/^memory$/) { #dimms
4215                 die "skip\n" if !$hotplug_features->{memory};
4216                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4217             } elsif ($opt eq 'cpuunits') {
4218                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4219             } elsif ($opt eq 'cpulimit') {
4220                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4221                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4222             } else {
4223                 die "skip\n";  # skip non-hot-pluggable options
4224             }
4225         };
4226         if (my $err = $@) {
4227             &$add_error($opt, $err) if $err ne "skip\n";
4228         } else {
4229             # save new config if hotplug was successful
4230             $conf->{$opt} = $value;
4231             delete $conf->{pending}->{$opt};
4232             PVE::QemuConfig->write_config($vmid, $conf);
4233             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4234         }
4235     }
4236 }
4237
4238 sub try_deallocate_drive {
4239     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4240
4241     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4242         my $volid = $drive->{file};
4243         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4244             my $sid = PVE::Storage::parse_volume_id($volid);
4245             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4246
4247             # check if the disk is really unused
4248             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4249                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4250             PVE::Storage::vdisk_free($storecfg, $volid);
4251             return 1;
4252         } else {
4253             # If vm is not owner of this disk remove from config
4254             return 1;
4255         }
4256     }
4257
4258     return undef;
4259 }
4260
4261 sub vmconfig_delete_or_detach_drive {
4262     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4263
4264     my $drive = parse_drive($opt, $conf->{$opt});
4265
4266     my $rpcenv = PVE::RPCEnvironment::get();
4267     my $authuser = $rpcenv->get_user();
4268
4269     if ($force) {
4270         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4271         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4272     } else {
4273         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4274     }
4275 }
4276
4277 sub vmconfig_apply_pending {
4278     my ($vmid, $conf, $storecfg) = @_;
4279
4280     # cold plug
4281
4282     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4283     while (my ($opt, $force) = each %$pending_delete_hash) {
4284         die "internal error" if $opt =~ m/^unused/;
4285         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4286         if (!defined($conf->{$opt})) {
4287             vmconfig_undelete_pending_option($conf, $opt);
4288             PVE::QemuConfig->write_config($vmid, $conf);
4289         } elsif (is_valid_drivename($opt)) {
4290             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4291             vmconfig_undelete_pending_option($conf, $opt);
4292             delete $conf->{$opt};
4293             PVE::QemuConfig->write_config($vmid, $conf);
4294         } else {
4295             vmconfig_undelete_pending_option($conf, $opt);
4296             delete $conf->{$opt};
4297             PVE::QemuConfig->write_config($vmid, $conf);
4298         }
4299     }
4300
4301     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4302
4303     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4304         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4305
4306         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4307             # skip if nothing changed
4308         } elsif (is_valid_drivename($opt)) {
4309             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4310                 if defined($conf->{$opt});
4311             $conf->{$opt} = $conf->{pending}->{$opt};
4312         } else {
4313             $conf->{$opt} = $conf->{pending}->{$opt};
4314         }
4315
4316         delete $conf->{pending}->{$opt};
4317         PVE::QemuConfig->write_config($vmid, $conf);
4318     }
4319 }
4320
4321 my $safe_num_ne = sub {
4322     my ($a, $b) = @_;
4323
4324     return 0 if !defined($a) && !defined($b);
4325     return 1 if !defined($a);
4326     return 1 if !defined($b);
4327
4328     return $a != $b;
4329 };
4330
4331 my $safe_string_ne = sub {
4332     my ($a, $b) = @_;
4333
4334     return 0 if !defined($a) && !defined($b);
4335     return 1 if !defined($a);
4336     return 1 if !defined($b);
4337
4338     return $a ne $b;
4339 };
4340
4341 sub vmconfig_update_net {
4342     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4343
4344     my $newnet = parse_net($value);
4345
4346     if ($conf->{$opt}) {
4347         my $oldnet = parse_net($conf->{$opt});
4348
4349         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4350             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4351             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4352             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4353
4354             # for non online change, we try to hot-unplug
4355             die "skip\n" if !$hotplug;
4356             vm_deviceunplug($vmid, $conf, $opt);
4357         } else {
4358
4359             die "internal error" if $opt !~ m/net(\d+)/;
4360             my $iface = "tap${vmid}i$1";
4361
4362             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4363                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4364                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4365                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4366                 PVE::Network::tap_unplug($iface);
4367                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4368             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4369                 # Rate can be applied on its own but any change above needs to
4370                 # include the rate in tap_plug since OVS resets everything.
4371                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4372             }
4373
4374             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4375                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4376             }
4377
4378             return 1;
4379         }
4380     }
4381
4382     if ($hotplug) {
4383         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4384     } else {
4385         die "skip\n";
4386     }
4387 }
4388
4389 sub vmconfig_update_disk {
4390     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4391
4392     # fixme: do we need force?
4393
4394     my $drive = parse_drive($opt, $value);
4395
4396     if ($conf->{$opt}) {
4397
4398         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4399
4400             my $media = $drive->{media} || 'disk';
4401             my $oldmedia = $old_drive->{media} || 'disk';
4402             die "unable to change media type\n" if $media ne $oldmedia;
4403
4404             if (!drive_is_cdrom($old_drive)) {
4405
4406                 if ($drive->{file} ne $old_drive->{file}) {
4407
4408                     die "skip\n" if !$hotplug;
4409
4410                     # unplug and register as unused
4411                     vm_deviceunplug($vmid, $conf, $opt);
4412                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4413
4414                 } else {
4415                     # update existing disk
4416
4417                     # skip non hotpluggable value
4418                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4419                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4420                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4421                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4422                         die "skip\n";
4423                     }
4424
4425                     # apply throttle
4426                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4427                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4428                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4429                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4430                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4431                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4432                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4433                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4434                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4435                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4436                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4437                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4438                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4439                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4440                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4441                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4442                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4443                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4444
4445                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4446                                                    ($drive->{mbps} || 0)*1024*1024,
4447                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4448                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4449                                                    $drive->{iops} || 0,
4450                                                    $drive->{iops_rd} || 0,
4451                                                    $drive->{iops_wr} || 0,
4452                                                    ($drive->{mbps_max} || 0)*1024*1024,
4453                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4454                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4455                                                    $drive->{iops_max} || 0,
4456                                                    $drive->{iops_rd_max} || 0,
4457                                                    $drive->{iops_wr_max} || 0,
4458                                                    $drive->{bps_max_length} || 1,
4459                                                    $drive->{bps_rd_max_length} || 1,
4460                                                    $drive->{bps_wr_max_length} || 1,
4461                                                    $drive->{iops_max_length} || 1,
4462                                                    $drive->{iops_rd_max_length} || 1,
4463                                                    $drive->{iops_wr_max_length} || 1);
4464
4465                     }
4466
4467                     return 1;
4468                 }
4469
4470             } else { # cdrom
4471
4472                 if ($drive->{file} eq 'none') {
4473                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4474                 } else {
4475                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4476                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4477                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4478                 }
4479
4480                 return 1;
4481             }
4482         }
4483     }
4484
4485     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4486     # hotplug new disks
4487     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4488     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4489 }
4490
4491 sub vm_start {
4492     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4493         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4494
4495     PVE::QemuConfig->lock_config($vmid, sub {
4496         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4497
4498         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4499
4500         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4501
4502         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4503
4504         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4505             vmconfig_apply_pending($vmid, $conf, $storecfg);
4506             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4507         }
4508
4509         my $defaults = load_defaults();
4510
4511         # set environment variable useful inside network script
4512         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4513
4514         my $local_volumes = {};
4515
4516         if ($targetstorage) {
4517             foreach_drive($conf, sub {
4518                 my ($ds, $drive) = @_;
4519
4520                 return if drive_is_cdrom($drive);
4521
4522                 my $volid = $drive->{file};
4523
4524                 return if !$volid;
4525
4526                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4527
4528                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4529                 return if $scfg->{shared};
4530                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4531             });
4532
4533             my $format = undef;
4534
4535             foreach my $opt (sort keys %$local_volumes) {
4536
4537                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4538                 my $drive = parse_drive($opt, $conf->{$opt});
4539
4540                 #if remote storage is specified, use default format
4541                 if ($targetstorage && $targetstorage ne "1") {
4542                     $storeid = $targetstorage;
4543                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4544                     $format = $defFormat;
4545                 } else {
4546                     #else we use same format than original
4547                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4548                     $format = qemu_img_format($scfg, $volid);
4549                 }
4550
4551                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4552                 my $newdrive = $drive;
4553                 $newdrive->{format} = $format;
4554                 $newdrive->{file} = $newvolid;
4555                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4556                 $local_volumes->{$opt} = $drivestr;
4557                 #pass drive to conf for command line
4558                 $conf->{$opt} = $drivestr;
4559             }
4560         }
4561
4562         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4563
4564         my $migrate_port = 0;
4565         my $migrate_uri;
4566         if ($statefile) {
4567             if ($statefile eq 'tcp') {
4568                 my $localip = "localhost";
4569                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4570                 my $nodename = PVE::INotify::nodename();
4571
4572                 if (!defined($migration_type)) {
4573                     if (defined($datacenterconf->{migration}->{type})) {
4574                         $migration_type = $datacenterconf->{migration}->{type};
4575                     } else {
4576                         $migration_type = 'secure';
4577                     }
4578                 }
4579
4580                 if ($migration_type eq 'insecure') {
4581                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4582                     if ($migrate_network_addr) {
4583                         $localip = $migrate_network_addr;
4584                     } else {
4585                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4586                     }
4587
4588                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4589                 }
4590
4591                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4592                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4593                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4594                 push @$cmd, '-incoming', $migrate_uri;
4595                 push @$cmd, '-S';
4596
4597             } elsif ($statefile eq 'unix') {
4598                 # should be default for secure migrations as a ssh TCP forward
4599                 # tunnel is not deterministic reliable ready and fails regurarly
4600                 # to set up in time, so use UNIX socket forwards
4601                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4602                 unlink $socket_addr;
4603
4604                 $migrate_uri = "unix:$socket_addr";
4605
4606                 push @$cmd, '-incoming', $migrate_uri;
4607                 push @$cmd, '-S';
4608
4609             } else {
4610                 push @$cmd, '-loadstate', $statefile;
4611             }
4612         } elsif ($paused) {
4613             push @$cmd, '-S';
4614         }
4615
4616         # host pci devices
4617         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4618           my $d = parse_hostpci($conf->{"hostpci$i"});
4619           next if !$d;
4620           my $pcidevices = $d->{pciid};
4621           foreach my $pcidevice (@$pcidevices) {
4622                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4623
4624                 my $info = pci_device_info("0000:$pciid");
4625                 die "IOMMU not present\n" if !check_iommu_support();
4626                 die "no pci device info for device '$pciid'\n" if !$info;
4627                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4628                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4629           }
4630         }
4631
4632         PVE::Storage::activate_volumes($storecfg, $vollist);
4633
4634         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4635             my $cmd = [];
4636             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4637             eval  { run_command($cmd); };
4638         }
4639
4640         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4641                                                   : $defaults->{cpuunits};
4642
4643         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4644
4645         my %properties = (
4646             Slice => 'qemu.slice',
4647             KillMode => 'none',
4648             CPUShares => $cpuunits
4649         );
4650
4651         if (my $cpulimit = $conf->{cpulimit}) {
4652             $properties{CPUQuota} = int($cpulimit * 100);
4653         }
4654         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4655
4656         if ($conf->{hugepages}) {
4657
4658             my $code = sub {
4659                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4660                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4661
4662                 PVE::QemuServer::Memory::hugepages_mount();
4663                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4664
4665                 eval  {
4666                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4667                     run_command($cmd, %run_params);
4668                 };
4669
4670                 if (my $err = $@) {
4671                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4672                     die $err;
4673                 }
4674
4675                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4676             };
4677             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4678
4679         } else {
4680             eval  {
4681                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4682                 run_command($cmd, %run_params);
4683             };
4684         }
4685
4686         if (my $err = $@) {
4687             # deactivate volumes if start fails
4688             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4689             die "start failed: $err";
4690         }
4691
4692         print "migration listens on $migrate_uri\n" if $migrate_uri;
4693
4694         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4695             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4696             warn $@ if $@;
4697         }
4698
4699         #start nbd server for storage migration
4700         if ($targetstorage) {
4701             my $nodename = PVE::INotify::nodename();
4702             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4703             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4704             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4705             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4706
4707             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4708
4709             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4710
4711             foreach my $opt (sort keys %$local_volumes) {
4712                 my $volid = $local_volumes->{$opt};
4713                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4714                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4715                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4716             }
4717         }
4718
4719         if ($migratedfrom) {
4720             eval {
4721                 set_migration_caps($vmid);
4722             };
4723             warn $@ if $@;
4724
4725             if ($spice_port) {
4726                 print "spice listens on port $spice_port\n";
4727                 if ($spice_ticket) {
4728                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4729                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4730                 }
4731             }
4732
4733         } else {
4734             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4735                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4736                     if $conf->{balloon};
4737             }
4738
4739             foreach my $opt (keys %$conf) {
4740                 next if $opt !~  m/^net\d+$/;
4741                 my $nicconf = parse_net($conf->{$opt});
4742                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4743             }
4744         }
4745
4746         vm_mon_cmd_nocheck($vmid, 'qom-set',
4747                     path => "machine/peripheral/balloon0",
4748                     property => "guest-stats-polling-interval",
4749                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4750
4751     });
4752 }
4753
4754 sub vm_mon_cmd {
4755     my ($vmid, $execute, %params) = @_;
4756
4757     my $cmd = { execute => $execute, arguments => \%params };
4758     vm_qmp_command($vmid, $cmd);
4759 }
4760
4761 sub vm_mon_cmd_nocheck {
4762     my ($vmid, $execute, %params) = @_;
4763
4764     my $cmd = { execute => $execute, arguments => \%params };
4765     vm_qmp_command($vmid, $cmd, 1);
4766 }
4767
4768 sub vm_qmp_command {
4769     my ($vmid, $cmd, $nocheck) = @_;
4770
4771     my $res;
4772
4773     my $timeout;
4774     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4775         $timeout = $cmd->{arguments}->{timeout};
4776         delete $cmd->{arguments}->{timeout};
4777     }
4778
4779     eval {
4780         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4781         my $sname = qmp_socket($vmid);
4782         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4783             my $qmpclient = PVE::QMPClient->new();
4784
4785             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4786         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4787             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4788                 if scalar(%{$cmd->{arguments}});
4789             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4790         } else {
4791             die "unable to open monitor socket\n";
4792         }
4793     };
4794     if (my $err = $@) {
4795         syslog("err", "VM $vmid qmp command failed - $err");
4796         die $err;
4797     }
4798
4799     return $res;
4800 }
4801
4802 sub vm_human_monitor_command {
4803     my ($vmid, $cmdline) = @_;
4804
4805     my $res;
4806
4807     my $cmd = {
4808         execute => 'human-monitor-command',
4809         arguments => { 'command-line' => $cmdline},
4810     };
4811
4812     return vm_qmp_command($vmid, $cmd);
4813 }
4814
4815 sub vm_commandline {
4816     my ($storecfg, $vmid) = @_;
4817
4818     my $conf = PVE::QemuConfig->load_config($vmid);
4819
4820     my $defaults = load_defaults();
4821
4822     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4823
4824     return PVE::Tools::cmd2string($cmd);
4825 }
4826
4827 sub vm_reset {
4828     my ($vmid, $skiplock) = @_;
4829
4830     PVE::QemuConfig->lock_config($vmid, sub {
4831
4832         my $conf = PVE::QemuConfig->load_config($vmid);
4833
4834         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4835
4836         vm_mon_cmd($vmid, "system_reset");
4837     });
4838 }
4839
4840 sub get_vm_volumes {
4841     my ($conf) = @_;
4842
4843     my $vollist = [];
4844     foreach_volid($conf, sub {
4845         my ($volid, $attr) = @_;
4846
4847         return if $volid =~ m|^/|;
4848
4849         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4850         return if !$sid;
4851
4852         push @$vollist, $volid;
4853     });
4854
4855     return $vollist;
4856 }
4857
4858 sub vm_stop_cleanup {
4859     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4860
4861     eval {
4862
4863         if (!$keepActive) {
4864             my $vollist = get_vm_volumes($conf);
4865             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4866         }
4867
4868         foreach my $ext (qw(mon qmp pid vnc qga)) {
4869             unlink "/var/run/qemu-server/${vmid}.$ext";
4870         }
4871
4872         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4873     };
4874     warn $@ if $@; # avoid errors - just warn
4875 }
4876
4877 # Note: use $nockeck to skip tests if VM configuration file exists.
4878 # We need that when migration VMs to other nodes (files already moved)
4879 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4880 sub vm_stop {
4881     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4882
4883     $force = 1 if !defined($force) && !$shutdown;
4884
4885     if ($migratedfrom){
4886         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4887         kill 15, $pid if $pid;
4888         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4889         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4890         return;
4891     }
4892
4893     PVE::QemuConfig->lock_config($vmid, sub {
4894
4895         my $pid = check_running($vmid, $nocheck);
4896         return if !$pid;
4897
4898         my $conf;
4899         if (!$nocheck) {
4900             $conf = PVE::QemuConfig->load_config($vmid);
4901             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4902             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4903                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4904                 $timeout = $opts->{down} if $opts->{down};
4905             }
4906         }
4907
4908         $timeout = 60 if !defined($timeout);
4909
4910         eval {
4911             if ($shutdown) {
4912                 if (defined($conf) && $conf->{agent}) {
4913                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4914                 } else {
4915                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4916                 }
4917             } else {
4918                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4919             }
4920         };
4921         my $err = $@;
4922
4923         if (!$err) {
4924             my $count = 0;
4925             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4926                 $count++;
4927                 sleep 1;
4928             }
4929
4930             if ($count >= $timeout) {
4931                 if ($force) {
4932                     warn "VM still running - terminating now with SIGTERM\n";
4933                     kill 15, $pid;
4934                 } else {
4935                     die "VM quit/powerdown failed - got timeout\n";
4936                 }
4937             } else {
4938                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4939                 return;
4940             }
4941         } else {
4942             if ($force) {
4943                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4944                 kill 15, $pid;
4945             } else {
4946                 die "VM quit/powerdown failed\n";
4947             }
4948         }
4949
4950         # wait again
4951         $timeout = 10;
4952
4953         my $count = 0;
4954         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4955             $count++;
4956             sleep 1;
4957         }
4958
4959         if ($count >= $timeout) {
4960             warn "VM still running - terminating now with SIGKILL\n";
4961             kill 9, $pid;
4962             sleep 1;
4963         }
4964
4965         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4966    });
4967 }
4968
4969 sub vm_suspend {
4970     my ($vmid, $skiplock) = @_;
4971
4972     PVE::QemuConfig->lock_config($vmid, sub {
4973
4974         my $conf = PVE::QemuConfig->load_config($vmid);
4975
4976         PVE::QemuConfig->check_lock($conf)
4977             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4978
4979         vm_mon_cmd($vmid, "stop");
4980     });
4981 }
4982
4983 sub vm_resume {
4984     my ($vmid, $skiplock, $nocheck) = @_;
4985
4986     PVE::QemuConfig->lock_config($vmid, sub {
4987
4988         if (!$nocheck) {
4989
4990             my $conf = PVE::QemuConfig->load_config($vmid);
4991
4992             PVE::QemuConfig->check_lock($conf)
4993                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4994
4995             vm_mon_cmd($vmid, "cont");
4996
4997         } else {
4998             vm_mon_cmd_nocheck($vmid, "cont");
4999         }
5000     });
5001 }
5002
5003 sub vm_sendkey {
5004     my ($vmid, $skiplock, $key) = @_;
5005
5006     PVE::QemuConfig->lock_config($vmid, sub {
5007
5008         my $conf = PVE::QemuConfig->load_config($vmid);
5009
5010         # there is no qmp command, so we use the human monitor command
5011         vm_human_monitor_command($vmid, "sendkey $key");
5012     });
5013 }
5014
5015 sub vm_destroy {
5016     my ($storecfg, $vmid, $skiplock) = @_;
5017
5018     PVE::QemuConfig->lock_config($vmid, sub {
5019
5020         my $conf = PVE::QemuConfig->load_config($vmid);
5021
5022         if (!check_running($vmid)) {
5023             destroy_vm($storecfg, $vmid, undef, $skiplock);
5024         } else {
5025             die "VM $vmid is running - destroy failed\n";
5026         }
5027     });
5028 }
5029
5030 # pci helpers
5031
5032 sub file_write {
5033     my ($filename, $buf) = @_;
5034
5035     my $fh = IO::File->new($filename, "w");
5036     return undef if !$fh;
5037
5038     my $res = print $fh $buf;
5039
5040     $fh->close();
5041
5042     return $res;
5043 }
5044
5045 sub pci_device_info {
5046     my ($name) = @_;
5047
5048     my $res;
5049
5050     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5051     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5052
5053     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5054     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5055
5056     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5057     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5058
5059     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5060     return undef if !defined($product) || $product !~ s/^0x//;
5061
5062     $res = {
5063         name => $name,
5064         vendor => $vendor,
5065         product => $product,
5066         domain => $domain,
5067         bus => $bus,
5068         slot => $slot,
5069         func => $func,
5070         irq => $irq,
5071         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5072     };
5073
5074     return $res;
5075 }
5076
5077 sub pci_dev_reset {
5078     my ($dev) = @_;
5079
5080     my $name = $dev->{name};
5081
5082     my $fn = "$pcisysfs/devices/$name/reset";
5083
5084     return file_write($fn, "1");
5085 }
5086
5087 sub pci_dev_bind_to_vfio {
5088     my ($dev) = @_;
5089
5090     my $name = $dev->{name};
5091
5092     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5093
5094     if (!-d $vfio_basedir) {
5095         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5096     }
5097     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5098
5099     my $testdir = "$vfio_basedir/$name";
5100     return 1 if -d $testdir;
5101
5102     my $data = "$dev->{vendor} $dev->{product}";
5103     return undef if !file_write("$vfio_basedir/new_id", $data);
5104
5105     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5106     if (!file_write($fn, $name)) {
5107         return undef if -f $fn;
5108     }
5109
5110     $fn = "$vfio_basedir/bind";
5111     if (! -d $testdir) {
5112         return undef if !file_write($fn, $name);
5113     }
5114
5115     return -d $testdir;
5116 }
5117
5118 sub pci_dev_group_bind_to_vfio {
5119     my ($pciid) = @_;
5120
5121     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5122
5123     if (!-d $vfio_basedir) {
5124         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5125     }
5126     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5127
5128     # get IOMMU group devices
5129     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5130       my @devs = grep /^0000:/, readdir($D);
5131     closedir($D);
5132
5133     foreach my $pciid (@devs) {
5134         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5135
5136         # pci bridges, switches or root ports are not supported
5137         # they have a pci_bus subdirectory so skip them
5138         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5139
5140         my $info = pci_device_info($1);
5141         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5142     }
5143
5144     return 1;
5145 }
5146
5147 # vzdump restore implementaion
5148
5149 sub tar_archive_read_firstfile {
5150     my $archive = shift;
5151
5152     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5153
5154     # try to detect archive type first
5155     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5156         die "unable to open file '$archive'\n";
5157     my $firstfile = <$fh>;
5158     kill 15, $pid;
5159     close $fh;
5160
5161     die "ERROR: archive contaions no data\n" if !$firstfile;
5162     chomp $firstfile;
5163
5164     return $firstfile;
5165 }
5166
5167 sub tar_restore_cleanup {
5168     my ($storecfg, $statfile) = @_;
5169
5170     print STDERR "starting cleanup\n";
5171
5172     if (my $fd = IO::File->new($statfile, "r")) {
5173         while (defined(my $line = <$fd>)) {
5174             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5175                 my $volid = $2;
5176                 eval {
5177                     if ($volid =~ m|^/|) {
5178                         unlink $volid || die 'unlink failed\n';
5179                     } else {
5180                         PVE::Storage::vdisk_free($storecfg, $volid);
5181                     }
5182                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5183                 };
5184                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5185             } else {
5186                 print STDERR "unable to parse line in statfile - $line";
5187             }
5188         }
5189         $fd->close();
5190     }
5191 }
5192
5193 sub restore_archive {
5194     my ($archive, $vmid, $user, $opts) = @_;
5195
5196     my $format = $opts->{format};
5197     my $comp;
5198
5199     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5200         $format = 'tar' if !$format;
5201         $comp = 'gzip';
5202     } elsif ($archive =~ m/\.tar$/) {
5203         $format = 'tar' if !$format;
5204     } elsif ($archive =~ m/.tar.lzo$/) {
5205         $format = 'tar' if !$format;
5206         $comp = 'lzop';
5207     } elsif ($archive =~ m/\.vma$/) {
5208         $format = 'vma' if !$format;
5209     } elsif ($archive =~ m/\.vma\.gz$/) {
5210         $format = 'vma' if !$format;
5211         $comp = 'gzip';
5212     } elsif ($archive =~ m/\.vma\.lzo$/) {
5213         $format = 'vma' if !$format;
5214         $comp = 'lzop';
5215     } else {
5216         $format = 'vma' if !$format; # default
5217     }
5218
5219     # try to detect archive format
5220     if ($format eq 'tar') {
5221         return restore_tar_archive($archive, $vmid, $user, $opts);
5222     } else {
5223         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5224     }
5225 }
5226
5227 sub restore_update_config_line {
5228     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5229
5230     return if $line =~ m/^\#qmdump\#/;
5231     return if $line =~ m/^\#vzdump\#/;
5232     return if $line =~ m/^lock:/;
5233     return if $line =~ m/^unused\d+:/;
5234     return if $line =~ m/^parent:/;
5235     return if $line =~ m/^template:/; # restored VM is never a template
5236
5237     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5238     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5239         # try to convert old 1.X settings
5240         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5241         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5242             my ($model, $macaddr) = split(/\=/, $devconfig);
5243             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5244             my $net = {
5245                 model => $model,
5246                 bridge => "vmbr$ind",
5247                 macaddr => $macaddr,
5248             };
5249             my $netstr = print_net($net);
5250
5251             print $outfd "net$cookie->{netcount}: $netstr\n";
5252             $cookie->{netcount}++;
5253         }
5254     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5255         my ($id, $netstr) = ($1, $2);
5256         my $net = parse_net($netstr);
5257         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5258         $netstr = print_net($net);
5259         print $outfd "$id: $netstr\n";
5260     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5261         my $virtdev = $1;
5262         my $value = $3;
5263         my $di = parse_drive($virtdev, $value);
5264         if (defined($di->{backup}) && !$di->{backup}) {
5265             print $outfd "#$line";
5266         } elsif ($map->{$virtdev}) {
5267             delete $di->{format}; # format can change on restore
5268             $di->{file} = $map->{$virtdev};
5269             $value = print_drive($vmid, $di);
5270             print $outfd "$virtdev: $value\n";
5271         } else {
5272             print $outfd $line;
5273         }
5274     } else {
5275         print $outfd $line;
5276     }
5277 }
5278
5279 sub scan_volids {
5280     my ($cfg, $vmid) = @_;
5281
5282     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5283
5284     my $volid_hash = {};
5285     foreach my $storeid (keys %$info) {
5286         foreach my $item (@{$info->{$storeid}}) {
5287             next if !($item->{volid} && $item->{size});
5288             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5289             $volid_hash->{$item->{volid}} = $item;
5290         }
5291     }
5292
5293     return $volid_hash;
5294 }
5295
5296 sub is_volume_in_use {
5297     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5298
5299     my $path = PVE::Storage::path($storecfg, $volid);
5300
5301     my $scan_config = sub {
5302         my ($cref, $snapname) = @_;
5303
5304         foreach my $key (keys %$cref) {
5305             my $value = $cref->{$key};
5306             if (is_valid_drivename($key)) {
5307                 next if $skip_drive && $key eq $skip_drive;
5308                 my $drive = parse_drive($key, $value);
5309                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5310                 return 1 if $volid eq $drive->{file};
5311                 if ($drive->{file} =~ m!^/!) {
5312                     return 1 if $drive->{file} eq $path;
5313                 } else {
5314                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5315                     next if !$storeid;
5316                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5317                     next if !$scfg;
5318                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5319                 }
5320             }
5321         }
5322
5323         return 0;
5324     };
5325
5326     return 1 if &$scan_config($conf);
5327
5328     undef $skip_drive;
5329
5330     foreach my $snapname (keys %{$conf->{snapshots}}) {
5331         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5332     }
5333
5334     return 0;
5335 }
5336
5337 sub update_disksize {
5338     my ($vmid, $conf, $volid_hash) = @_;
5339
5340     my $changes;
5341
5342     my $used = {};
5343
5344     # Note: it is allowed to define multiple storages with same path (alias), so
5345     # we need to check both 'volid' and real 'path' (two different volid can point
5346     # to the same path).
5347
5348     my $usedpath = {};
5349
5350     # update size info
5351     foreach my $opt (keys %$conf) {
5352         if (is_valid_drivename($opt)) {
5353             my $drive = parse_drive($opt, $conf->{$opt});
5354             my $volid = $drive->{file};
5355             next if !$volid;
5356
5357             $used->{$volid} = 1;
5358             if ($volid_hash->{$volid} &&
5359                 (my $path = $volid_hash->{$volid}->{path})) {
5360                 $usedpath->{$path} = 1;
5361             }
5362
5363             next if drive_is_cdrom($drive);
5364             next if !$volid_hash->{$volid};
5365
5366             $drive->{size} = $volid_hash->{$volid}->{size};
5367             my $new = print_drive($vmid, $drive);
5368             if ($new ne $conf->{$opt}) {
5369                 $changes = 1;
5370                 $conf->{$opt} = $new;
5371             }
5372         }
5373     }
5374
5375     # remove 'unusedX' entry if volume is used
5376     foreach my $opt (keys %$conf) {
5377         next if $opt !~ m/^unused\d+$/;
5378         my $volid = $conf->{$opt};
5379         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5380         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5381             $changes = 1;
5382             delete $conf->{$opt};
5383         }
5384     }
5385
5386     foreach my $volid (sort keys %$volid_hash) {
5387         next if $volid =~ m/vm-$vmid-state-/;
5388         next if $used->{$volid};
5389         my $path = $volid_hash->{$volid}->{path};
5390         next if !$path; # just to be sure
5391         next if $usedpath->{$path};
5392         $changes = 1;
5393         PVE::QemuConfig->add_unused_volume($conf, $volid);
5394         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5395     }
5396
5397     return $changes;
5398 }
5399
5400 sub rescan {
5401     my ($vmid, $nolock) = @_;
5402
5403     my $cfg = PVE::Storage::config();
5404
5405     my $volid_hash = scan_volids($cfg, $vmid);
5406
5407     my $updatefn =  sub {
5408         my ($vmid) = @_;
5409
5410         my $conf = PVE::QemuConfig->load_config($vmid);
5411
5412         PVE::QemuConfig->check_lock($conf);
5413
5414         my $vm_volids = {};
5415         foreach my $volid (keys %$volid_hash) {
5416             my $info = $volid_hash->{$volid};
5417             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5418         }
5419
5420         my $changes = update_disksize($vmid, $conf, $vm_volids);
5421
5422         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5423     };
5424
5425     if (defined($vmid)) {
5426         if ($nolock) {
5427             &$updatefn($vmid);
5428         } else {
5429             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5430         }
5431     } else {
5432         my $vmlist = config_list();
5433         foreach my $vmid (keys %$vmlist) {
5434             if ($nolock) {
5435                 &$updatefn($vmid);
5436             } else {
5437                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5438             }
5439         }
5440     }
5441 }
5442
5443 sub restore_vma_archive {
5444     my ($archive, $vmid, $user, $opts, $comp) = @_;
5445
5446     my $input = $archive eq '-' ? "<&STDIN" : undef;
5447     my $readfrom = $archive;
5448
5449     my $uncomp = '';
5450     if ($comp) {
5451         $readfrom = '-';
5452         my $qarchive = PVE::Tools::shellquote($archive);
5453         if ($comp eq 'gzip') {
5454             $uncomp = "zcat $qarchive|";
5455         } elsif ($comp eq 'lzop') {
5456             $uncomp = "lzop -d -c $qarchive|";
5457         } else {
5458             die "unknown compression method '$comp'\n";
5459         }
5460
5461     }
5462
5463     my $tmpdir = "/var/tmp/vzdumptmp$$";
5464     rmtree $tmpdir;
5465
5466     # disable interrupts (always do cleanups)
5467     local $SIG{INT} =
5468         local $SIG{TERM} =
5469         local $SIG{QUIT} =
5470         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5471
5472     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5473     POSIX::mkfifo($mapfifo, 0600);
5474     my $fifofh;
5475
5476     my $openfifo = sub {
5477         open($fifofh, '>', $mapfifo) || die $!;
5478     };
5479
5480     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5481
5482     my $oldtimeout;
5483     my $timeout = 5;
5484
5485     my $devinfo = {};
5486
5487     my $rpcenv = PVE::RPCEnvironment::get();
5488
5489     my $conffile = PVE::QemuConfig->config_file($vmid);
5490     my $tmpfn = "$conffile.$$.tmp";
5491
5492     # Note: $oldconf is undef if VM does not exists
5493     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5494     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5495
5496     my $print_devmap = sub {
5497         my $virtdev_hash = {};
5498
5499         my $cfgfn = "$tmpdir/qemu-server.conf";
5500
5501         # we can read the config - that is already extracted
5502         my $fh = IO::File->new($cfgfn, "r") ||
5503             "unable to read qemu-server.conf - $!\n";
5504
5505         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5506         if (-f $fwcfgfn) {
5507             my $pve_firewall_dir = '/etc/pve/firewall';
5508             mkdir $pve_firewall_dir; # make sure the dir exists
5509             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5510         }
5511
5512         while (defined(my $line = <$fh>)) {
5513             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5514                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5515                 die "archive does not contain data for drive '$virtdev'\n"
5516                     if !$devinfo->{$devname};
5517                 if (defined($opts->{storage})) {
5518                     $storeid = $opts->{storage} || 'local';
5519                 } elsif (!$storeid) {
5520                     $storeid = 'local';
5521                 }
5522                 $format = 'raw' if !$format;
5523                 $devinfo->{$devname}->{devname} = $devname;
5524                 $devinfo->{$devname}->{virtdev} = $virtdev;
5525                 $devinfo->{$devname}->{format} = $format;
5526                 $devinfo->{$devname}->{storeid} = $storeid;
5527
5528                 # check permission on storage
5529                 my $pool = $opts->{pool}; # todo: do we need that?
5530                 if ($user ne 'root@pam') {
5531                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5532                 }
5533
5534                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5535             }
5536         }
5537
5538         foreach my $devname (keys %$devinfo) {
5539             die "found no device mapping information for device '$devname'\n"
5540                 if !$devinfo->{$devname}->{virtdev};
5541         }
5542
5543         my $cfg = PVE::Storage::config();
5544
5545         # create empty/temp config
5546         if ($oldconf) {
5547             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5548             foreach_drive($oldconf, sub {
5549                 my ($ds, $drive) = @_;
5550
5551                 return if drive_is_cdrom($drive);
5552
5553                 my $volid = $drive->{file};
5554
5555                 return if !$volid || $volid =~ m|^/|;
5556
5557                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5558                 return if !$path || !$owner || ($owner != $vmid);
5559
5560                 # Note: only delete disk we want to restore
5561                 # other volumes will become unused
5562                 if ($virtdev_hash->{$ds}) {
5563                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5564                     if (my $err = $@) {
5565                         warn $err;
5566                     }
5567                 }
5568             });
5569
5570             # delete vmstate files
5571             # since after the restore we have no snapshots anymore
5572             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5573                 my $snap = $oldconf->{snapshots}->{$snapname};
5574                 if ($snap->{vmstate}) {
5575                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5576                     if (my $err = $@) {
5577                         warn $err;
5578                     }
5579                 }
5580             }
5581         }
5582
5583         my $map = {};
5584         foreach my $virtdev (sort keys %$virtdev_hash) {
5585             my $d = $virtdev_hash->{$virtdev};
5586             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5587             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5588
5589             # test if requested format is supported
5590             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5591             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5592             $d->{format} = $defFormat if !$supported;
5593
5594             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5595                                                   $d->{format}, undef, $alloc_size);
5596             print STDERR "new volume ID is '$volid'\n";
5597             $d->{volid} = $volid;
5598             my $path = PVE::Storage::path($cfg, $volid);
5599
5600             PVE::Storage::activate_volumes($cfg,[$volid]);
5601
5602             my $write_zeros = 1;
5603             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5604                 $write_zeros = 0;
5605             }
5606
5607             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5608
5609             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5610             $map->{$virtdev} = $volid;
5611         }
5612
5613         $fh->seek(0, 0) || die "seek failed - $!\n";
5614
5615         my $outfd = new IO::File ($tmpfn, "w") ||
5616             die "unable to write config for VM $vmid\n";
5617
5618         my $cookie = { netcount => 0 };
5619         while (defined(my $line = <$fh>)) {
5620             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5621         }
5622
5623         $fh->close();
5624         $outfd->close();
5625     };
5626
5627     eval {
5628         # enable interrupts
5629         local $SIG{INT} =
5630             local $SIG{TERM} =
5631             local $SIG{QUIT} =
5632             local $SIG{HUP} =
5633             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5634         local $SIG{ALRM} = sub { die "got timeout\n"; };
5635
5636         $oldtimeout = alarm($timeout);
5637
5638         my $parser = sub {
5639             my $line = shift;
5640
5641             print "$line\n";
5642
5643             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5644                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5645                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5646             } elsif ($line =~ m/^CTIME: /) {
5647                 # we correctly received the vma config, so we can disable
5648                 # the timeout now for disk allocation (set to 10 minutes, so
5649                 # that we always timeout if something goes wrong)
5650                 alarm(600);
5651                 &$print_devmap();
5652                 print $fifofh "done\n";
5653                 my $tmp = $oldtimeout || 0;
5654                 $oldtimeout = undef;
5655                 alarm($tmp);
5656                 close($fifofh);
5657             }
5658         };
5659
5660         print "restore vma archive: $cmd\n";
5661         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5662     };
5663     my $err = $@;
5664
5665     alarm($oldtimeout) if $oldtimeout;
5666
5667     my $vollist = [];
5668     foreach my $devname (keys %$devinfo) {
5669         my $volid = $devinfo->{$devname}->{volid};
5670         push @$vollist, $volid if $volid;
5671     }
5672
5673     my $cfg = PVE::Storage::config();
5674     PVE::Storage::deactivate_volumes($cfg, $vollist);
5675
5676     unlink $mapfifo;
5677
5678     if ($err) {
5679         rmtree $tmpdir;
5680         unlink $tmpfn;
5681
5682         foreach my $devname (keys %$devinfo) {
5683             my $volid = $devinfo->{$devname}->{volid};
5684             next if !$volid;
5685             eval {
5686                 if ($volid =~ m|^/|) {
5687                     unlink $volid || die 'unlink failed\n';
5688                 } else {
5689                     PVE::Storage::vdisk_free($cfg, $volid);
5690                 }
5691                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5692             };
5693             print STDERR "unable to cleanup '$volid' - $@" if $@;
5694         }
5695         die $err;
5696     }
5697
5698     rmtree $tmpdir;
5699
5700     rename($tmpfn, $conffile) ||
5701         die "unable to commit configuration file '$conffile'\n";
5702
5703     PVE::Cluster::cfs_update(); # make sure we read new file
5704
5705     eval { rescan($vmid, 1); };
5706     warn $@ if $@;
5707 }
5708
5709 sub restore_tar_archive {
5710     my ($archive, $vmid, $user, $opts) = @_;
5711
5712     if ($archive ne '-') {
5713         my $firstfile = tar_archive_read_firstfile($archive);
5714         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5715             if $firstfile ne 'qemu-server.conf';
5716     }
5717
5718     my $storecfg = PVE::Storage::config();
5719
5720     # destroy existing data - keep empty config
5721     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5722     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5723
5724     my $tocmd = "/usr/lib/qemu-server/qmextract";
5725
5726     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5727     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5728     $tocmd .= ' --prealloc' if $opts->{prealloc};
5729     $tocmd .= ' --info' if $opts->{info};
5730
5731     # tar option "xf" does not autodetect compression when read from STDIN,
5732     # so we pipe to zcat
5733     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5734         PVE::Tools::shellquote("--to-command=$tocmd");
5735
5736     my $tmpdir = "/var/tmp/vzdumptmp$$";
5737     mkpath $tmpdir;
5738
5739     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5740     local $ENV{VZDUMP_VMID} = $vmid;
5741     local $ENV{VZDUMP_USER} = $user;
5742
5743     my $conffile = PVE::QemuConfig->config_file($vmid);
5744     my $tmpfn = "$conffile.$$.tmp";
5745
5746     # disable interrupts (always do cleanups)
5747     local $SIG{INT} =
5748         local $SIG{TERM} =
5749         local $SIG{QUIT} =
5750         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5751
5752     eval {
5753         # enable interrupts
5754         local $SIG{INT} =
5755             local $SIG{TERM} =
5756             local $SIG{QUIT} =
5757             local $SIG{HUP} =
5758             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5759
5760         if ($archive eq '-') {
5761             print "extracting archive from STDIN\n";
5762             run_command($cmd, input => "<&STDIN");
5763         } else {
5764             print "extracting archive '$archive'\n";
5765             run_command($cmd);
5766         }
5767
5768         return if $opts->{info};
5769
5770         # read new mapping
5771         my $map = {};
5772         my $statfile = "$tmpdir/qmrestore.stat";
5773         if (my $fd = IO::File->new($statfile, "r")) {
5774             while (defined (my $line = <$fd>)) {
5775                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5776                     $map->{$1} = $2 if $1;
5777                 } else {
5778                     print STDERR "unable to parse line in statfile - $line\n";
5779                 }
5780             }
5781             $fd->close();
5782         }
5783
5784         my $confsrc = "$tmpdir/qemu-server.conf";
5785
5786         my $srcfd = new IO::File($confsrc, "r") ||
5787             die "unable to open file '$confsrc'\n";
5788
5789         my $outfd = new IO::File ($tmpfn, "w") ||
5790             die "unable to write config for VM $vmid\n";
5791
5792         my $cookie = { netcount => 0 };
5793         while (defined (my $line = <$srcfd>)) {
5794             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5795         }
5796
5797         $srcfd->close();
5798         $outfd->close();
5799     };
5800     my $err = $@;
5801
5802     if ($err) {
5803
5804         unlink $tmpfn;
5805
5806         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5807
5808         die $err;
5809     }
5810
5811     rmtree $tmpdir;
5812
5813     rename $tmpfn, $conffile ||
5814         die "unable to commit configuration file '$conffile'\n";
5815
5816     PVE::Cluster::cfs_update(); # make sure we read new file
5817
5818     eval { rescan($vmid, 1); };
5819     warn $@ if $@;
5820 };
5821
5822 sub foreach_storage_used_by_vm {
5823     my ($conf, $func) = @_;
5824
5825     my $sidhash = {};
5826
5827     foreach my $ds (keys %$conf) {
5828         next if !is_valid_drivename($ds);
5829
5830         my $drive = parse_drive($ds, $conf->{$ds});
5831         next if !$drive;
5832         next if drive_is_cdrom($drive);
5833
5834         my $volid = $drive->{file};
5835
5836         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5837         $sidhash->{$sid} = $sid if $sid;
5838     }
5839
5840     foreach my $sid (sort keys %$sidhash) {
5841         &$func($sid);
5842     }
5843 }
5844
5845 sub do_snapshots_with_qemu {
5846     my ($storecfg, $volid) = @_;
5847
5848     my $storage_name = PVE::Storage::parse_volume_id($volid);
5849
5850     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5851         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5852         return 1;
5853     }
5854
5855     if ($volid =~ m/\.(qcow2|qed)$/){
5856         return 1;
5857     }
5858
5859     return undef;
5860 }
5861
5862 sub qga_check_running {
5863     my ($vmid) = @_;
5864
5865     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5866     if ($@) {
5867         warn "Qemu Guest Agent are not running - $@";
5868         return 0;
5869     }
5870     return 1;
5871 }
5872
5873 sub template_create {
5874     my ($vmid, $conf, $disk) = @_;
5875
5876     my $storecfg = PVE::Storage::config();
5877
5878     foreach_drive($conf, sub {
5879         my ($ds, $drive) = @_;
5880
5881         return if drive_is_cdrom($drive);
5882         return if $disk && $ds ne $disk;
5883
5884         my $volid = $drive->{file};
5885         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5886
5887         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5888         $drive->{file} = $voliddst;
5889         $conf->{$ds} = print_drive($vmid, $drive);
5890         PVE::QemuConfig->write_config($vmid, $conf);
5891     });
5892 }
5893
5894 sub qemu_img_convert {
5895     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5896
5897     my $storecfg = PVE::Storage::config();
5898     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5899     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5900
5901     if ($src_storeid && $dst_storeid) {
5902
5903         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5904
5905         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5906         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5907
5908         my $src_format = qemu_img_format($src_scfg, $src_volname);
5909         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5910
5911         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5912         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5913
5914         my $cmd = [];
5915         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5916         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5917         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5918         if ($is_zero_initialized) {
5919             push @$cmd, "zeroinit:$dst_path";
5920         } else {
5921             push @$cmd, $dst_path;
5922         }
5923
5924         my $parser = sub {
5925             my $line = shift;
5926             if($line =~ m/\((\S+)\/100\%\)/){
5927                 my $percent = $1;
5928                 my $transferred = int($size * $percent / 100);
5929                 my $remaining = $size - $transferred;
5930
5931                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5932             }
5933
5934         };
5935
5936         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5937         my $err = $@;
5938         die "copy failed: $err" if $err;
5939     }
5940 }
5941
5942 sub qemu_img_format {
5943     my ($scfg, $volname) = @_;
5944
5945     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5946         return $1;
5947     } else {
5948         return "raw";
5949     }
5950 }
5951
5952 sub qemu_drive_mirror {
5953     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5954
5955     $jobs = {} if !$jobs;
5956
5957     my $qemu_target;
5958     my $format;
5959     $jobs->{"drive-$drive"} = {};
5960
5961     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5962         my $server = $1;
5963         my $port = $2;
5964         my $exportname = $3;
5965
5966         $format = "nbd";
5967         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5968         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5969         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5970
5971         my $pid = fork();
5972         if (!defined($pid)) {
5973             die "forking socat tunnel failed\n";
5974         } elsif ($pid == 0) {
5975             exec(@$cmd);
5976             warn "exec failed: $!\n";
5977             POSIX::_exit(-1);
5978         }
5979         $jobs->{"drive-$drive"}->{pid} = $pid;
5980
5981         my $timeout = 0;
5982         while (!-S $unixsocket) {
5983             die "nbd connection helper timed out\n"
5984                 if $timeout++ > 5;
5985             sleep 1;
5986         }
5987     } else {
5988         my $storecfg = PVE::Storage::config();
5989         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5990
5991         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5992
5993         $format = qemu_img_format($dst_scfg, $dst_volname);
5994
5995         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5996
5997         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5998     }
5999
6000     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6001     $opts->{format} = $format if $format;
6002
6003     print "drive mirror is starting for drive-$drive\n";
6004
6005     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6006
6007     if (my $err = $@) {
6008         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6009         die "mirroring error: $err";
6010     }
6011
6012     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6013 }
6014
6015 sub qemu_drive_mirror_monitor {
6016     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6017
6018     eval {
6019         my $err_complete = 0;
6020
6021         while (1) {
6022             die "storage migration timed out\n" if $err_complete > 300;
6023
6024             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6025
6026             my $running_mirror_jobs = {};
6027             foreach my $stat (@$stats) {
6028                 next if $stat->{type} ne 'mirror';
6029                 $running_mirror_jobs->{$stat->{device}} = $stat;
6030             }
6031
6032             my $readycounter = 0;
6033
6034             foreach my $job (keys %$jobs) {
6035
6036                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6037                     print "$job : finished\n";
6038                     delete $jobs->{$job};
6039                     next;
6040                 }
6041
6042                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6043
6044                 my $busy = $running_mirror_jobs->{$job}->{busy};
6045                 my $ready = $running_mirror_jobs->{$job}->{ready};
6046                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6047                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6048                     my $remaining = $total - $transferred;
6049                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6050
6051                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6052                 }
6053
6054                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6055             }
6056
6057             last if scalar(keys %$jobs) == 0;
6058
6059             if ($readycounter == scalar(keys %$jobs)) {
6060                 print "all mirroring jobs are ready \n";
6061                 last if $skipcomplete; #do the complete later
6062
6063                 if ($vmiddst && $vmiddst != $vmid) {
6064                     if ($qga) {
6065                         print "freeze filesystem\n";
6066                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6067                     } else {
6068                         print "suspend vm\n";
6069                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6070                     }
6071
6072                     # if we clone a disk for a new target vm, we don't switch the disk
6073                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6074
6075                     if ($qga) {
6076                         print "unfreeze filesystem\n";
6077                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6078                     } else {
6079                         print "resume vm\n";
6080                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6081                     }
6082
6083                     last;
6084                 } else {
6085
6086                     foreach my $job (keys %$jobs) {
6087                         # try to switch the disk if source and destination are on the same guest
6088                         print "$job: Completing block job...\n";
6089
6090                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6091                         if ($@ =~ m/cannot be completed/) {
6092                             print "$job: Block job cannot be completed, try again.\n";
6093                             $err_complete++;
6094                         }else {
6095                             print "$job: Completed successfully.\n";
6096                             $jobs->{$job}->{complete} = 1;
6097                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6098                         }
6099                     }
6100                 }
6101             }
6102             sleep 1;
6103         }
6104     };
6105     my $err = $@;
6106
6107     if ($err) {
6108         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6109         die "mirroring error: $err";
6110     }
6111
6112 }
6113
6114 sub qemu_blockjobs_cancel {
6115     my ($vmid, $jobs) = @_;
6116
6117     foreach my $job (keys %$jobs) {
6118         print "$job: Cancelling block job\n";
6119         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6120         $jobs->{$job}->{cancel} = 1;
6121     }
6122
6123     while (1) {
6124         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6125
6126         my $running_jobs = {};
6127         foreach my $stat (@$stats) {
6128             $running_jobs->{$stat->{device}} = $stat;
6129         }
6130
6131         foreach my $job (keys %$jobs) {
6132
6133             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6134                 print "$job: Done.\n";
6135                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6136                 delete $jobs->{$job};
6137             }
6138         }
6139
6140         last if scalar(keys %$jobs) == 0;
6141
6142         sleep 1;
6143     }
6144 }
6145
6146 sub qemu_blockjobs_finish_tunnel {
6147    my ($vmid, $job, $cpid) = @_;
6148
6149    return if !$cpid;
6150
6151    for (my $i = 1; $i < 20; $i++) {
6152         my $waitpid = waitpid($cpid, WNOHANG);
6153         last if (defined($waitpid) && ($waitpid == $cpid));
6154
6155         if ($i == 10) {
6156             kill(15, $cpid);
6157         } elsif ($i >= 15) {
6158             kill(9, $cpid);
6159         }
6160         sleep (1);
6161     }
6162     unlink "/run/qemu-server/$vmid.mirror-$job";
6163 }
6164
6165 sub clone_disk {
6166     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6167         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6168
6169     my $newvolid;
6170
6171     if (!$full) {
6172         print "create linked clone of drive $drivename ($drive->{file})\n";
6173         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6174         push @$newvollist, $newvolid;
6175     } else {
6176
6177         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6178         $storeid = $storage if $storage;
6179
6180         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6181         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6182
6183         print "create full clone of drive $drivename ($drive->{file})\n";
6184         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6185         push @$newvollist, $newvolid;
6186
6187         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6188
6189         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6190         if (!$running || $snapname) {
6191             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6192         } else {
6193
6194             my $kvmver = get_running_qemu_version ($vmid);
6195             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6196                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6197                     if $drive->{iothread};
6198             }
6199
6200             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6201         }
6202     }
6203
6204     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6205
6206     my $disk = $drive;
6207     $disk->{format} = undef;
6208     $disk->{file} = $newvolid;
6209     $disk->{size} = $size;
6210
6211     return $disk;
6212 }
6213
6214 # this only works if VM is running
6215 sub get_current_qemu_machine {
6216     my ($vmid) = @_;
6217
6218     my $cmd = { execute => 'query-machines', arguments => {} };
6219     my $res = vm_qmp_command($vmid, $cmd);
6220
6221     my ($current, $default);
6222     foreach my $e (@$res) {
6223         $default = $e->{name} if $e->{'is-default'};
6224         $current = $e->{name} if $e->{'is-current'};
6225     }
6226
6227     # fallback to the default machine if current is not supported by qemu
6228     return $current || $default || 'pc';
6229 }
6230
6231 sub get_running_qemu_version {
6232     my ($vmid) = @_;
6233     my $cmd = { execute => 'query-version', arguments => {} };
6234     my $res = vm_qmp_command($vmid, $cmd);
6235     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6236 }
6237
6238 sub qemu_machine_feature_enabled {
6239     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6240
6241     my $current_major;
6242     my $current_minor;
6243
6244     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6245
6246         $current_major = $3;
6247         $current_minor = $4;
6248
6249     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6250
6251         $current_major = $1;
6252         $current_minor = $2;
6253     }
6254
6255     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6256
6257
6258 }
6259
6260 sub qemu_machine_pxe {
6261     my ($vmid, $conf, $machine) = @_;
6262
6263     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6264
6265     foreach my $opt (keys %$conf) {
6266         next if $opt !~ m/^net(\d+)$/;
6267         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6268         next if !$net;
6269         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6270         return $machine.".pxe" if $romfile =~ m/pxe/;
6271         last;
6272     }
6273
6274     return $machine;
6275 }
6276
6277 sub qemu_use_old_bios_files {
6278     my ($machine_type) = @_;
6279
6280     return if !$machine_type;
6281
6282     my $use_old_bios_files = undef;
6283
6284     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6285         $machine_type = $1;
6286         $use_old_bios_files = 1;
6287     } else {
6288         my $kvmver = kvm_user_version();
6289         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6290         # load new efi bios files on migration. So this hack is required to allow
6291         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6292         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6293         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6294     }
6295
6296     return ($use_old_bios_files, $machine_type);
6297 }
6298
6299 sub create_efidisk {
6300     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6301
6302     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6303
6304     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6305     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6306     PVE::Storage::activate_volumes($storecfg, [$volid]);
6307
6308     my $path = PVE::Storage::path($storecfg, $volid);
6309     eval {
6310         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6311     };
6312     die "Copying EFI vars image failed: $@" if $@;
6313
6314     return ($volid, $vars_size);
6315 }
6316
6317 sub lspci {
6318
6319     my $devices = {};
6320
6321     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6322             my (undef, $id, $function) = @_;
6323             my $res = { id => $id, function => $function};
6324             push @{$devices->{$id}}, $res;
6325     });
6326
6327     # Entries should be sorted by functions.
6328     foreach my $id (keys %$devices) {
6329         my $dev = $devices->{$id};
6330         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6331     }
6332
6333     return $devices;
6334 }
6335
6336 sub vm_iothreads_list {
6337     my ($vmid) = @_;
6338
6339     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6340
6341     my $iothreads = {};
6342     foreach my $iothread (@$res) {
6343         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6344     }
6345
6346     return $iothreads;
6347 }
6348
6349 sub scsihw_infos {
6350     my ($conf, $drive) = @_;
6351
6352     my $maxdev = 0;
6353
6354     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6355         $maxdev = 7;
6356     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6357         $maxdev = 1;
6358     } else {
6359         $maxdev = 256;
6360     }
6361
6362     my $controller = int($drive->{index} / $maxdev);
6363     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6364
6365     return ($maxdev, $controller, $controller_prefix);
6366 }
6367
6368 sub add_hyperv_enlightenments {
6369     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6370
6371     return if $winversion < 6;
6372     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6373
6374     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6375
6376     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6377         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6378         push @$cpuFlags , 'hv_vapic';
6379         push @$cpuFlags , 'hv_time';
6380     } else {
6381         push @$cpuFlags , 'hv_spinlocks=0xffff';
6382     }
6383
6384     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6385         push @$cpuFlags , 'hv_reset';
6386         push @$cpuFlags , 'hv_vpindex';
6387         push @$cpuFlags , 'hv_runtime';
6388     }
6389
6390     if ($winversion >= 7) {
6391         push @$cpuFlags , 'hv_relaxed';
6392     }
6393 }
6394
6395 sub windows_version {
6396     my ($ostype) = @_;
6397
6398     return 0 if !$ostype;
6399
6400     my $winversion = 0;
6401
6402     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6403         $winversion = 5;
6404     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6405         $winversion = 6;
6406     } elsif ($ostype =~ m/^win(\d+)$/) {
6407         $winversion = $1;
6408     }
6409
6410     return $winversion;
6411 }
6412
6413 sub resolve_dst_disk_format {
6414         my ($storecfg, $storeid, $src_volname, $format) = @_;
6415         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6416
6417         if (!$format) {
6418             # if no target format is specified, use the source disk format as hint
6419             if ($src_volname) {
6420                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6421                 $format = qemu_img_format($scfg, $src_volname);
6422             } else {
6423                 return $defFormat;
6424             }
6425         }
6426
6427         # test if requested format is supported - else use default
6428         my $supported = grep { $_ eq $format } @$validFormats;
6429         $format = $defFormat if !$supported;
6430         return $format;
6431 }
6432
6433 # bash completion helper
6434
6435 sub complete_backup_archives {
6436     my ($cmdname, $pname, $cvalue) = @_;
6437
6438     my $cfg = PVE::Storage::config();
6439
6440     my $storeid;
6441
6442     if ($cvalue =~ m/^([^:]+):/) {
6443         $storeid = $1;
6444     }
6445
6446     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6447
6448     my $res = [];
6449     foreach my $id (keys %$data) {
6450         foreach my $item (@{$data->{$id}}) {
6451             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6452             push @$res, $item->{volid} if defined($item->{volid});
6453         }
6454     }
6455
6456     return $res;
6457 }
6458
6459 my $complete_vmid_full = sub {
6460     my ($running) = @_;
6461
6462     my $idlist = vmstatus();
6463
6464     my $res = [];
6465
6466     foreach my $id (keys %$idlist) {
6467         my $d = $idlist->{$id};
6468         if (defined($running)) {
6469             next if $d->{template};
6470             next if $running && $d->{status} ne 'running';
6471             next if !$running && $d->{status} eq 'running';
6472         }
6473         push @$res, $id;
6474
6475     }
6476     return $res;
6477 };
6478
6479 sub complete_vmid {
6480     return &$complete_vmid_full();
6481 }
6482
6483 sub complete_vmid_stopped {
6484     return &$complete_vmid_full(0);
6485 }
6486
6487 sub complete_vmid_running {
6488     return &$complete_vmid_full(1);
6489 }
6490
6491 sub complete_storage {
6492
6493     my $cfg = PVE::Storage::config();
6494     my $ids = $cfg->{ids};
6495
6496     my $res = [];
6497     foreach my $sid (keys %$ids) {
6498         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6499         next if !$ids->{$sid}->{content}->{images};
6500         push @$res, $sid;
6501     }
6502
6503     return $res;
6504 }
6505
6506 sub nbd_stop {
6507     my ($vmid) = @_;
6508
6509     vm_mon_cmd($vmid, 'nbd-server-stop');
6510 }
6511
6512 1;