PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('skiplock', {
  60     description => "Ignore locks - only root is allowed to use this option.",
  61     type => 'boolean',
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  66     description => "Some command save/restore state from this location.",
  67     type => 'string',
  68     maxLength => 128,
  69     optional => 1,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  73     description => "The name of the snapshot.",
  74     type => 'string', format => 'pve-configid',
  75     maxLength => 40,
  76 });
  77
  78 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  79     type => 'string',
  80     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  81     description => "The drive's backing file's data format.",
  82     optional => 1,
  83 });
  84
  85 #no warnings 'redefine';
  86
  87 sub cgroups_write {
  88    my ($controller, $vmid, $option, $value) = @_;
  89
  90    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  91    PVE::ProcFSTools::write_proc_entry($path, $value);
  92
  93 }
  94
  95 my $nodename = PVE::INotify::nodename();
  96
  97 mkdir "/etc/pve/nodes/$nodename";
  98 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  99 mkdir $confdir;
 100
 101 my $var_run_tmpdir = "/var/run/qemu-server";
 102 mkdir $var_run_tmpdir;
 103
 104 my $lock_dir = "/var/lock/qemu-server";
 105 mkdir $lock_dir;
 106
 107 my $pcisysfs = "/sys/bus/pci";
 108
 109 my $cpu_vendor_list = {
 110     # Intel CPUs
 111     486 => 'GenuineIntel',
 112     pentium => 'GenuineIntel',
 113     pentium2  => 'GenuineIntel',
 114     pentium3  => 'GenuineIntel',
 115     coreduo => 'GenuineIntel',
 116     core2duo => 'GenuineIntel',
 117     Conroe  => 'GenuineIntel',
 118     Penryn  => 'GenuineIntel',
 119     Nehalem  => 'GenuineIntel',
 120     Westmere => 'GenuineIntel',
 121     SandyBridge => 'GenuineIntel',
 122     IvyBridge => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-noTSX' => 'GenuineIntel',
 125     Broadwell => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127
 128     # AMD CPUs
 129     athlon => 'AuthenticAMD',
 130     phenom  => 'AuthenticAMD',
 131     Opteron_G1  => 'AuthenticAMD',
 132     Opteron_G2  => 'AuthenticAMD',
 133     Opteron_G3  => 'AuthenticAMD',
 134     Opteron_G4  => 'AuthenticAMD',
 135     Opteron_G5  => 'AuthenticAMD',
 136
 137     # generic types, use vendor from host node
 138     host => 'default',
 139     kvm32 => 'default',
 140     kvm64 => 'default',
 141     qemu32 => 'default',
 142     qemu64 => 'default',
 143 };
 144
 145 my $cpu_fmt = {
 146     cputype => {
 147         description => "Emulated CPU type.",
 148         type => 'string',
 149         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 150         format_description => 'cputype',
 151         default => 'kvm64',
 152         default_key => 1,
 153     },
 154     hidden => {
 155         description => "Do not identify as a KVM virtual machine.",
 156         type => 'boolean',
 157         optional => 1,
 158         default => 0
 159     },
 160 };
 161
 162 my $watchdog_fmt = {
 163     model => {
 164         default_key => 1,
 165         type => 'string',
 166         enum => [qw(i6300esb ib700)],
 167         description => "Watchdog type to emulate.",
 168         default => 'i6300esb',
 169         optional => 1,
 170     },
 171     action => {
 172         type => 'string',
 173         enum => [qw(reset shutdown poweroff pause debug none)],
 174         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 175         optional => 1,
 176     },
 177 };
 178 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 179
 180 my $confdesc = {
 181     onboot => {
 182         optional => 1,
 183         type => 'boolean',
 184         description => "Specifies whether a VM will be started during system bootup.",
 185         default => 0,
 186     },
 187     autostart => {
 188         optional => 1,
 189         type => 'boolean',
 190         description => "Automatic restart after crash (currently ignored).",
 191         default => 0,
 192     },
 193     hotplug => {
 194         optional => 1,
 195         type => 'string', format => 'pve-hotplug-features',
 196         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 197         default => 'network,disk,usb',
 198     },
 199     reboot => {
 200         optional => 1,
 201         type => 'boolean',
 202         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 203         default => 1,
 204     },
 205     lock => {
 206         optional => 1,
 207         type => 'string',
 208         description => "Lock/unlock the VM.",
 209         enum => [qw(migrate backup snapshot rollback)],
 210     },
 211     cpulimit => {
 212         optional => 1,
 213         type => 'number',
 214         description => "Limit of CPU usage.",
 215         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 216         minimum => 0,
 217         maximum => 128,
 218         default => 0,
 219     },
 220     cpuunits => {
 221         optional => 1,
 222         type => 'integer',
 223         description => "CPU weight for a VM.",
 224         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 225         minimum => 0,
 226         maximum => 500000,
 227         default => 1024,
 228     },
 229     memory => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 233         minimum => 16,
 234         default => 512,
 235     },
 236     balloon => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 240         minimum => 0,
 241     },
 242     shares => {
 243         optional => 1,
 244         type => 'integer',
 245         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 246         minimum => 0,
 247         maximum => 50000,
 248         default => 1000,
 249     },
 250     keyboard => {
 251         optional => 1,
 252         type => 'string',
 253         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 254         enum => PVE::Tools::kvmkeymaplist(),
 255         default => 'en-us',
 256     },
 257     name => {
 258         optional => 1,
 259         type => 'string', format => 'dns-name',
 260         description => "Set a name for the VM. Only used on the configuration web interface.",
 261     },
 262     scsihw => {
 263         optional => 1,
 264         type => 'string',
 265         description => "SCSI controller model",
 266         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 267         default => 'lsi',
 268     },
 269     description => {
 270         optional => 1,
 271         type => 'string',
 272         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 273     },
 274     ostype => {
 275         optional => 1,
 276         type => 'string',
 277         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 278         description => "Specify guest operating system.",
 279         verbose_description => <<EODESC,
 280 Specify guest operating system. This is used to enable special
 281 optimization/features for specific operating systems:
 282
 283 [horizontal]
 284 other;; unspecified OS
 285 wxp;; Microsoft Windows XP
 286 w2k;; Microsoft Windows 2000
 287 w2k3;; Microsoft Windows 2003
 288 w2k8;; Microsoft Windows 2008
 289 wvista;; Microsoft Windows Vista
 290 win7;; Microsoft Windows 7
 291 win8;; Microsoft Windows 8/2012
 292 l24;; Linux 2.4 Kernel
 293 l26;; Linux 2.6/3.X Kernel
 294 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 295 EODESC
 296     },
 297     boot => {
 298         optional => 1,
 299         type => 'string',
 300         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 301         pattern => '[acdn]{1,4}',
 302         default => 'cdn',
 303     },
 304     bootdisk => {
 305         optional => 1,
 306         type => 'string', format => 'pve-qm-bootdisk',
 307         description => "Enable booting from specified disk.",
 308         pattern => '(ide|sata|scsi|virtio)\d+',
 309     },
 310     smp => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "The number of CPUs. Please use option -sockets instead.",
 314         minimum => 1,
 315         default => 1,
 316     },
 317     sockets => {
 318         optional => 1,
 319         type => 'integer',
 320         description => "The number of CPU sockets.",
 321         minimum => 1,
 322         default => 1,
 323     },
 324     cores => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "The number of cores per socket.",
 328         minimum => 1,
 329         default => 1,
 330     },
 331     numa => {
 332         optional => 1,
 333         type => 'boolean',
 334         description => "Enable/disable NUMA.",
 335         default => 0,
 336     },
 337     hugepages => {
 338         optional => 1,
 339         type => 'string',
 340         description => "Enable/disable hugepages memory.",
 341         enum => [qw(any 2 1024)],
 342     },
 343     vcpus => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Number of hotplugged vcpus.",
 347         minimum => 1,
 348         default => 0,
 349     },
 350     acpi => {
 351         optional => 1,
 352         type => 'boolean',
 353         description => "Enable/disable ACPI.",
 354         default => 1,
 355     },
 356     agent => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable Qemu GuestAgent.",
 360         default => 0,
 361     },
 362     kvm => {
 363         optional => 1,
 364         type => 'boolean',
 365         description => "Enable/disable KVM hardware virtualization.",
 366         default => 1,
 367     },
 368     tdf => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Enable/disable time drift fix.",
 372         default => 0,
 373     },
 374     localtime => {
 375         optional => 1,
 376         type => 'boolean',
 377         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 378     },
 379     freeze => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 383     },
 384     vga => {
 385         optional => 1,
 386         type => 'string',
 387         description => "Select the VGA type.",
 388         verbose_description => "Select the VGA type. If you want to use high resolution" .
 389             " modes (>= 1280x1024x16) then you should use the options " .
 390             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 391             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 392             "display sever. For win* OS you can select how many independent " .
 393             "displays you want, Linux guests can add displays them self. " .
 394             "You can also run without any graphic card, using a serial device" .
 395             " as terminal.",
 396         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 397     },
 398     watchdog => {
 399         optional => 1,
 400         type => 'string', format => 'pve-qm-watchdog',
 401         description => "Create a virtual hardware watchdog device.",
 402         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 403             " (by a guest action), the watchdog must be periodically polled " .
 404             "by an agent inside the guest or else the watchdog will reset " .
 405             "the guest (or execute the respective action specified)",
 406     },
 407     startdate => {
 408         optional => 1,
 409         type => 'string',
 410         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 411         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 412         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 413         default => 'now',
 414     },
 415     startup =>  get_standard_option('pve-startup-order'),
 416     template => {
 417         optional => 1,
 418         type => 'boolean',
 419         description => "Enable/disable Template.",
 420         default => 0,
 421     },
 422     args => {
 423         optional => 1,
 424         type => 'string',
 425         description => "Arbitrary arguments passed to kvm.",
 426         verbose_description => <<EODESCR,
 427 Arbitrary arguments passed to kvm, for example:
 428
 429 args: -no-reboot -no-hpet
 430
 431 NOTE: this option is for experts only.
 432 EODESCR
 433     },
 434     tablet => {
 435         optional => 1,
 436         type => 'boolean',
 437         default => 1,
 438         description => "Enable/disable the USB tablet device.",
 439         verbose_description => "Enable/disable the USB tablet device. This device is " .
 440             "usually needed to allow absolute mouse positioning with VNC. " .
 441             "Else the mouse runs out of sync with normal VNC clients. " .
 442             "If you're running lots of console-only guests on one host, " .
 443             "you may consider disabling this to save some context switches. " .
 444             "This is turned off by default if you use spice (-vga=qxl).",
 445     },
 446     migrate_speed => {
 447         optional => 1,
 448         type => 'integer',
 449         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 450         minimum => 0,
 451         default => 0,
 452     },
 453     migrate_downtime => {
 454         optional => 1,
 455         type => 'number',
 456         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 457         minimum => 0,
 458         default => 0.1,
 459     },
 460     cdrom => {
 461         optional => 1,
 462         type => 'string', format => 'pve-qm-ide',
 463         typetext => 'volume',
 464         description => "This is an alias for option -ide2",
 465     },
 466     cpu => {
 467         optional => 1,
 468         description => "Emulated CPU type.",
 469         type => 'string',
 470         format => $cpu_fmt,
 471     },
 472     parent => get_standard_option('pve-snapshot-name', {
 473         optional => 1,
 474         description => "Parent snapshot name. This is used internally, and should not be modified.",
 475     }),
 476     snaptime => {
 477         optional => 1,
 478         description => "Timestamp for snapshots.",
 479         type => 'integer',
 480         minimum => 0,
 481     },
 482     vmstate => {
 483         optional => 1,
 484         type => 'string', format => 'pve-volume-id',
 485         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 486     },
 487     machine => {
 488         description => "Specific the Qemu machine type.",
 489         type => 'string',
 490         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 491         maxLength => 40,
 492         optional => 1,
 493     },
 494     smbios1 => {
 495         description => "Specify SMBIOS type 1 fields.",
 496         type => 'string', format => 'pve-qm-smbios1',
 497         maxLength => 256,
 498         optional => 1,
 499     },
 500     protection => {
 501         optional => 1,
 502         type => 'boolean',
 503         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 504         default => 0,
 505     },
 506     bios => {
 507         optional => 1,
 508         type => 'string',
 509         enum => [ qw(seabios ovmf) ],
 510         description => "Select BIOS implementation.",
 511         default => 'seabios',
 512     },
 513 };
 514
 515 # what about other qemu settings ?
 516 #cpu => 'string',
 517 #machine => 'string',
 518 #fda => 'file',
 519 #fdb => 'file',
 520 #mtdblock => 'file',
 521 #sd => 'file',
 522 #pflash => 'file',
 523 #snapshot => 'bool',
 524 #bootp => 'file',
 525 ##tftp => 'dir',
 526 ##smb => 'dir',
 527 #kernel => 'file',
 528 #append => 'string',
 529 #initrd => 'file',
 530 ##soundhw => 'string',
 531
 532 while (my ($k, $v) = each %$confdesc) {
 533     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 534 }
 535
 536 my $MAX_IDE_DISKS = 4;
 537 my $MAX_SCSI_DISKS = 14;
 538 my $MAX_VIRTIO_DISKS = 16;
 539 my $MAX_SATA_DISKS = 6;
 540 my $MAX_USB_DEVICES = 5;
 541 my $MAX_NETS = 32;
 542 my $MAX_UNUSED_DISKS = 8;
 543 my $MAX_HOSTPCI_DEVICES = 4;
 544 my $MAX_SERIAL_PORTS = 4;
 545 my $MAX_PARALLEL_PORTS = 3;
 546 my $MAX_NUMA = 8;
 547
 548 my $numa_fmt = {
 549     cpus => {
 550         type => "string",
 551         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 552         description => "CPUs accessing this NUMA node.",
 553         format_description => "id[-id];...",
 554     },
 555     memory => {
 556         type => "number",
 557         description => "Amount of memory this NUMA node provides.",
 558         optional => 1,
 559     },
 560     hostnodes => {
 561         type => "string",
 562         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 563         description => "Host NUMA nodes to use.",
 564         format_description => "id[-id];...",
 565         optional => 1,
 566     },
 567     policy => {
 568         type => 'string',
 569         enum => [qw(preferred bind interleave)],
 570         description => "NUMA allocation policy.",
 571         optional => 1,
 572     },
 573 };
 574 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 575 my $numadesc = {
 576     optional => 1,
 577     type => 'string', format => $numa_fmt,
 578     description => "NUMA topology.",
 579 };
 580 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 581
 582 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 583     $confdesc->{"numa$i"} = $numadesc;
 584 }
 585
 586 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 587                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 588                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 589 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 590
 591 my $net_fmt_bridge_descr = <<__EOD__;
 592 Bridge to attach the network device to. The Proxmox VE standard bridge
 593 is called 'vmbr0'.
 594
 595 If you do not specify a bridge, we create a kvm user (NATed) network
 596 device, which provides DHCP and DNS services. The following addresses
 597 are used:
 598
 599  10.0.2.2   Gateway
 600  10.0.2.3   DNS Server
 601  10.0.2.4   SMB Server
 602
 603 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 604 __EOD__
 605
 606 my $net_fmt = {
 607     macaddr => {
 608         type => 'string',
 609         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 610         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 611         format_description => "XX:XX:XX:XX:XX:XX",
 612         optional => 1,
 613     },
 614     model => {
 615         type => 'string',
 616         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 617         format_description => 'model',
 618         enum => $nic_model_list,
 619         default_key => 1,
 620     },
 621     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 622     bridge => {
 623         type => 'string',
 624         description => $net_fmt_bridge_descr,
 625         format_description => 'bridge',
 626         optional => 1,
 627     },
 628     queues => {
 629         type => 'integer',
 630         minimum => 0, maximum => 16,
 631         description => 'Number of packet queues to be used on the device.',
 632         optional => 1,
 633     },
 634     rate => {
 635         type => 'number',
 636         minimum => 0,
 637         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 638         optional => 1,
 639     },
 640     tag => {
 641         type => 'integer',
 642         minimum => 1, maximum => 4094,
 643         description => 'VLAN tag to apply to packets on this interface.',
 644         optional => 1,
 645     },
 646     trunks => {
 647         type => 'string',
 648         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 649         description => 'VLAN trunks to pass through this interface.',
 650         format_description => 'vlanid[;vlanid...]',
 651         optional => 1,
 652     },
 653     firewall => {
 654         type => 'boolean',
 655         description => 'Whether this interface should be protected by the firewall.',
 656         optional => 1,
 657     },
 658     link_down => {
 659         type => 'boolean',
 660         description => 'Whether this interface should be disconnected (like pulling the plug).',
 661         optional => 1,
 662     },
 663 };
 664
 665 my $netdesc = {
 666     optional => 1,
 667     type => 'string', format => $net_fmt,
 668     description => "Specify network devices.",
 669 };
 670
 671 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 672
 673 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 674     $confdesc->{"net$i"} = $netdesc;
 675 }
 676
 677 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 678 sub verify_volume_id_or_qm_path {
 679     my ($volid, $noerr) = @_;
 680
 681     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 682         return $volid;
 683     }
 684
 685     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 686     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 687     if ($@) {
 688         return undef if $noerr;
 689         die $@;
 690     }
 691     return $volid;
 692 }
 693
 694 my $drivename_hash;
 695
 696 my %drivedesc_base = (
 697     volume => { alias => 'file' },
 698     file => {
 699         type => 'string',
 700         format => 'pve-volume-id-or-qm-path',
 701         default_key => 1,
 702         format_description => 'volume',
 703         description => "The drive's backing volume.",
 704     },
 705     media => {
 706         type => 'string',
 707         enum => [qw(cdrom disk)],
 708         description => "The drive's media type.",
 709         default => 'disk',
 710         optional => 1
 711     },
 712     cyls => {
 713         type => 'integer',
 714         description => "Force the drive's physical geometry to have a specific cylinder count.",
 715         optional => 1
 716     },
 717     heads => {
 718         type => 'integer',
 719         description => "Force the drive's physical geometry to have a specific head count.",
 720         optional => 1
 721     },
 722     secs => {
 723         type => 'integer',
 724         description => "Force the drive's physical geometry to have a specific sector count.",
 725         optional => 1
 726     },
 727     trans => {
 728         type => 'string',
 729         enum => [qw(none lba auto)],
 730         description => "Force disk geometry bios translation mode.",
 731         optional => 1,
 732     },
 733     snapshot => {
 734         type => 'boolean',
 735         description => "Whether the drive should be included when making snapshots.",
 736         optional => 1,
 737     },
 738     cache => {
 739         type => 'string',
 740         enum => [qw(none writethrough writeback unsafe directsync)],
 741         description => "The drive's cache mode",
 742         optional => 1,
 743     },
 744     format => get_standard_option('pve-qm-image-format'),
 745     size => {
 746         type => 'string',
 747         format => 'disk-size',
 748         format_description => 'DiskSize',
 749         description => "Disk size. This is purely informational and has no effect.",
 750         optional => 1,
 751     },
 752     backup => {
 753         type => 'boolean',
 754         description => "Whether the drive should be included when making backups.",
 755         optional => 1,
 756     },
 757     werror => {
 758         type => 'string',
 759         enum => [qw(enospc ignore report stop)],
 760         description => 'Write error action.',
 761         optional => 1,
 762     },
 763     aio => {
 764         type => 'string',
 765         enum => [qw(native threads)],
 766         description => 'AIO type to use.',
 767         optional => 1,
 768     },
 769     discard => {
 770         type => 'string',
 771         enum => [qw(ignore on)],
 772         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 773         optional => 1,
 774     },
 775     detect_zeroes => {
 776         type => 'boolean',
 777         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 778         optional => 1,
 779     },
 780     serial => {
 781         type => 'string',
 782         format => 'urlencoded',
 783         format_description => 'serial',
 784         maxLength => 20*3, # *3 since it's %xx url enoded
 785         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 786         optional => 1,
 787     }
 788 );
 789
 790 my %rerror_fmt = (
 791     rerror => {
 792         type => 'string',
 793         enum => [qw(ignore report stop)],
 794         description => 'Read error action.',
 795         optional => 1,
 796     },
 797 );
 798
 799 my %iothread_fmt = ( iothread => {
 800         type => 'boolean',
 801         description => "Whether to use iothreads for this drive",
 802         optional => 1,
 803 });
 804
 805 my %model_fmt = (
 806     model => {
 807         type => 'string',
 808         format => 'urlencoded',
 809         format_description => 'model',
 810         maxLength => 40*3, # *3 since it's %xx url enoded
 811         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 812         optional => 1,
 813     },
 814 );
 815
 816 my %queues_fmt = (
 817     queues => {
 818         type => 'integer',
 819         description => "Number of queues.",
 820         minimum => 2,
 821         optional => 1
 822     }
 823 );
 824
 825 my $add_throttle_desc = sub {
 826     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 827     my $d = {
 828         type => $type,
 829         format_description => $unit,
 830         description => "Maximum $what in $longunit.",
 831         optional => 1,
 832     };
 833     $d->{minimum} = $minimum if defined($minimum);
 834     $drivedesc_base{$key} = $d;
 835 };
 836 # throughput: (leaky bucket)
 837 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 838 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 839 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 840 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 841 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 842 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 843 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 844 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 845 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 846
 847 # pools: (pool of IO before throttling starts taking effect)
 848 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 849 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 850 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 851 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 852 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 853 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 854
 855 # burst lengths
 856 $add_throttle_desc->('bps_max_length',  'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 857 $add_throttle_desc->('bps_rd_length',   'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 858 $add_throttle_desc->('bps_wr_length',   'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 859 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 860 $add_throttle_desc->('iops_rd_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 861 $add_throttle_desc->('iops_wr_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 862
 863 my $ide_fmt = {
 864     %drivedesc_base,
 865     %rerror_fmt,
 866     %model_fmt,
 867 };
 868 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 869
 870 my $idedesc = {
 871     optional => 1,
 872     type => 'string', format => $ide_fmt,
 873     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 874 };
 875 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 876
 877 my $scsi_fmt = {
 878     %drivedesc_base,
 879     %iothread_fmt,
 880     %queues_fmt,
 881 };
 882 my $scsidesc = {
 883     optional => 1,
 884     type => 'string', format => $scsi_fmt,
 885     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 886 };
 887 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 888
 889 my $sata_fmt = {
 890     %drivedesc_base,
 891     %rerror_fmt,
 892 };
 893 my $satadesc = {
 894     optional => 1,
 895     type => 'string', format => $sata_fmt,
 896     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 897 };
 898 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 899
 900 my $virtio_fmt = {
 901     %drivedesc_base,
 902     %iothread_fmt,
 903     %rerror_fmt,
 904 };
 905 my $virtiodesc = {
 906     optional => 1,
 907     type => 'string', format => $virtio_fmt,
 908     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 909 };
 910 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 911
 912 my $alldrive_fmt = {
 913     %drivedesc_base,
 914     %rerror_fmt,
 915     %iothread_fmt,
 916     %model_fmt,
 917     %queues_fmt,
 918 };
 919
 920 my $efidisk_fmt = {
 921     volume => { alias => 'file' },
 922     file => {
 923         type => 'string',
 924         format => 'pve-volume-id-or-qm-path',
 925         default_key => 1,
 926         format_description => 'volume',
 927         description => "The drive's backing volume.",
 928     },
 929     format => get_standard_option('pve-qm-image-format'),
 930     size => {
 931         type => 'string',
 932         format => 'disk-size',
 933         format_description => 'DiskSize',
 934         description => "Disk size. This is purely informational and has no effect.",
 935         optional => 1,
 936     },
 937 };
 938
 939 my $efidisk_desc = {
 940     optional => 1,
 941     type => 'string', format => $efidisk_fmt,
 942     description => "Configure a Disk for storing EFI vars",
 943 };
 944
 945 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 946
 947 my $usb_fmt = {
 948     host => {
 949         default_key => 1,
 950         type => 'string', format => 'pve-qm-usb-device',
 951         format_description => 'HOSTUSBDEVICE|spice',
 952         description => <<EODESCR,
 953 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 954
 955  'bus-port(.port)*' (decimal numbers) or
 956  'vendor_id:product_id' (hexadeciaml numbers) or
 957  'spice'
 958
 959 You can use the 'lsusb -t' command to list existing usb devices.
 960
 961 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 962
 963 The value 'spice' can be used to add a usb redirection devices for spice.
 964 EODESCR
 965     },
 966     usb3 => {
 967         optional => 1,
 968         type => 'boolean',
 969         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 970         default => 0,
 971     },
 972 };
 973
 974 my $usbdesc = {
 975     optional => 1,
 976     type => 'string', format => $usb_fmt,
 977     description => "Configure an USB device (n is 0 to 4).",
 978 };
 979 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 980
 981 # NOTE: the match-groups of this regex are used in parse_hostpci
 982 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 983 my $hostpci_fmt = {
 984     host => {
 985         default_key => 1,
 986         type => 'string',
 987         pattern => qr/$PCIRE(;$PCIRE)*/,
 988         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 989         description => <<EODESCR,
 990 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 991 of PCI virtual functions of the host. HOSTPCIID syntax is:
 992
 993 'bus:dev.func' (hexadecimal numbers)
 994
 995 You can us the 'lspci' command to list existing PCI devices.
 996 EODESCR
 997     },
 998     rombar => {
 999         type => 'boolean',
1000         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1001         optional => 1,
1002         default => 1,
1003     },
1004     pcie => {
1005         type => 'boolean',
1006         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1007         optional => 1,
1008         default => 0,
1009     },
1010     'x-vga' => {
1011         type => 'boolean',
1012         description =>  "Enable vfio-vga device support.",
1013         optional => 1,
1014         default => 0,
1015     },
1016 };
1017 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1018
1019 my $hostpcidesc = {
1020         optional => 1,
1021         type => 'string', format => 'pve-qm-hostpci',
1022         description => "Map host PCI devices into guest.",
1023         verbose_description =>  <<EODESCR,
1024 Map host PCI devices into guest.
1025
1026 NOTE: This option allows direct access to host hardware. So it is no longer
1027 possible to migrate such machines - use with special care.
1028
1029 CAUTION: Experimental! User reported problems with this option.
1030 EODESCR
1031 };
1032 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1033
1034 my $serialdesc = {
1035         optional => 1,
1036         type => 'string',
1037         pattern => '(/dev/.+|socket)',
1038         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1039         verbose_description =>  <<EODESCR,
1040 Create a serial device inside the VM (n is 0 to 3), and pass through a
1041 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1042 host side (use 'qm terminal' to open a terminal connection).
1043
1044 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1045
1046 CAUTION: Experimental! User reported problems with this option.
1047 EODESCR
1048 };
1049
1050 my $paralleldesc= {
1051         optional => 1,
1052         type => 'string',
1053         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1054         description =>  "Map host parallel devices (n is 0 to 2).",
1055         verbose_description =>  <<EODESCR,
1056 Map host parallel devices (n is 0 to 2).
1057
1058 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1059
1060 CAUTION: Experimental! User reported problems with this option.
1061 EODESCR
1062 };
1063
1064 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1065     $confdesc->{"parallel$i"} = $paralleldesc;
1066 }
1067
1068 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1069     $confdesc->{"serial$i"} = $serialdesc;
1070 }
1071
1072 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1073     $confdesc->{"hostpci$i"} = $hostpcidesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1077     $drivename_hash->{"ide$i"} = 1;
1078     $confdesc->{"ide$i"} = $idedesc;
1079 }
1080
1081 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1082     $drivename_hash->{"sata$i"} = 1;
1083     $confdesc->{"sata$i"} = $satadesc;
1084 }
1085
1086 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1087     $drivename_hash->{"scsi$i"} = 1;
1088     $confdesc->{"scsi$i"} = $scsidesc ;
1089 }
1090
1091 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1092     $drivename_hash->{"virtio$i"} = 1;
1093     $confdesc->{"virtio$i"} = $virtiodesc;
1094 }
1095
1096 $drivename_hash->{efidisk0} = 1;
1097 $confdesc->{efidisk0} = $efidisk_desc;
1098
1099 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1100     $confdesc->{"usb$i"} = $usbdesc;
1101 }
1102
1103 my $unuseddesc = {
1104     optional => 1,
1105     type => 'string', format => 'pve-volume-id',
1106     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1107 };
1108
1109 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1110     $confdesc->{"unused$i"} = $unuseddesc;
1111 }
1112
1113 my $kvm_api_version = 0;
1114
1115 sub kvm_version {
1116
1117     return $kvm_api_version if $kvm_api_version;
1118
1119     my $fh = IO::File->new("</dev/kvm") ||
1120         return 0;
1121
1122     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1123         $kvm_api_version = $v;
1124     }
1125
1126     $fh->close();
1127
1128     return  $kvm_api_version;
1129 }
1130
1131 my $kvm_user_version;
1132
1133 sub kvm_user_version {
1134
1135     return $kvm_user_version if $kvm_user_version;
1136
1137     $kvm_user_version = 'unknown';
1138
1139     my $code = sub {
1140         my $line = shift;
1141         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1142             $kvm_user_version = $2;
1143         }
1144     };
1145
1146     eval { run_command("kvm -version", outfunc => $code); };
1147     warn $@ if $@;
1148
1149     return $kvm_user_version;
1150
1151 }
1152
1153 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1154
1155 sub valid_drive_names {
1156     # order is important - used to autoselect boot disk
1157     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1158             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1159             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1160             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1161             'efidisk0');
1162 }
1163
1164 sub is_valid_drivename {
1165     my $dev = shift;
1166
1167     return defined($drivename_hash->{$dev});
1168 }
1169
1170 sub option_exists {
1171     my $key = shift;
1172     return defined($confdesc->{$key});
1173 }
1174
1175 sub nic_models {
1176     return $nic_model_list;
1177 }
1178
1179 sub os_list_description {
1180
1181     return {
1182         other => 'Other',
1183         wxp => 'Windows XP',
1184         w2k => 'Windows 2000',
1185         w2k3 =>, 'Windows 2003',
1186         w2k8 => 'Windows 2008',
1187         wvista => 'Windows Vista',
1188         win7 => 'Windows 7',
1189         win8 => 'Windows 8/2012',
1190         l24 => 'Linux 2.4',
1191         l26 => 'Linux 2.6',
1192     };
1193 }
1194
1195 my $cdrom_path;
1196
1197 sub get_cdrom_path {
1198
1199     return  $cdrom_path if $cdrom_path;
1200
1201     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1202     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1203     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1204 }
1205
1206 sub get_iso_path {
1207     my ($storecfg, $vmid, $cdrom) = @_;
1208
1209     if ($cdrom eq 'cdrom') {
1210         return get_cdrom_path();
1211     } elsif ($cdrom eq 'none') {
1212         return '';
1213     } elsif ($cdrom =~ m|^/|) {
1214         return $cdrom;
1215     } else {
1216         return PVE::Storage::path($storecfg, $cdrom);
1217     }
1218 }
1219
1220 # try to convert old style file names to volume IDs
1221 sub filename_to_volume_id {
1222     my ($vmid, $file, $media) = @_;
1223
1224     if (!($file eq 'none' || $file eq 'cdrom' ||
1225           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1226
1227         return undef if $file =~ m|/|;
1228
1229         if ($media && $media eq 'cdrom') {
1230             $file = "local:iso/$file";
1231         } else {
1232             $file = "local:$vmid/$file";
1233         }
1234     }
1235
1236     return $file;
1237 }
1238
1239 sub verify_media_type {
1240     my ($opt, $vtype, $media) = @_;
1241
1242     return if !$media;
1243
1244     my $etype;
1245     if ($media eq 'disk') {
1246         $etype = 'images';
1247     } elsif ($media eq 'cdrom') {
1248         $etype = 'iso';
1249     } else {
1250         die "internal error";
1251     }
1252
1253     return if ($vtype eq $etype);
1254
1255     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1256 }
1257
1258 sub cleanup_drive_path {
1259     my ($opt, $storecfg, $drive) = @_;
1260
1261     # try to convert filesystem paths to volume IDs
1262
1263     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1264         ($drive->{file} !~ m|^/dev/.+|) &&
1265         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1266         ($drive->{file} !~ m/^\d+$/)) {
1267         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1268         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1269         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1270         verify_media_type($opt, $vtype, $drive->{media});
1271         $drive->{file} = $volid;
1272     }
1273
1274     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1275 }
1276
1277 sub parse_hotplug_features {
1278     my ($data) = @_;
1279
1280     my $res = {};
1281
1282     return $res if $data eq '0';
1283
1284     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1285
1286     foreach my $feature (PVE::Tools::split_list($data)) {
1287         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1288             $res->{$1} = 1;
1289         } else {
1290             die "invalid hotplug feature '$feature'\n";
1291         }
1292     }
1293     return $res;
1294 }
1295
1296 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1297 sub pve_verify_hotplug_features {
1298     my ($value, $noerr) = @_;
1299
1300     return $value if parse_hotplug_features($value);
1301
1302     return undef if $noerr;
1303
1304     die "unable to parse hotplug option\n";
1305 }
1306
1307 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1308 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1309 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1310 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1311 #        [,iothread=on][,serial=serial][,model=model]
1312
1313 sub parse_drive {
1314     my ($key, $data) = @_;
1315
1316     my ($interface, $index);
1317
1318     if ($key =~ m/^([^\d]+)(\d+)$/) {
1319         $interface = $1;
1320         $index = $2;
1321     } else {
1322         return undef;
1323     }
1324
1325     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1326                                      : $confdesc->{$key}->{format};
1327     if (!$desc) {
1328         warn "invalid drive key: $key\n";
1329         return undef;
1330     }
1331     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1332     return undef if !$res;
1333     $res->{interface} = $interface;
1334     $res->{index} = $index;
1335
1336     my $error = 0;
1337     foreach my $opt (qw(bps bps_rd bps_wr)) {
1338         if (my $bps = defined(delete $res->{$opt})) {
1339             if (defined($res->{"m$opt"})) {
1340                 warn "both $opt and m$opt specified\n";
1341                 ++$error;
1342                 next;
1343             }
1344             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1345         }
1346     }
1347
1348     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1349     for my $requirement (
1350         [bps_max_length => 'mbps_max'],
1351         [bps_rd_max_length => 'mbps_rd_max'],
1352         [bps_wr_max_length => 'mbps_wr_max'],
1353         [iops_max_length => 'iops_max'],
1354         [iops_rd_max_length => 'iops_rd_max'],
1355         [iops_wr_max_length => 'iops_wr_max']) {
1356         my ($option, $requires) = @$requirement;
1357         if ($res->{$option} && !$res->{$requires}) {
1358             warn "$option requires $requires\n";
1359             ++$error;
1360         }
1361     }
1362
1363     return undef if $error;
1364
1365     return undef if $res->{mbps_rd} && $res->{mbps};
1366     return undef if $res->{mbps_wr} && $res->{mbps};
1367     return undef if $res->{iops_rd} && $res->{iops};
1368     return undef if $res->{iops_wr} && $res->{iops};
1369
1370     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1371         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1372         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1373         return undef if $res->{interface} eq 'virtio';
1374     }
1375
1376     if (my $size = $res->{size}) {
1377         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1378     }
1379
1380     return $res;
1381 }
1382
1383 sub print_drive {
1384     my ($vmid, $drive) = @_;
1385     my $data = { %$drive };
1386     delete $data->{$_} for qw(index interface);
1387     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1388 }
1389
1390 sub scsi_inquiry {
1391     my($fh, $noerr) = @_;
1392
1393     my $SG_IO = 0x2285;
1394     my $SG_GET_VERSION_NUM = 0x2282;
1395
1396     my $versionbuf = "\x00" x 8;
1397     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1398     if (!$ret) {
1399         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1400         return undef;
1401     }
1402     my $version = unpack("I", $versionbuf);
1403     if ($version < 30000) {
1404         die "scsi generic interface too old\n"  if !$noerr;
1405         return undef;
1406     }
1407
1408     my $buf = "\x00" x 36;
1409     my $sensebuf = "\x00" x 8;
1410     my $cmd = pack("C x3 C x1", 0x12, 36);
1411
1412     # see /usr/include/scsi/sg.h
1413     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1414
1415     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1416                       length($sensebuf), 0, length($buf), $buf,
1417                       $cmd, $sensebuf, 6000);
1418
1419     $ret = ioctl($fh, $SG_IO, $packet);
1420     if (!$ret) {
1421         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1422         return undef;
1423     }
1424
1425     my @res = unpack($sg_io_hdr_t, $packet);
1426     if ($res[17] || $res[18]) {
1427         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1428         return undef;
1429     }
1430
1431     my $res = {};
1432     (my $byte0, my $byte1, $res->{vendor},
1433      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1434
1435     $res->{removable} = $byte1 & 128 ? 1 : 0;
1436     $res->{type} = $byte0 & 31;
1437
1438     return $res;
1439 }
1440
1441 sub path_is_scsi {
1442     my ($path) = @_;
1443
1444     my $fh = IO::File->new("+<$path") || return undef;
1445     my $res = scsi_inquiry($fh, 1);
1446     close($fh);
1447
1448     return $res;
1449 }
1450
1451 sub machine_type_is_q35 {
1452     my ($conf) = @_;
1453
1454     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1455 }
1456
1457 sub print_tabletdevice_full {
1458     my ($conf) = @_;
1459
1460     my $q35 = machine_type_is_q35($conf);
1461
1462     # we use uhci for old VMs because tablet driver was buggy in older qemu
1463     my $usbbus = $q35 ? "ehci" : "uhci";
1464
1465     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1466 }
1467
1468 sub print_drivedevice_full {
1469     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1470
1471     my $device = '';
1472     my $maxdev = 0;
1473
1474     if ($drive->{interface} eq 'virtio') {
1475         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1476         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1477         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1478     } elsif ($drive->{interface} eq 'scsi') {
1479
1480         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1481         my $unit = $drive->{index} % $maxdev;
1482         my $devicetype = 'hd';
1483         my $path = '';
1484         if (drive_is_cdrom($drive)) {
1485             $devicetype = 'cd';
1486         } else {
1487             if ($drive->{file} =~ m|^/|) {
1488                 $path = $drive->{file};
1489                 if (my $info = path_is_scsi($path)) {
1490                     if ($info->{type} == 0) {
1491                         $devicetype = 'block';
1492                     } elsif ($info->{type} == 1) { # tape
1493                         $devicetype = 'generic';
1494                     }
1495                 }
1496             } else {
1497                  $path = PVE::Storage::path($storecfg, $drive->{file});
1498             }
1499
1500             if($path =~ m/^iscsi\:\/\//){
1501                 $devicetype = 'generic';
1502             }
1503         }
1504
1505         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1506             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1507         } else {
1508             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1509         }
1510
1511     } elsif ($drive->{interface} eq 'ide'){
1512         $maxdev = 2;
1513         my $controller = int($drive->{index} / $maxdev);
1514         my $unit = $drive->{index} % $maxdev;
1515         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1516
1517         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1518         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1519             $model = URI::Escape::uri_unescape($model);
1520             $device .= ",model=$model";
1521         }
1522     } elsif ($drive->{interface} eq 'sata'){
1523         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1524         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1525         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1526     } elsif ($drive->{interface} eq 'usb') {
1527         die "implement me";
1528         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1529     } else {
1530         die "unsupported interface type";
1531     }
1532
1533     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1534
1535     return $device;
1536 }
1537
1538 sub get_initiator_name {
1539     my $initiator;
1540
1541     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1542     while (defined(my $line = <$fh>)) {
1543         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1544         $initiator = $1;
1545         last;
1546     }
1547     $fh->close();
1548
1549     return $initiator;
1550 }
1551
1552 sub print_drive_full {
1553     my ($storecfg, $vmid, $drive) = @_;
1554
1555     my $path;
1556     my $volid = $drive->{file};
1557     my $format;
1558
1559     if (drive_is_cdrom($drive)) {
1560         $path = get_iso_path($storecfg, $vmid, $volid);
1561     } else {
1562         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1563         if ($storeid) {
1564             $path = PVE::Storage::path($storecfg, $volid);
1565             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1566             $format = qemu_img_format($scfg, $volname);
1567         } else {
1568             $path = $volid;
1569             $format = "raw";
1570         }
1571    }
1572
1573     my $opts = '';
1574     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1575     foreach my $o (@qemu_drive_options) {
1576         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1577     }
1578     if (my $serial = $drive->{serial}) {
1579         $serial = URI::Escape::uri_unescape($serial);
1580         $opts .= ",serial=$serial";
1581     }
1582
1583     $opts .= ",format=$format" if $format && !$drive->{format};
1584
1585     foreach my $o (qw(bps bps_rd bps_wr)) {
1586         my $v = $drive->{"m$o"};
1587         $opts .= ",$o=" . int($v*1024*1024) if $v;
1588     }
1589
1590     my $cache_direct = 0;
1591
1592     if (my $cache = $drive->{cache}) {
1593         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1594     } elsif (!drive_is_cdrom($drive)) {
1595         $opts .= ",cache=none";
1596         $cache_direct = 1;
1597     }
1598
1599     # aio native works only with O_DIRECT
1600     if (!$drive->{aio}) {
1601         if($cache_direct) {
1602             $opts .= ",aio=native";
1603         } else {
1604             $opts .= ",aio=threads";
1605         }
1606     }
1607
1608     if (!drive_is_cdrom($drive)) {
1609         my $detectzeroes;
1610         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1611             $detectzeroes = 'off';
1612         } elsif ($drive->{discard}) {
1613             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1614         } else {
1615             # This used to be our default with discard not being specified:
1616             $detectzeroes = 'on';
1617         }
1618         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1619     }
1620
1621     my $pathinfo = $path ? "file=$path," : '';
1622
1623     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1624 }
1625
1626 sub print_netdevice_full {
1627     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1628
1629     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1630
1631     my $device = $net->{model};
1632     if ($net->{model} eq 'virtio') {
1633          $device = 'virtio-net-pci';
1634      };
1635
1636     my $pciaddr = print_pci_addr("$netid", $bridges);
1637     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1638     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1639         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1640         my $vectors = $net->{queues} * 2 + 2;
1641         $tmpstr .= ",vectors=$vectors,mq=on";
1642     }
1643     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1644
1645     if ($use_old_bios_files) {
1646         my $romfile;
1647         if ($device eq 'virtio-net-pci') {
1648             $romfile = 'pxe-virtio.rom';
1649         } elsif ($device eq 'e1000') {
1650             $romfile = 'pxe-e1000.rom';
1651         } elsif ($device eq 'ne2k') {
1652             $romfile = 'pxe-ne2k_pci.rom';
1653         } elsif ($device eq 'pcnet') {
1654             $romfile = 'pxe-pcnet.rom';
1655         } elsif ($device eq 'rtl8139') {
1656             $romfile = 'pxe-rtl8139.rom';
1657         }
1658         $tmpstr .= ",romfile=$romfile" if $romfile;
1659     }
1660
1661     return $tmpstr;
1662 }
1663
1664 sub print_netdev_full {
1665     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1666
1667     my $i = '';
1668     if ($netid =~ m/^net(\d+)$/) {
1669         $i = int($1);
1670     }
1671
1672     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1673
1674     my $ifname = "tap${vmid}i$i";
1675
1676     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1677     die "interface name '$ifname' is too long (max 15 character)\n"
1678         if length($ifname) >= 16;
1679
1680     my $vhostparam = '';
1681     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1682
1683     my $vmname = $conf->{name} || "vm$vmid";
1684
1685     my $netdev = "";
1686     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1687
1688     if ($net->{bridge}) {
1689         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1690     } else {
1691         $netdev = "type=user,id=$netid,hostname=$vmname";
1692     }
1693
1694     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1695
1696     return $netdev;
1697 }
1698
1699
1700 sub print_cpu_device {
1701     my ($conf, $id) = @_;
1702
1703     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1704     my $cpu = $nokvm ? "qemu64" : "kvm64";
1705     if (my $cputype = $conf->{cpu}) {
1706         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1707             or die "Cannot parse cpu description: $cputype\n";
1708         $cpu = $cpuconf->{cputype};
1709     }
1710
1711     my $sockets = 1;
1712     $sockets = $conf->{sockets} if  $conf->{sockets};
1713     my $cores = $conf->{cores} || 1;
1714
1715     my $current_core = ($id - 1) % $cores;
1716     my $current_socket = int(($id - $current_core)/$cores);
1717
1718     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1719 }
1720
1721 sub drive_is_cdrom {
1722     my ($drive) = @_;
1723
1724     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1725
1726 }
1727
1728 sub parse_number_sets {
1729     my ($set) = @_;
1730     my $res = [];
1731     foreach my $part (split(/;/, $set)) {
1732         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1733             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1734             push @$res, [ $1, $2 ];
1735         } else {
1736             die "invalid range: $part\n";
1737         }
1738     }
1739     return $res;
1740 }
1741
1742 sub parse_numa {
1743     my ($data) = @_;
1744
1745     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1746     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1747     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1748     return $res;
1749 }
1750
1751 sub parse_hostpci {
1752     my ($value) = @_;
1753
1754     return undef if !$value;
1755
1756     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1757
1758     my @idlist = split(/;/, $res->{host});
1759     delete $res->{host};
1760     foreach my $id (@idlist) {
1761         if ($id =~ /^$PCIRE$/) {
1762             if (defined($2)) {
1763                 push @{$res->{pciid}}, { id => $1, function => $2 };
1764             } else {
1765                 my $pcidevices = lspci($1);
1766                 $res->{pciid} = $pcidevices->{$1};
1767             }
1768         } else {
1769             # should have been caught by parse_property_string already
1770             die "failed to parse PCI id: $id\n";
1771         }
1772     }
1773     return $res;
1774 }
1775
1776 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1777 sub parse_net {
1778     my ($data) = @_;
1779
1780     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1781     if ($@) {
1782         warn $@;
1783         return undef;
1784     }
1785     if (!defined($res->{macaddr})) {
1786         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1787         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1788     }
1789     return $res;
1790 }
1791
1792 sub print_net {
1793     my $net = shift;
1794
1795     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1796 }
1797
1798 sub add_random_macs {
1799     my ($settings) = @_;
1800
1801     foreach my $opt (keys %$settings) {
1802         next if $opt !~ m/^net(\d+)$/;
1803         my $net = parse_net($settings->{$opt});
1804         next if !$net;
1805         $settings->{$opt} = print_net($net);
1806     }
1807 }
1808
1809 sub vm_is_volid_owner {
1810     my ($storecfg, $vmid, $volid) = @_;
1811
1812     if ($volid !~  m|^/|) {
1813         my ($path, $owner);
1814         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1815         if ($owner && ($owner == $vmid)) {
1816             return 1;
1817         }
1818     }
1819
1820     return undef;
1821 }
1822
1823 sub split_flagged_list {
1824     my $text = shift || '';
1825     $text =~ s/[,;]/ /g;
1826     $text =~ s/^\s+//;
1827     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1828 }
1829
1830 sub join_flagged_list {
1831     my ($how, $lst) = @_;
1832     join $how, map { $lst->{$_} . $_ } keys %$lst;
1833 }
1834
1835 sub vmconfig_delete_pending_option {
1836     my ($conf, $key, $force) = @_;
1837
1838     delete $conf->{pending}->{$key};
1839     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1840     $pending_delete_hash->{$key} = $force ? '!' : '';
1841     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1842 }
1843
1844 sub vmconfig_undelete_pending_option {
1845     my ($conf, $key) = @_;
1846
1847     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1848     delete $pending_delete_hash->{$key};
1849
1850     if (%$pending_delete_hash) {
1851         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1852     } else {
1853         delete $conf->{pending}->{delete};
1854     }
1855 }
1856
1857 sub vmconfig_register_unused_drive {
1858     my ($storecfg, $vmid, $conf, $drive) = @_;
1859
1860     if (!drive_is_cdrom($drive)) {
1861         my $volid = $drive->{file};
1862         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1863             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1864         }
1865     }
1866 }
1867
1868 sub vmconfig_cleanup_pending {
1869     my ($conf) = @_;
1870
1871     # remove pending changes when nothing changed
1872     my $changes;
1873     foreach my $opt (keys %{$conf->{pending}}) {
1874         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1875             $changes = 1;
1876             delete $conf->{pending}->{$opt};
1877         }
1878     }
1879
1880     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1881     my $pending_delete_hash = {};
1882     while (my ($opt, $force) = each %$current_delete_hash) {
1883         if (defined($conf->{$opt})) {
1884             $pending_delete_hash->{$opt} = $force;
1885         } else {
1886             $changes = 1;
1887         }
1888     }
1889
1890     if (%$pending_delete_hash) {
1891         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1892     } else {
1893         delete $conf->{pending}->{delete};
1894     }
1895
1896     return $changes;
1897 }
1898
1899 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1900 my $smbios1_fmt = {
1901     uuid => {
1902         type => 'string',
1903         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1904         format_description => 'UUID',
1905         description => "Set SMBIOS1 UUID.",
1906         optional => 1,
1907     },
1908     version => {
1909         type => 'string',
1910         pattern => '\S+',
1911         format_description => 'string',
1912         description => "Set SMBIOS1 version.",
1913         optional => 1,
1914     },
1915     serial => {
1916         type => 'string',
1917         pattern => '\S+',
1918         format_description => 'string',
1919         description => "Set SMBIOS1 serial number.",
1920         optional => 1,
1921     },
1922     manufacturer => {
1923         type => 'string',
1924         pattern => '\S+',
1925         format_description => 'string',
1926         description => "Set SMBIOS1 manufacturer.",
1927         optional => 1,
1928     },
1929     product => {
1930         type => 'string',
1931         pattern => '\S+',
1932         format_description => 'string',
1933         description => "Set SMBIOS1 product ID.",
1934         optional => 1,
1935     },
1936     sku => {
1937         type => 'string',
1938         pattern => '\S+',
1939         format_description => 'string',
1940         description => "Set SMBIOS1 SKU string.",
1941         optional => 1,
1942     },
1943     family => {
1944         type => 'string',
1945         pattern => '\S+',
1946         format_description => 'string',
1947         description => "Set SMBIOS1 family string.",
1948         optional => 1,
1949     },
1950 };
1951
1952 sub parse_smbios1 {
1953     my ($data) = @_;
1954
1955     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1956     warn $@ if $@;
1957     return $res;
1958 }
1959
1960 sub print_smbios1 {
1961     my ($smbios1) = @_;
1962     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1963 }
1964
1965 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1966
1967 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1968 sub verify_bootdisk {
1969     my ($value, $noerr) = @_;
1970
1971     return $value if is_valid_drivename($value);
1972
1973     return undef if $noerr;
1974
1975     die "invalid boot disk '$value'\n";
1976 }
1977
1978 sub parse_watchdog {
1979     my ($value) = @_;
1980
1981     return undef if !$value;
1982
1983     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1984     warn $@ if $@;
1985     return $res;
1986 }
1987
1988 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1989 sub verify_usb_device {
1990     my ($value, $noerr) = @_;
1991
1992     return $value if parse_usb_device($value);
1993
1994     return undef if $noerr;
1995
1996     die "unable to parse usb device\n";
1997 }
1998
1999 # add JSON properties for create and set function
2000 sub json_config_properties {
2001     my $prop = shift;
2002
2003     foreach my $opt (keys %$confdesc) {
2004         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2005         $prop->{$opt} = $confdesc->{$opt};
2006     }
2007
2008     return $prop;
2009 }
2010
2011 sub check_type {
2012     my ($key, $value) = @_;
2013
2014     die "unknown setting '$key'\n" if !$confdesc->{$key};
2015
2016     my $type = $confdesc->{$key}->{type};
2017
2018     if (!defined($value)) {
2019         die "got undefined value\n";
2020     }
2021
2022     if ($value =~ m/[\n\r]/) {
2023         die "property contains a line feed\n";
2024     }
2025
2026     if ($type eq 'boolean') {
2027         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2028         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2029         die "type check ('boolean') failed - got '$value'\n";
2030     } elsif ($type eq 'integer') {
2031         return int($1) if $value =~ m/^(\d+)$/;
2032         die "type check ('integer') failed - got '$value'\n";
2033     } elsif ($type eq 'number') {
2034         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2035         die "type check ('number') failed - got '$value'\n";
2036     } elsif ($type eq 'string') {
2037         if (my $fmt = $confdesc->{$key}->{format}) {
2038             PVE::JSONSchema::check_format($fmt, $value);
2039             return $value;
2040         }
2041         $value =~ s/^\"(.*)\"$/$1/;
2042         return $value;
2043     } else {
2044         die "internal error"
2045     }
2046 }
2047
2048 sub check_iommu_support{
2049     #fixme : need to check IOMMU support
2050     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2051
2052     my $iommu=1;
2053     return $iommu;
2054
2055 }
2056
2057 sub touch_config {
2058     my ($vmid) = @_;
2059
2060     my $conf = PVE::QemuConfig->config_file($vmid);
2061     utime undef, undef, $conf;
2062 }
2063
2064 sub destroy_vm {
2065     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2066
2067     my $conffile = PVE::QemuConfig->config_file($vmid);
2068
2069     my $conf = PVE::QemuConfig->load_config($vmid);
2070
2071     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2072
2073     # only remove disks owned by this VM
2074     foreach_drive($conf, sub {
2075         my ($ds, $drive) = @_;
2076
2077         return if drive_is_cdrom($drive);
2078
2079         my $volid = $drive->{file};
2080
2081         return if !$volid || $volid =~ m|^/|;
2082
2083         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2084         return if !$path || !$owner || ($owner != $vmid);
2085
2086         PVE::Storage::vdisk_free($storecfg, $volid);
2087     });
2088
2089     if ($keep_empty_config) {
2090         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2091     } else {
2092         unlink $conffile;
2093     }
2094
2095     # also remove unused disk
2096     eval {
2097         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2098
2099         eval {
2100             PVE::Storage::foreach_volid($dl, sub {
2101                 my ($volid, $sid, $volname, $d) = @_;
2102                 PVE::Storage::vdisk_free($storecfg, $volid);
2103             });
2104         };
2105         warn $@ if $@;
2106
2107     };
2108     warn $@ if $@;
2109 }
2110
2111 sub parse_vm_config {
2112     my ($filename, $raw) = @_;
2113
2114     return undef if !defined($raw);
2115
2116     my $res = {
2117         digest => Digest::SHA::sha1_hex($raw),
2118         snapshots => {},
2119         pending => {},
2120     };
2121
2122     $filename =~ m|/qemu-server/(\d+)\.conf$|
2123         || die "got strange filename '$filename'";
2124
2125     my $vmid = $1;
2126
2127     my $conf = $res;
2128     my $descr;
2129     my $section = '';
2130
2131     my @lines = split(/\n/, $raw);
2132     foreach my $line (@lines) {
2133         next if $line =~ m/^\s*$/;
2134
2135         if ($line =~ m/^\[PENDING\]\s*$/i) {
2136             $section = 'pending';
2137             if (defined($descr)) {
2138                 $descr =~ s/\s+$//;
2139                 $conf->{description} = $descr;
2140             }
2141             $descr = undef;
2142             $conf = $res->{$section} = {};
2143             next;
2144
2145         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2146             $section = $1;
2147             if (defined($descr)) {
2148                 $descr =~ s/\s+$//;
2149                 $conf->{description} = $descr;
2150             }
2151             $descr = undef;
2152             $conf = $res->{snapshots}->{$section} = {};
2153             next;
2154         }
2155
2156         if ($line =~ m/^\#(.*)\s*$/) {
2157             $descr = '' if !defined($descr);
2158             $descr .= PVE::Tools::decode_text($1) . "\n";
2159             next;
2160         }
2161
2162         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2163             $descr = '' if !defined($descr);
2164             $descr .= PVE::Tools::decode_text($2);
2165         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2166             $conf->{snapstate} = $1;
2167         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2168             my $key = $1;
2169             my $value = $2;
2170             $conf->{$key} = $value;
2171         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2172             my $value = $1;
2173             if ($section eq 'pending') {
2174                 $conf->{delete} = $value; # we parse this later
2175             } else {
2176                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2177             }
2178         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2179             my $key = $1;
2180             my $value = $2;
2181             eval { $value = check_type($key, $value); };
2182             if ($@) {
2183                 warn "vm $vmid - unable to parse value of '$key' - $@";
2184             } else {
2185                 $key = 'ide2' if $key eq 'cdrom';
2186                 my $fmt = $confdesc->{$key}->{format};
2187                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2188                     my $v = parse_drive($key, $value);
2189                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2190                         $v->{file} = $volid;
2191                         $value = print_drive($vmid, $v);
2192                     } else {
2193                         warn "vm $vmid - unable to parse value of '$key'\n";
2194                         next;
2195                     }
2196                 }
2197
2198                 $conf->{$key} = $value;
2199             }
2200         }
2201     }
2202
2203     if (defined($descr)) {
2204         $descr =~ s/\s+$//;
2205         $conf->{description} = $descr;
2206     }
2207     delete $res->{snapstate}; # just to be sure
2208
2209     return $res;
2210 }
2211
2212 sub write_vm_config {
2213     my ($filename, $conf) = @_;
2214
2215     delete $conf->{snapstate}; # just to be sure
2216
2217     if ($conf->{cdrom}) {
2218         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2219         $conf->{ide2} = $conf->{cdrom};
2220         delete $conf->{cdrom};
2221     }
2222
2223     # we do not use 'smp' any longer
2224     if ($conf->{sockets}) {
2225         delete $conf->{smp};
2226     } elsif ($conf->{smp}) {
2227         $conf->{sockets} = $conf->{smp};
2228         delete $conf->{cores};
2229         delete $conf->{smp};
2230     }
2231
2232     my $used_volids = {};
2233
2234     my $cleanup_config = sub {
2235         my ($cref, $pending, $snapname) = @_;
2236
2237         foreach my $key (keys %$cref) {
2238             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2239                 $key eq 'snapstate' || $key eq 'pending';
2240             my $value = $cref->{$key};
2241             if ($key eq 'delete') {
2242                 die "propertry 'delete' is only allowed in [PENDING]\n"
2243                     if !$pending;
2244                 # fixme: check syntax?
2245                 next;
2246             }
2247             eval { $value = check_type($key, $value); };
2248             die "unable to parse value of '$key' - $@" if $@;
2249
2250             $cref->{$key} = $value;
2251
2252             if (!$snapname && is_valid_drivename($key)) {
2253                 my $drive = parse_drive($key, $value);
2254                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2255             }
2256         }
2257     };
2258
2259     &$cleanup_config($conf);
2260
2261     &$cleanup_config($conf->{pending}, 1);
2262
2263     foreach my $snapname (keys %{$conf->{snapshots}}) {
2264         die "internal error" if $snapname eq 'pending';
2265         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2266     }
2267
2268     # remove 'unusedX' settings if we re-add a volume
2269     foreach my $key (keys %$conf) {
2270         my $value = $conf->{$key};
2271         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2272             delete $conf->{$key};
2273         }
2274     }
2275
2276     my $generate_raw_config = sub {
2277         my ($conf, $pending) = @_;
2278
2279         my $raw = '';
2280
2281         # add description as comment to top of file
2282         if (defined(my $descr = $conf->{description})) {
2283             if ($descr) {
2284                 foreach my $cl (split(/\n/, $descr)) {
2285                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2286                 }
2287             } else {
2288                 $raw .= "#\n" if $pending;
2289             }
2290         }
2291
2292         foreach my $key (sort keys %$conf) {
2293             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2294             $raw .= "$key: $conf->{$key}\n";
2295         }
2296         return $raw;
2297     };
2298
2299     my $raw = &$generate_raw_config($conf);
2300
2301     if (scalar(keys %{$conf->{pending}})){
2302         $raw .= "\n[PENDING]\n";
2303         $raw .= &$generate_raw_config($conf->{pending}, 1);
2304     }
2305
2306     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2307         $raw .= "\n[$snapname]\n";
2308         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2309     }
2310
2311     return $raw;
2312 }
2313
2314 sub load_defaults {
2315
2316     my $res = {};
2317
2318     # we use static defaults from our JSON schema configuration
2319     foreach my $key (keys %$confdesc) {
2320         if (defined(my $default = $confdesc->{$key}->{default})) {
2321             $res->{$key} = $default;
2322         }
2323     }
2324
2325     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2326     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2327
2328     return $res;
2329 }
2330
2331 sub config_list {
2332     my $vmlist = PVE::Cluster::get_vmlist();
2333     my $res = {};
2334     return $res if !$vmlist || !$vmlist->{ids};
2335     my $ids = $vmlist->{ids};
2336
2337     foreach my $vmid (keys %$ids) {
2338         my $d = $ids->{$vmid};
2339         next if !$d->{node} || $d->{node} ne $nodename;
2340         next if !$d->{type} || $d->{type} ne 'qemu';
2341         $res->{$vmid}->{exists} = 1;
2342     }
2343     return $res;
2344 }
2345
2346 # test if VM uses local resources (to prevent migration)
2347 sub check_local_resources {
2348     my ($conf, $noerr) = @_;
2349
2350     my $loc_res = 0;
2351
2352     $loc_res = 1 if $conf->{hostusb}; # old syntax
2353     $loc_res = 1 if $conf->{hostpci}; # old syntax
2354
2355     foreach my $k (keys %$conf) {
2356         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2357         # sockets are safe: they will recreated be on the target side post-migrate
2358         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2359         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2360     }
2361
2362     die "VM uses local resources\n" if $loc_res && !$noerr;
2363
2364     return $loc_res;
2365 }
2366
2367 # check if used storages are available on all nodes (use by migrate)
2368 sub check_storage_availability {
2369     my ($storecfg, $conf, $node) = @_;
2370
2371     foreach_drive($conf, sub {
2372         my ($ds, $drive) = @_;
2373
2374         my $volid = $drive->{file};
2375         return if !$volid;
2376
2377         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2378         return if !$sid;
2379
2380         # check if storage is available on both nodes
2381         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2382         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2383    });
2384 }
2385
2386 # list nodes where all VM images are available (used by has_feature API)
2387 sub shared_nodes {
2388     my ($conf, $storecfg) = @_;
2389
2390     my $nodelist = PVE::Cluster::get_nodelist();
2391     my $nodehash = { map { $_ => 1 } @$nodelist };
2392     my $nodename = PVE::INotify::nodename();
2393
2394     foreach_drive($conf, sub {
2395         my ($ds, $drive) = @_;
2396
2397         my $volid = $drive->{file};
2398         return if !$volid;
2399
2400         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2401         if ($storeid) {
2402             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2403             if ($scfg->{disable}) {
2404                 $nodehash = {};
2405             } elsif (my $avail = $scfg->{nodes}) {
2406                 foreach my $node (keys %$nodehash) {
2407                     delete $nodehash->{$node} if !$avail->{$node};
2408                 }
2409             } elsif (!$scfg->{shared}) {
2410                 foreach my $node (keys %$nodehash) {
2411                     delete $nodehash->{$node} if $node ne $nodename
2412                 }
2413             }
2414         }
2415     });
2416
2417     return $nodehash
2418 }
2419
2420 sub check_cmdline {
2421     my ($pidfile, $pid) = @_;
2422
2423     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2424     if (defined($fh)) {
2425         my $line = <$fh>;
2426         $fh->close;
2427         return undef if !$line;
2428         my @param = split(/\0/, $line);
2429
2430         my $cmd = $param[0];
2431         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2432
2433         for (my $i = 0; $i < scalar (@param); $i++) {
2434             my $p = $param[$i];
2435             next if !$p;
2436             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2437                 my $p = $param[$i+1];
2438                 return 1 if $p && ($p eq $pidfile);
2439                 return undef;
2440             }
2441         }
2442     }
2443     return undef;
2444 }
2445
2446 sub check_running {
2447     my ($vmid, $nocheck, $node) = @_;
2448
2449     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2450
2451     die "unable to find configuration file for VM $vmid - no such machine\n"
2452         if !$nocheck && ! -f $filename;
2453
2454     my $pidfile = pidfile_name($vmid);
2455
2456     if (my $fd = IO::File->new("<$pidfile")) {
2457         my $st = stat($fd);
2458         my $line = <$fd>;
2459         close($fd);
2460
2461         my $mtime = $st->mtime;
2462         if ($mtime > time()) {
2463             warn "file '$filename' modified in future\n";
2464         }
2465
2466         if ($line =~ m/^(\d+)$/) {
2467             my $pid = $1;
2468             if (check_cmdline($pidfile, $pid)) {
2469                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2470                     return $pid;
2471                 }
2472             }
2473         }
2474     }
2475
2476     return undef;
2477 }
2478
2479 sub vzlist {
2480
2481     my $vzlist = config_list();
2482
2483     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2484
2485     while (defined(my $de = $fd->read)) {
2486         next if $de !~ m/^(\d+)\.pid$/;
2487         my $vmid = $1;
2488         next if !defined($vzlist->{$vmid});
2489         if (my $pid = check_running($vmid)) {
2490             $vzlist->{$vmid}->{pid} = $pid;
2491         }
2492     }
2493
2494     return $vzlist;
2495 }
2496
2497 sub disksize {
2498     my ($storecfg, $conf) = @_;
2499
2500     my $bootdisk = $conf->{bootdisk};
2501     return undef if !$bootdisk;
2502     return undef if !is_valid_drivename($bootdisk);
2503
2504     return undef if !$conf->{$bootdisk};
2505
2506     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2507     return undef if !defined($drive);
2508
2509     return undef if drive_is_cdrom($drive);
2510
2511     my $volid = $drive->{file};
2512     return undef if !$volid;
2513
2514     return $drive->{size};
2515 }
2516
2517 my $last_proc_pid_stat;
2518
2519 # get VM status information
2520 # This must be fast and should not block ($full == false)
2521 # We only query KVM using QMP if $full == true (this can be slow)
2522 sub vmstatus {
2523     my ($opt_vmid, $full) = @_;
2524
2525     my $res = {};
2526
2527     my $storecfg = PVE::Storage::config();
2528
2529     my $list = vzlist();
2530     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2531
2532     my $cpucount = $cpuinfo->{cpus} || 1;
2533
2534     foreach my $vmid (keys %$list) {
2535         next if $opt_vmid && ($vmid ne $opt_vmid);
2536
2537         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2538         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2539
2540         my $d = {};
2541         $d->{pid} = $list->{$vmid}->{pid};
2542
2543         # fixme: better status?
2544         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2545
2546         my $size = disksize($storecfg, $conf);
2547         if (defined($size)) {
2548             $d->{disk} = 0; # no info available
2549             $d->{maxdisk} = $size;
2550         } else {
2551             $d->{disk} = 0;
2552             $d->{maxdisk} = 0;
2553         }
2554
2555         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2556         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2557         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2558
2559         $d->{name} = $conf->{name} || "VM $vmid";
2560         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2561
2562         if ($conf->{balloon}) {
2563             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2564             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2565         }
2566
2567         $d->{uptime} = 0;
2568         $d->{cpu} = 0;
2569         $d->{mem} = 0;
2570
2571         $d->{netout} = 0;
2572         $d->{netin} = 0;
2573
2574         $d->{diskread} = 0;
2575         $d->{diskwrite} = 0;
2576
2577         $d->{template} = PVE::QemuConfig->is_template($conf);
2578
2579         $res->{$vmid} = $d;
2580     }
2581
2582     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2583     foreach my $dev (keys %$netdev) {
2584         next if $dev !~ m/^tap([1-9]\d*)i/;
2585         my $vmid = $1;
2586         my $d = $res->{$vmid};
2587         next if !$d;
2588
2589         $d->{netout} += $netdev->{$dev}->{receive};
2590         $d->{netin} += $netdev->{$dev}->{transmit};
2591
2592         if ($full) {
2593             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2594             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2595         }
2596
2597     }
2598
2599     my $ctime = gettimeofday;
2600
2601     foreach my $vmid (keys %$list) {
2602
2603         my $d = $res->{$vmid};
2604         my $pid = $d->{pid};
2605         next if !$pid;
2606
2607         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2608         next if !$pstat; # not running
2609
2610         my $used = $pstat->{utime} + $pstat->{stime};
2611
2612         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2613
2614         if ($pstat->{vsize}) {
2615             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2616         }
2617
2618         my $old = $last_proc_pid_stat->{$pid};
2619         if (!$old) {
2620             $last_proc_pid_stat->{$pid} = {
2621                 time => $ctime,
2622                 used => $used,
2623                 cpu => 0,
2624             };
2625             next;
2626         }
2627
2628         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2629
2630         if ($dtime > 1000) {
2631             my $dutime = $used -  $old->{used};
2632
2633             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2634             $last_proc_pid_stat->{$pid} = {
2635                 time => $ctime,
2636                 used => $used,
2637                 cpu => $d->{cpu},
2638             };
2639         } else {
2640             $d->{cpu} = $old->{cpu};
2641         }
2642     }
2643
2644     return $res if !$full;
2645
2646     my $qmpclient = PVE::QMPClient->new();
2647
2648     my $ballooncb = sub {
2649         my ($vmid, $resp) = @_;
2650
2651         my $info = $resp->{'return'};
2652         return if !$info->{max_mem};
2653
2654         my $d = $res->{$vmid};
2655
2656         # use memory assigned to VM
2657         $d->{maxmem} = $info->{max_mem};
2658         $d->{balloon} = $info->{actual};
2659
2660         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2661             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2662             $d->{freemem} = $info->{free_mem};
2663         }
2664
2665         $d->{ballooninfo} = $info;
2666     };
2667
2668     my $blockstatscb = sub {
2669         my ($vmid, $resp) = @_;
2670         my $data = $resp->{'return'} || [];
2671         my $totalrdbytes = 0;
2672         my $totalwrbytes = 0;
2673
2674         for my $blockstat (@$data) {
2675             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2676             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2677
2678             $blockstat->{device} =~ s/drive-//;
2679             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2680         }
2681         $res->{$vmid}->{diskread} = $totalrdbytes;
2682         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2683     };
2684
2685     my $statuscb = sub {
2686         my ($vmid, $resp) = @_;
2687
2688         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2689         # this fails if ballon driver is not loaded, so this must be
2690         # the last commnand (following command are aborted if this fails).
2691         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2692
2693         my $status = 'unknown';
2694         if (!defined($status = $resp->{'return'}->{status})) {
2695             warn "unable to get VM status\n";
2696             return;
2697         }
2698
2699         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2700     };
2701
2702     foreach my $vmid (keys %$list) {
2703         next if $opt_vmid && ($vmid ne $opt_vmid);
2704         next if !$res->{$vmid}->{pid}; # not running
2705         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2706     }
2707
2708     $qmpclient->queue_execute(undef, 1);
2709
2710     foreach my $vmid (keys %$list) {
2711         next if $opt_vmid && ($vmid ne $opt_vmid);
2712         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2713     }
2714
2715     return $res;
2716 }
2717
2718 sub foreach_drive {
2719     my ($conf, $func, @param) = @_;
2720
2721     foreach my $ds (valid_drive_names()) {
2722         next if !defined($conf->{$ds});
2723
2724         my $drive = parse_drive($ds, $conf->{$ds});
2725         next if !$drive;
2726
2727         &$func($ds, $drive, @param);
2728     }
2729 }
2730
2731 sub foreach_volid {
2732     my ($conf, $func, @param) = @_;
2733
2734     my $volhash = {};
2735
2736     my $test_volid = sub {
2737         my ($volid, $is_cdrom) = @_;
2738
2739         return if !$volid;
2740
2741         $volhash->{$volid} = $is_cdrom || 0;
2742     };
2743
2744     foreach_drive($conf, sub {
2745         my ($ds, $drive) = @_;
2746         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2747     });
2748
2749     foreach my $snapname (keys %{$conf->{snapshots}}) {
2750         my $snap = $conf->{snapshots}->{$snapname};
2751         &$test_volid($snap->{vmstate}, 0);
2752         foreach_drive($snap, sub {
2753             my ($ds, $drive) = @_;
2754             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2755         });
2756     }
2757
2758     foreach my $volid (keys %$volhash) {
2759         &$func($volid, $volhash->{$volid}, @param);
2760     }
2761 }
2762
2763 sub vga_conf_has_spice {
2764     my ($vga) = @_;
2765
2766     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2767
2768     return $1 || 1;
2769 }
2770
2771 sub config_to_command {
2772     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2773
2774     my $cmd = [];
2775     my $globalFlags = [];
2776     my $machineFlags = [];
2777     my $rtcFlags = [];
2778     my $cpuFlags = [];
2779     my $devices = [];
2780     my $pciaddr = '';
2781     my $bridges = {};
2782     my $kvmver = kvm_user_version();
2783     my $vernum = 0; # unknown
2784     my $ostype = $conf->{ostype};
2785     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2786         $vernum = $1*1000000+$2*1000;
2787     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2788         $vernum = $1*1000000+$2*1000+$3;
2789     }
2790
2791     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2792
2793     my $have_ovz = -f '/proc/vz/vestat';
2794
2795     my $q35 = machine_type_is_q35($conf);
2796     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2797     my $machine_type = $forcemachine || $conf->{machine};
2798     my $use_old_bios_files = undef;
2799     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2800
2801     my $cpuunits = defined($conf->{cpuunits}) ?
2802             $conf->{cpuunits} : $defaults->{cpuunits};
2803
2804     push @$cmd, '/usr/bin/kvm';
2805
2806     push @$cmd, '-id', $vmid;
2807
2808     my $use_virtio = 0;
2809
2810     my $qmpsocket = qmp_socket($vmid);
2811     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2812     push @$cmd, '-mon', "chardev=qmp,mode=control";
2813
2814
2815     push @$cmd, '-pidfile' , pidfile_name($vmid);
2816
2817     push @$cmd, '-daemonize';
2818
2819     if ($conf->{smbios1}) {
2820         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2821     }
2822
2823     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2824         my $ovmfbase;
2825
2826         # prefer the OVMF_CODE variant
2827         if (-f $OVMF_CODE) {
2828             $ovmfbase = $OVMF_CODE;
2829         } elsif (-f $OVMF_IMG) {
2830             $ovmfbase = $OVMF_IMG;
2831         }
2832
2833         die "no uefi base img found\n" if !$ovmfbase;
2834         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2835
2836         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2837             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2838             my $format = $d->{format} // 'raw';
2839             my $path;
2840             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2841             if ($storeid) {
2842                 $path = PVE::Storage::path($storecfg, $d->{file});
2843                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2844                 $format = qemu_img_format($scfg, $volname);
2845             } else {
2846                 $path = $d->{file};
2847                 $format = "raw";
2848             }
2849             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2850         } elsif ($ovmfbase eq $OVMF_CODE) {
2851             warn "using uefi without permanent efivars disk\n";
2852             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2853             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2854             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2855         } else {
2856             # if the base img is not OVMF_CODE, we do not have to bother
2857             # to create/use a vars image, since it will not be used anyway
2858             # this can only happen if someone manually deletes the OVMF_CODE image
2859             # or has an old pve-qemu-kvm version installed.
2860             # both should not happen, but we ignore it here
2861         }
2862     }
2863
2864
2865     # add usb controllers
2866     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2867     push @$devices, @usbcontrollers if @usbcontrollers;
2868     my $vga = $conf->{vga};
2869
2870     my $qxlnum = vga_conf_has_spice($vga);
2871     $vga = 'qxl' if $qxlnum;
2872
2873     if (!$vga) {
2874         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2875                                 $conf->{ostype} eq 'win7' ||
2876                                 $conf->{ostype} eq 'w2k8')) {
2877             $vga = 'std';
2878         } else {
2879             $vga = 'cirrus';
2880         }
2881     }
2882
2883     # enable absolute mouse coordinates (needed by vnc)
2884     my $tablet;
2885     if (defined($conf->{tablet})) {
2886         $tablet = $conf->{tablet};
2887     } else {
2888         $tablet = $defaults->{tablet};
2889         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2890         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2891     }
2892
2893     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2894
2895     my $kvm_off = 0;
2896     # host pci devices
2897     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2898         my $d = parse_hostpci($conf->{"hostpci$i"});
2899         next if !$d;
2900
2901         my $pcie = $d->{pcie};
2902         if($pcie){
2903             die "q35 machine model is not enabled" if !$q35;
2904             $pciaddr = print_pcie_addr("hostpci$i");
2905         }else{
2906             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2907         }
2908
2909         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2910         my $xvga = '';
2911         if ($d->{'x-vga'}) {
2912             $xvga = ',x-vga=on';
2913             $kvm_off = 1;
2914             $vga = 'none';
2915             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2916                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2917             }
2918             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2919                 $xvga = "";
2920             }
2921         }
2922         my $pcidevices = $d->{pciid};
2923         my $multifunction = 1 if @$pcidevices > 1;
2924
2925         my $j=0;
2926         foreach my $pcidevice (@$pcidevices) {
2927
2928             my $id = "hostpci$i";
2929             $id .= ".$j" if $multifunction;
2930             my $addr = $pciaddr;
2931             $addr .= ".$j" if $multifunction;
2932             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2933
2934             if($j == 0){
2935                 $devicestr .= "$rombar$xvga";
2936                 $devicestr .= ",multifunction=on" if $multifunction;
2937             }
2938
2939             push @$devices, '-device', $devicestr;
2940             $j++;
2941         }
2942     }
2943
2944     # usb devices
2945     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2946     push @$devices, @usbdevices if @usbdevices;
2947     # serial devices
2948     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2949         if (my $path = $conf->{"serial$i"}) {
2950             if ($path eq 'socket') {
2951                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2952                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2953                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2954             } else {
2955                 die "no such serial device\n" if ! -c $path;
2956                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2957                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2958             }
2959         }
2960     }
2961
2962     # parallel devices
2963     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2964         if (my $path = $conf->{"parallel$i"}) {
2965             die "no such parallel device\n" if ! -c $path;
2966             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2967             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2968             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2969         }
2970     }
2971
2972     my $vmname = $conf->{name} || "vm$vmid";
2973
2974     push @$cmd, '-name', $vmname;
2975
2976     my $sockets = 1;
2977     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2978     $sockets = $conf->{sockets} if  $conf->{sockets};
2979
2980     my $cores = $conf->{cores} || 1;
2981
2982     my $maxcpus = $sockets * $cores;
2983
2984     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2985
2986     my $allowed_vcpus = $cpuinfo->{cpus};
2987
2988     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2989         if ($allowed_vcpus < $maxcpus);
2990
2991     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
2992
2993         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2994         for (my $i = 2; $i <= $vcpus; $i++)  {
2995             my $cpustr = print_cpu_device($conf,$i);
2996             push @$cmd, '-device', $cpustr;
2997         }
2998
2999     } else {
3000
3001         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3002     }
3003     push @$cmd, '-nodefaults';
3004
3005     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3006
3007     my $bootindex_hash = {};
3008     my $i = 1;
3009     foreach my $o (split(//, $bootorder)) {
3010         $bootindex_hash->{$o} = $i*100;
3011         $i++;
3012     }
3013
3014     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3015
3016     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3017
3018     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3019
3020     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3021
3022     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3023         my $socket = vnc_socket($vmid);
3024         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3025     } else {
3026         push @$cmd, '-nographic';
3027     }
3028
3029     # time drift fix
3030     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3031
3032     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3033     my $useLocaltime = $conf->{localtime};
3034
3035     if ($ostype) {
3036         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3037
3038         if ($ostype =~ m/^w/) { # windows
3039             $useLocaltime = 1 if !defined($conf->{localtime});
3040
3041             # use time drift fix when acpi is enabled
3042             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3043                 $tdf = 1 if !defined($conf->{tdf});
3044             }
3045         }
3046
3047         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3048             $ostype eq 'wvista') {
3049             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3050             push @$cmd, '-no-hpet';
3051             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3052                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3053                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3054                 push @$cpuFlags , 'hv_time' if !$nokvm;
3055
3056                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
3057                     push @$cpuFlags , 'hv_reset' if !$nokvm;
3058                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
3059                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
3060                 }
3061
3062             } else {
3063                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3064             }
3065         }
3066
3067         if ($ostype eq 'win7' || $ostype eq 'win8') {
3068             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3069         }
3070     }
3071
3072     push @$rtcFlags, 'driftfix=slew' if $tdf;
3073
3074     if ($nokvm) {
3075         push @$machineFlags, 'accel=tcg';
3076     } else {
3077         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3078     }
3079
3080     if ($machine_type) {
3081         push @$machineFlags, "type=${machine_type}";
3082     }
3083
3084     if ($conf->{startdate}) {
3085         push @$rtcFlags, "base=$conf->{startdate}";
3086     } elsif ($useLocaltime) {
3087         push @$rtcFlags, 'base=localtime';
3088     }
3089
3090     my $cpu = $nokvm ? "qemu64" : "kvm64";
3091     if (my $cputype = $conf->{cpu}) {
3092         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3093             or die "Cannot parse cpu description: $cputype\n";
3094         $cpu = $cpuconf->{cputype};
3095         $kvm_off = 1 if $cpuconf->{hidden};
3096     }
3097
3098     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3099
3100     push @$cpuFlags , '-x2apic'
3101         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3102
3103     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3104
3105     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3106
3107     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3108
3109         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3110         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3111     }
3112
3113     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3114
3115     push @$cpuFlags, 'kvm=off' if $kvm_off;
3116
3117     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3118         die "internal error"; # should not happen
3119
3120     push @$cpuFlags, "vendor=${cpu_vendor}"
3121         if $cpu_vendor ne 'default';
3122
3123     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3124
3125     push @$cmd, '-cpu', $cpu;
3126
3127     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3128
3129     push @$cmd, '-S' if $conf->{freeze};
3130
3131     # set keyboard layout
3132     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3133     push @$cmd, '-k', $kb if $kb;
3134
3135     # enable sound
3136     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3137     #push @$cmd, '-soundhw', 'es1370';
3138     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3139
3140     if($conf->{agent}) {
3141         my $qgasocket = qmp_socket($vmid, 1);
3142         my $pciaddr = print_pci_addr("qga0", $bridges);
3143         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3144         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3145         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3146     }
3147
3148     my $spice_port;
3149
3150     if ($qxlnum) {
3151         if ($qxlnum > 1) {
3152             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3153                 for(my $i = 1; $i < $qxlnum; $i++){
3154                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3155                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3156                 }
3157             } else {
3158                 # assume other OS works like Linux
3159                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3160                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3161             }
3162         }
3163
3164         my $pciaddr = print_pci_addr("spice", $bridges);
3165
3166         my $nodename = PVE::INotify::nodename();
3167         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3168         $spice_port = PVE::Tools::next_spice_port($pfamily);
3169
3170         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3171
3172         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3173         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3174         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3175     }
3176
3177     # enable balloon by default, unless explicitly disabled
3178     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3179         $pciaddr = print_pci_addr("balloon0", $bridges);
3180         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3181     }
3182
3183     if ($conf->{watchdog}) {
3184         my $wdopts = parse_watchdog($conf->{watchdog});
3185         $pciaddr = print_pci_addr("watchdog", $bridges);
3186         my $watchdog = $wdopts->{model} || 'i6300esb';
3187         push @$devices, '-device', "$watchdog$pciaddr";
3188         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3189     }
3190
3191     my $vollist = [];
3192     my $scsicontroller = {};
3193     my $ahcicontroller = {};
3194     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3195
3196     # Add iscsi initiator name if available
3197     if (my $initiator = get_initiator_name()) {
3198         push @$devices, '-iscsi', "initiator-name=$initiator";
3199     }
3200
3201     foreach_drive($conf, sub {
3202         my ($ds, $drive) = @_;
3203
3204         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3205             push @$vollist, $drive->{file};
3206         }
3207
3208         $use_virtio = 1 if $ds =~ m/^virtio/;
3209
3210         if (drive_is_cdrom ($drive)) {
3211             if ($bootindex_hash->{d}) {
3212                 $drive->{bootindex} = $bootindex_hash->{d};
3213                 $bootindex_hash->{d} += 1;
3214             }
3215         } else {
3216             if ($bootindex_hash->{c}) {
3217                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3218                 $bootindex_hash->{c} += 1;
3219             }
3220         }
3221
3222         if($drive->{interface} eq 'virtio'){
3223            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3224         }
3225
3226         if ($drive->{interface} eq 'scsi') {
3227
3228             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3229
3230             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3231             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3232
3233             my $iothread = '';
3234             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3235                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3236                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3237             } elsif ($drive->{iothread}) {
3238                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3239             }
3240
3241             my $queues = '';
3242             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3243                 $queues = ",num_queues=$drive->{queues}";
3244             }
3245
3246             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3247             $scsicontroller->{$controller}=1;
3248         }
3249
3250         if ($drive->{interface} eq 'sata') {
3251            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3252            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3253            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3254            $ahcicontroller->{$controller}=1;
3255         }
3256
3257         if ($drive->{interface} eq 'efidisk') {
3258             # this will be added somewhere else
3259             return;
3260         }
3261
3262         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3263         push @$devices, '-drive',$drive_cmd;
3264         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3265     });
3266
3267     for (my $i = 0; $i < $MAX_NETS; $i++) {
3268          next if !$conf->{"net$i"};
3269          my $d = parse_net($conf->{"net$i"});
3270          next if !$d;
3271
3272          $use_virtio = 1 if $d->{model} eq 'virtio';
3273
3274          if ($bootindex_hash->{n}) {
3275             $d->{bootindex} = $bootindex_hash->{n};
3276             $bootindex_hash->{n} += 1;
3277          }
3278
3279          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3280          push @$devices, '-netdev', $netdevfull;
3281
3282          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3283          push @$devices, '-device', $netdevicefull;
3284     }
3285
3286     if (!$q35) {
3287         # add pci bridges
3288         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3289            $bridges->{1} = 1;
3290            $bridges->{2} = 1;
3291         }
3292
3293         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3294
3295         while (my ($k, $v) = each %$bridges) {
3296             $pciaddr = print_pci_addr("pci.$k");
3297             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3298         }
3299     }
3300
3301     # add custom args
3302     if ($conf->{args}) {
3303         my $aa = PVE::Tools::split_args($conf->{args});
3304         push @$cmd, @$aa;
3305     }
3306
3307     push @$cmd, @$devices;
3308     push @$cmd, '-rtc', join(',', @$rtcFlags)
3309         if scalar(@$rtcFlags);
3310     push @$cmd, '-machine', join(',', @$machineFlags)
3311         if scalar(@$machineFlags);
3312     push @$cmd, '-global', join(',', @$globalFlags)
3313         if scalar(@$globalFlags);
3314
3315     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3316 }
3317
3318 sub vnc_socket {
3319     my ($vmid) = @_;
3320     return "${var_run_tmpdir}/$vmid.vnc";
3321 }
3322
3323 sub spice_port {
3324     my ($vmid) = @_;
3325
3326     my $res = vm_mon_cmd($vmid, 'query-spice');
3327
3328     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3329 }
3330
3331 sub qmp_socket {
3332     my ($vmid, $qga) = @_;
3333     my $sockettype = $qga ? 'qga' : 'qmp';
3334     return "${var_run_tmpdir}/$vmid.$sockettype";
3335 }
3336
3337 sub pidfile_name {
3338     my ($vmid) = @_;
3339     return "${var_run_tmpdir}/$vmid.pid";
3340 }
3341
3342 sub vm_devices_list {
3343     my ($vmid) = @_;
3344
3345     my $res = vm_mon_cmd($vmid, 'query-pci');
3346     my $devices = {};
3347     foreach my $pcibus (@$res) {
3348         foreach my $device (@{$pcibus->{devices}}) {
3349             next if !$device->{'qdev_id'};
3350             if ($device->{'pci_bridge'}) {
3351                 $devices->{$device->{'qdev_id'}} = 1;
3352                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3353                     next if !$bridge_device->{'qdev_id'};
3354                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3355                     $devices->{$device->{'qdev_id'}}++;
3356                 }
3357             } else {
3358                 $devices->{$device->{'qdev_id'}} = 1;
3359             }
3360         }
3361     }
3362
3363     my $resblock = vm_mon_cmd($vmid, 'query-block');
3364     foreach my $block (@$resblock) {
3365         if($block->{device} =~ m/^drive-(\S+)/){
3366                 $devices->{$1} = 1;
3367         }
3368     }
3369
3370     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3371     foreach my $mice (@$resmice) {
3372         if ($mice->{name} eq 'QEMU HID Tablet') {
3373             $devices->{tablet} = 1;
3374             last;
3375         }
3376     }
3377
3378     # for usb devices there is no query-usb
3379     # but we can iterate over the entries in
3380     # qom-list path=/machine/peripheral
3381     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3382     foreach my $per (@$resperipheral) {
3383         if ($per->{name} =~ m/^usb\d+$/) {
3384             $devices->{$per->{name}} = 1;
3385         }
3386     }
3387
3388     return $devices;
3389 }
3390
3391 sub vm_deviceplug {
3392     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3393
3394     my $q35 = machine_type_is_q35($conf);
3395
3396     my $devices_list = vm_devices_list($vmid);
3397     return 1 if defined($devices_list->{$deviceid});
3398
3399     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3400
3401     if ($deviceid eq 'tablet') {
3402
3403         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3404
3405     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3406
3407         die "usb hotplug currently not reliable\n";
3408         # since we can't reliably hot unplug all added usb devices
3409         # and usb passthrough disables live migration
3410         # we disable usb hotplugging for now
3411         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3412
3413     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3414
3415         qemu_iothread_add($vmid, $deviceid, $device);
3416
3417         qemu_driveadd($storecfg, $vmid, $device);
3418         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3419
3420         qemu_deviceadd($vmid, $devicefull);
3421         eval { qemu_deviceaddverify($vmid, $deviceid); };
3422         if (my $err = $@) {
3423             eval { qemu_drivedel($vmid, $deviceid); };
3424             warn $@ if $@;
3425             die $err;
3426         }
3427
3428     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3429
3430
3431         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3432         my $pciaddr = print_pci_addr($deviceid);
3433         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3434
3435         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3436
3437         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3438             qemu_iothread_add($vmid, $deviceid, $device);
3439             $devicefull .= ",iothread=iothread-$deviceid";
3440         }
3441
3442         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3443             $devicefull .= ",num_queues=$device->{queues}";
3444         }
3445
3446         qemu_deviceadd($vmid, $devicefull);
3447         qemu_deviceaddverify($vmid, $deviceid);
3448
3449     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3450
3451         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3452         qemu_driveadd($storecfg, $vmid, $device);
3453
3454         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3455         eval { qemu_deviceadd($vmid, $devicefull); };
3456         if (my $err = $@) {
3457             eval { qemu_drivedel($vmid, $deviceid); };
3458             warn $@ if $@;
3459             die $err;
3460         }
3461
3462     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3463
3464         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3465
3466         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3467         my $use_old_bios_files = undef;
3468         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3469
3470         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3471         qemu_deviceadd($vmid, $netdevicefull);
3472         eval { qemu_deviceaddverify($vmid, $deviceid); };
3473         if (my $err = $@) {
3474             eval { qemu_netdevdel($vmid, $deviceid); };
3475             warn $@ if $@;
3476             die $err;
3477         }
3478
3479     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3480
3481         my $bridgeid = $2;
3482         my $pciaddr = print_pci_addr($deviceid);
3483         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3484
3485         qemu_deviceadd($vmid, $devicefull);
3486         qemu_deviceaddverify($vmid, $deviceid);
3487
3488     } else {
3489         die "can't hotplug device '$deviceid'\n";
3490     }
3491
3492     return 1;
3493 }
3494
3495 # fixme: this should raise exceptions on error!
3496 sub vm_deviceunplug {
3497     my ($vmid, $conf, $deviceid) = @_;
3498
3499     my $devices_list = vm_devices_list($vmid);
3500     return 1 if !defined($devices_list->{$deviceid});
3501
3502     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3503
3504     if ($deviceid eq 'tablet') {
3505
3506         qemu_devicedel($vmid, $deviceid);
3507
3508     } elsif ($deviceid =~ m/^usb\d+$/) {
3509
3510         die "usb hotplug currently not reliable\n";
3511         # when unplugging usb devices this way,
3512         # there may be remaining usb controllers/hubs
3513         # so we disable it for now
3514         qemu_devicedel($vmid, $deviceid);
3515         qemu_devicedelverify($vmid, $deviceid);
3516
3517     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3518
3519         qemu_devicedel($vmid, $deviceid);
3520         qemu_devicedelverify($vmid, $deviceid);
3521         qemu_drivedel($vmid, $deviceid);
3522         qemu_iothread_del($conf, $vmid, $deviceid);
3523
3524     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3525
3526         qemu_devicedel($vmid, $deviceid);
3527         qemu_devicedelverify($vmid, $deviceid);
3528         qemu_iothread_del($conf, $vmid, $deviceid);
3529
3530     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3531
3532         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3533         my $device = parse_drive($deviceid, $conf->{$deviceid});
3534         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3535
3536         qemu_devicedel($vmid, $deviceid);
3537         qemu_drivedel($vmid, $deviceid);
3538         qemu_deletescsihw($conf, $vmid, $deviceid);
3539
3540     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3541
3542         qemu_devicedel($vmid, $deviceid);
3543         qemu_devicedelverify($vmid, $deviceid);
3544         qemu_netdevdel($vmid, $deviceid);
3545
3546     } else {
3547         die "can't unplug device '$deviceid'\n";
3548     }
3549
3550     return 1;
3551 }
3552
3553 sub qemu_deviceadd {
3554     my ($vmid, $devicefull) = @_;
3555
3556     $devicefull = "driver=".$devicefull;
3557     my %options =  split(/[=,]/, $devicefull);
3558
3559     vm_mon_cmd($vmid, "device_add" , %options);
3560 }
3561
3562 sub qemu_devicedel {
3563     my ($vmid, $deviceid) = @_;
3564
3565     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3566 }
3567
3568 sub qemu_iothread_add {
3569     my($vmid, $deviceid, $device) = @_;
3570
3571     if ($device->{iothread}) {
3572         my $iothreads = vm_iothreads_list($vmid);
3573         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3574     }
3575 }
3576
3577 sub qemu_iothread_del {
3578     my($conf, $vmid, $deviceid) = @_;
3579
3580     my $device = parse_drive($deviceid, $conf->{$deviceid});
3581     if ($device->{iothread}) {
3582         my $iothreads = vm_iothreads_list($vmid);
3583         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3584     }
3585 }
3586
3587 sub qemu_objectadd {
3588     my($vmid, $objectid, $qomtype) = @_;
3589
3590     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3591
3592     return 1;
3593 }
3594
3595 sub qemu_objectdel {
3596     my($vmid, $objectid) = @_;
3597
3598     vm_mon_cmd($vmid, "object-del", id => $objectid);
3599
3600     return 1;
3601 }
3602
3603 sub qemu_driveadd {
3604     my ($storecfg, $vmid, $device) = @_;
3605
3606     my $drive = print_drive_full($storecfg, $vmid, $device);
3607     $drive =~ s/\\/\\\\/g;
3608     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3609
3610     # If the command succeeds qemu prints: "OK"
3611     return 1 if $ret =~ m/OK/s;
3612
3613     die "adding drive failed: $ret\n";
3614 }
3615
3616 sub qemu_drivedel {
3617     my($vmid, $deviceid) = @_;
3618
3619     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3620     $ret =~ s/^\s+//;
3621
3622     return 1 if $ret eq "";
3623
3624     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3625     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3626
3627     die "deleting drive $deviceid failed : $ret\n";
3628 }
3629
3630 sub qemu_deviceaddverify {
3631     my ($vmid, $deviceid) = @_;
3632
3633     for (my $i = 0; $i <= 5; $i++) {
3634          my $devices_list = vm_devices_list($vmid);
3635          return 1 if defined($devices_list->{$deviceid});
3636          sleep 1;
3637     }
3638
3639     die "error on hotplug device '$deviceid'\n";
3640 }
3641
3642
3643 sub qemu_devicedelverify {
3644     my ($vmid, $deviceid) = @_;
3645
3646     # need to verify that the device is correctly removed as device_del
3647     # is async and empty return is not reliable
3648
3649     for (my $i = 0; $i <= 5; $i++) {
3650          my $devices_list = vm_devices_list($vmid);
3651          return 1 if !defined($devices_list->{$deviceid});
3652          sleep 1;
3653     }
3654
3655     die "error on hot-unplugging device '$deviceid'\n";
3656 }
3657
3658 sub qemu_findorcreatescsihw {
3659     my ($storecfg, $conf, $vmid, $device) = @_;
3660
3661     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3662
3663     my $scsihwid="$controller_prefix$controller";
3664     my $devices_list = vm_devices_list($vmid);
3665
3666     if(!defined($devices_list->{$scsihwid})) {
3667         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3668     }
3669
3670     return 1;
3671 }
3672
3673 sub qemu_deletescsihw {
3674     my ($conf, $vmid, $opt) = @_;
3675
3676     my $device = parse_drive($opt, $conf->{$opt});
3677
3678     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3679         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3680         return 1;
3681     }
3682
3683     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3684
3685     my $devices_list = vm_devices_list($vmid);
3686     foreach my $opt (keys %{$devices_list}) {
3687         if (PVE::QemuServer::is_valid_drivename($opt)) {
3688             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3689             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3690                 return 1;
3691             }
3692         }
3693     }
3694
3695     my $scsihwid="scsihw$controller";
3696
3697     vm_deviceunplug($vmid, $conf, $scsihwid);
3698
3699     return 1;
3700 }
3701
3702 sub qemu_add_pci_bridge {
3703     my ($storecfg, $conf, $vmid, $device) = @_;
3704
3705     my $bridges = {};
3706
3707     my $bridgeid;
3708
3709     print_pci_addr($device, $bridges);
3710
3711     while (my ($k, $v) = each %$bridges) {
3712         $bridgeid = $k;
3713     }
3714     return 1 if !defined($bridgeid) || $bridgeid < 1;
3715
3716     my $bridge = "pci.$bridgeid";
3717     my $devices_list = vm_devices_list($vmid);
3718
3719     if (!defined($devices_list->{$bridge})) {
3720         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3721     }
3722
3723     return 1;
3724 }
3725
3726 sub qemu_set_link_status {
3727     my ($vmid, $device, $up) = @_;
3728
3729     vm_mon_cmd($vmid, "set_link", name => $device,
3730                up => $up ? JSON::true : JSON::false);
3731 }
3732
3733 sub qemu_netdevadd {
3734     my ($vmid, $conf, $device, $deviceid) = @_;
3735
3736     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3737     my %options =  split(/[=,]/, $netdev);
3738
3739     vm_mon_cmd($vmid, "netdev_add",  %options);
3740     return 1;
3741 }
3742
3743 sub qemu_netdevdel {
3744     my ($vmid, $deviceid) = @_;
3745
3746     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3747 }
3748
3749 sub qemu_usb_hotplug {
3750     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3751
3752     return if !$device;
3753
3754     # remove the old one first
3755     vm_deviceunplug($vmid, $conf, $deviceid);
3756
3757     # check if xhci controller is necessary and available
3758     if ($device->{usb3}) {
3759
3760         my $devicelist = vm_devices_list($vmid);
3761
3762         if (!$devicelist->{xhci}) {
3763             my $pciaddr = print_pci_addr("xhci");
3764             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3765         }
3766     }
3767     my $d = parse_usb_device($device->{host});
3768     $d->{usb3} = $device->{usb3};
3769
3770     # add the new one
3771     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3772 }
3773
3774 sub qemu_cpu_hotplug {
3775     my ($vmid, $conf, $vcpus) = @_;
3776
3777     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3778
3779     my $sockets = 1;
3780     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3781     $sockets = $conf->{sockets} if  $conf->{sockets};
3782     my $cores = $conf->{cores} || 1;
3783     my $maxcpus = $sockets * $cores;
3784
3785     $vcpus = $maxcpus if !$vcpus;
3786
3787     die "you can't add more vcpus than maxcpus\n"
3788         if $vcpus > $maxcpus;
3789
3790     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3791
3792     if ($vcpus < $currentvcpus) {
3793
3794         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3795
3796             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3797                 qemu_devicedel($vmid, "cpu$i");
3798                 my $retry = 0;
3799                 my $currentrunningvcpus = undef;
3800                 while (1) {
3801                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3802                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3803                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3804                     $retry++;
3805                     sleep 1;
3806                 }
3807                 #update conf after each succesfull cpu unplug
3808                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3809                 PVE::QemuConfig->write_config($vmid, $conf);
3810             }
3811         } else {
3812             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3813         }
3814
3815         return;
3816     }
3817
3818     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3819     die "vcpus in running vm does not match its configuration\n"
3820         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3821
3822     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3823
3824         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3825             my $cpustr = print_cpu_device($conf, $i);
3826             qemu_deviceadd($vmid, $cpustr);
3827
3828             my $retry = 0;
3829             my $currentrunningvcpus = undef;
3830             while (1) {
3831                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3832                 last if scalar(@{$currentrunningvcpus}) == $i;
3833                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3834                 sleep 1;
3835                 $retry++;
3836             }
3837             #update conf after each succesfull cpu hotplug
3838             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3839             PVE::QemuConfig->write_config($vmid, $conf);
3840         }
3841     } else {
3842
3843         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3844             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3845         }
3846     }
3847 }
3848
3849 sub qemu_block_set_io_throttle {
3850     my ($vmid, $deviceid,
3851         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3852         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3853         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3854         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3855
3856     return if !check_running($vmid) ;
3857
3858     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3859         bps => int($bps),
3860         bps_rd => int($bps_rd),
3861         bps_wr => int($bps_wr),
3862         iops => int($iops),
3863         iops_rd => int($iops_rd),
3864         iops_wr => int($iops_wr),
3865         bps_max => int($bps_max),
3866         bps_rd_max => int($bps_rd_max),
3867         bps_wr_max => int($bps_wr_max),
3868         iops_max => int($iops_max),
3869         iops_rd_max => int($iops_rd_max),
3870         iops_wr_max => int($iops_wr_max),
3871         bps_max_length => int($bps_max_length),
3872         bps_rd_max_length => int($bps_rd_max_length),
3873         bps_wr_max_length => int($bps_wr_max_length),
3874         iops_max_length => int($iops_max_length),
3875         iops_rd_max_length => int($iops_rd_max_length),
3876         iops_wr_max_length => int($iops_wr_max_length),
3877     );
3878
3879 }
3880
3881 # old code, only used to shutdown old VM after update
3882 sub __read_avail {
3883     my ($fh, $timeout) = @_;
3884
3885     my $sel = new IO::Select;
3886     $sel->add($fh);
3887
3888     my $res = '';
3889     my $buf;
3890
3891     my @ready;
3892     while (scalar (@ready = $sel->can_read($timeout))) {
3893         my $count;
3894         if ($count = $fh->sysread($buf, 8192)) {
3895             if ($buf =~ /^(.*)\(qemu\) $/s) {
3896                 $res .= $1;
3897                 last;
3898             } else {
3899                 $res .= $buf;
3900             }
3901         } else {
3902             if (!defined($count)) {
3903                 die "$!\n";
3904             }
3905             last;
3906         }
3907     }
3908
3909     die "monitor read timeout\n" if !scalar(@ready);
3910
3911     return $res;
3912 }
3913
3914 # old code, only used to shutdown old VM after update
3915 sub vm_monitor_command {
3916     my ($vmid, $cmdstr, $nocheck) = @_;
3917
3918     my $res;
3919
3920     eval {
3921         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3922
3923         my $sname = "${var_run_tmpdir}/$vmid.mon";
3924
3925         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3926             die "unable to connect to VM $vmid socket - $!\n";
3927
3928         my $timeout = 3;
3929
3930         # hack: migrate sometime blocks the monitor (when migrate_downtime
3931         # is set)
3932         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3933             $timeout = 60*60; # 1 hour
3934         }
3935
3936         # read banner;
3937         my $data = __read_avail($sock, $timeout);
3938
3939         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3940             die "got unexpected qemu monitor banner\n";
3941         }
3942
3943         my $sel = new IO::Select;
3944         $sel->add($sock);
3945
3946         if (!scalar(my @ready = $sel->can_write($timeout))) {
3947             die "monitor write error - timeout";
3948         }
3949
3950         my $fullcmd = "$cmdstr\r";
3951
3952         # syslog('info', "VM $vmid monitor command: $cmdstr");
3953
3954         my $b;
3955         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3956             die "monitor write error - $!";
3957         }
3958
3959         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3960
3961         $timeout = 20;
3962
3963         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3964             $timeout = 60*60; # 1 hour
3965         } elsif ($cmdstr =~ m/^(eject|change)/) {
3966             $timeout = 60; # note: cdrom mount command is slow
3967         }
3968         if ($res = __read_avail($sock, $timeout)) {
3969
3970             my @lines = split("\r?\n", $res);
3971
3972             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3973
3974             $res = join("\n", @lines);
3975             $res .= "\n";
3976         }
3977     };
3978
3979     my $err = $@;
3980
3981     if ($err) {
3982         syslog("err", "VM $vmid monitor command failed - $err");
3983         die $err;
3984     }
3985
3986     return $res;
3987 }
3988
3989 sub qemu_block_resize {
3990     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3991
3992     my $running = check_running($vmid);
3993
3994     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3995
3996     return if !$running;
3997
3998     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3999
4000 }
4001
4002 sub qemu_volume_snapshot {
4003     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4004
4005     my $running = check_running($vmid);
4006
4007     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4008         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4009     } else {
4010         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4011     }
4012 }
4013
4014 sub qemu_volume_snapshot_delete {
4015     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4016
4017     my $running = check_running($vmid);
4018
4019     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4020
4021     return if !$running;
4022
4023     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4024 }
4025
4026 sub set_migration_caps {
4027     my ($vmid) = @_;
4028
4029     my $cap_ref = [];
4030
4031     my $enabled_cap = {
4032         "auto-converge" => 1,
4033         "xbzrle" => 1,
4034         "x-rdma-pin-all" => 0,
4035         "zero-blocks" => 0,
4036         "compress" => 0
4037     };
4038
4039     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4040
4041     for my $supported_capability (@$supported_capabilities) {
4042         push @$cap_ref, {
4043             capability => $supported_capability->{capability},
4044             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4045         };
4046     }
4047
4048     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4049 }
4050
4051 my $fast_plug_option = {
4052     'lock' => 1,
4053     'name' => 1,
4054     'onboot' => 1,
4055     'shares' => 1,
4056     'startup' => 1,
4057     'description' => 1,
4058     'protection' => 1,
4059 };
4060
4061 # hotplug changes in [PENDING]
4062 # $selection hash can be used to only apply specified options, for
4063 # example: { cores => 1 } (only apply changed 'cores')
4064 # $errors ref is used to return error messages
4065 sub vmconfig_hotplug_pending {
4066     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4067
4068     my $defaults = load_defaults();
4069
4070     # commit values which do not have any impact on running VM first
4071     # Note: those option cannot raise errors, we we do not care about
4072     # $selection and always apply them.
4073
4074     my $add_error = sub {
4075         my ($opt, $msg) = @_;
4076         $errors->{$opt} = "hotplug problem - $msg";
4077     };
4078
4079     my $changes = 0;
4080     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4081         if ($fast_plug_option->{$opt}) {
4082             $conf->{$opt} = $conf->{pending}->{$opt};
4083             delete $conf->{pending}->{$opt};
4084             $changes = 1;
4085         }
4086     }
4087
4088     if ($changes) {
4089         PVE::QemuConfig->write_config($vmid, $conf);
4090         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4091     }
4092
4093     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4094
4095     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4096     while (my ($opt, $force) = each %$pending_delete_hash) {
4097         next if $selection && !$selection->{$opt};
4098         eval {
4099             if ($opt eq 'hotplug') {
4100                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4101             } elsif ($opt eq 'tablet') {
4102                 die "skip\n" if !$hotplug_features->{usb};
4103                 if ($defaults->{tablet}) {
4104                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4105                 } else {
4106                     vm_deviceunplug($vmid, $conf, $opt);
4107                 }
4108             } elsif ($opt =~ m/^usb\d+/) {
4109                 die "skip\n";
4110                 # since we cannot reliably hot unplug usb devices
4111                 # we are disabling it
4112                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4113                 vm_deviceunplug($vmid, $conf, $opt);
4114             } elsif ($opt eq 'vcpus') {
4115                 die "skip\n" if !$hotplug_features->{cpu};
4116                 qemu_cpu_hotplug($vmid, $conf, undef);
4117             } elsif ($opt eq 'balloon') {
4118                 # enable balloon device is not hotpluggable
4119                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4120             } elsif ($fast_plug_option->{$opt}) {
4121                 # do nothing
4122             } elsif ($opt =~ m/^net(\d+)$/) {
4123                 die "skip\n" if !$hotplug_features->{network};
4124                 vm_deviceunplug($vmid, $conf, $opt);
4125             } elsif (is_valid_drivename($opt)) {
4126                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4127                 vm_deviceunplug($vmid, $conf, $opt);
4128                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4129             } elsif ($opt =~ m/^memory$/) {
4130                 die "skip\n" if !$hotplug_features->{memory};
4131                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4132             } elsif ($opt eq 'cpuunits') {
4133                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4134             } elsif ($opt eq 'cpulimit') {
4135                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4136             } else {
4137                 die "skip\n";
4138             }
4139         };
4140         if (my $err = $@) {
4141             &$add_error($opt, $err) if $err ne "skip\n";
4142         } else {
4143             # save new config if hotplug was successful
4144             delete $conf->{$opt};
4145             vmconfig_undelete_pending_option($conf, $opt);
4146             PVE::QemuConfig->write_config($vmid, $conf);
4147             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4148         }
4149     }
4150
4151     foreach my $opt (keys %{$conf->{pending}}) {
4152         next if $selection && !$selection->{$opt};
4153         my $value = $conf->{pending}->{$opt};
4154         eval {
4155             if ($opt eq 'hotplug') {
4156                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4157             } elsif ($opt eq 'tablet') {
4158                 die "skip\n" if !$hotplug_features->{usb};
4159                 if ($value == 1) {
4160                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4161                 } elsif ($value == 0) {
4162                     vm_deviceunplug($vmid, $conf, $opt);
4163                 }
4164             } elsif ($opt =~ m/^usb\d+$/) {
4165                 die "skip\n";
4166                 # since we cannot reliably hot unplug usb devices
4167                 # we are disabling it
4168                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4169                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4170                 die "skip\n" if !$d;
4171                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4172             } elsif ($opt eq 'vcpus') {
4173                 die "skip\n" if !$hotplug_features->{cpu};
4174                 qemu_cpu_hotplug($vmid, $conf, $value);
4175             } elsif ($opt eq 'balloon') {
4176                 # enable/disable balloning device is not hotpluggable
4177                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4178                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4179                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4180
4181                 # allow manual ballooning if shares is set to zero
4182                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4183                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4184                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4185                 }
4186             } elsif ($opt =~ m/^net(\d+)$/) {
4187                 # some changes can be done without hotplug
4188                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4189                                     $vmid, $opt, $value);
4190             } elsif (is_valid_drivename($opt)) {
4191                 # some changes can be done without hotplug
4192                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4193                                      $vmid, $opt, $value, 1);
4194             } elsif ($opt =~ m/^memory$/) { #dimms
4195                 die "skip\n" if !$hotplug_features->{memory};
4196                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4197             } elsif ($opt eq 'cpuunits') {
4198                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4199             } elsif ($opt eq 'cpulimit') {
4200                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4201                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4202             } else {
4203                 die "skip\n";  # skip non-hot-pluggable options
4204             }
4205         };
4206         if (my $err = $@) {
4207             &$add_error($opt, $err) if $err ne "skip\n";
4208         } else {
4209             # save new config if hotplug was successful
4210             $conf->{$opt} = $value;
4211             delete $conf->{pending}->{$opt};
4212             PVE::QemuConfig->write_config($vmid, $conf);
4213             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4214         }
4215     }
4216 }
4217
4218 sub try_deallocate_drive {
4219     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4220
4221     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4222         my $volid = $drive->{file};
4223         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4224             my $sid = PVE::Storage::parse_volume_id($volid);
4225             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4226
4227             # check if the disk is really unused
4228             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4229                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4230             PVE::Storage::vdisk_free($storecfg, $volid);
4231             return 1;
4232         } else {
4233             # If vm is not owner of this disk remove from config
4234             return 1;
4235         }
4236     }
4237
4238     return undef;
4239 }
4240
4241 sub vmconfig_delete_or_detach_drive {
4242     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4243
4244     my $drive = parse_drive($opt, $conf->{$opt});
4245
4246     my $rpcenv = PVE::RPCEnvironment::get();
4247     my $authuser = $rpcenv->get_user();
4248
4249     if ($force) {
4250         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4251         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4252     } else {
4253         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4254     }
4255 }
4256
4257 sub vmconfig_apply_pending {
4258     my ($vmid, $conf, $storecfg) = @_;
4259
4260     # cold plug
4261
4262     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4263     while (my ($opt, $force) = each %$pending_delete_hash) {
4264         die "internal error" if $opt =~ m/^unused/;
4265         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4266         if (!defined($conf->{$opt})) {
4267             vmconfig_undelete_pending_option($conf, $opt);
4268             PVE::QemuConfig->write_config($vmid, $conf);
4269         } elsif (is_valid_drivename($opt)) {
4270             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4271             vmconfig_undelete_pending_option($conf, $opt);
4272             delete $conf->{$opt};
4273             PVE::QemuConfig->write_config($vmid, $conf);
4274         } else {
4275             vmconfig_undelete_pending_option($conf, $opt);
4276             delete $conf->{$opt};
4277             PVE::QemuConfig->write_config($vmid, $conf);
4278         }
4279     }
4280
4281     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4282
4283     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4284         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4285
4286         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4287             # skip if nothing changed
4288         } elsif (is_valid_drivename($opt)) {
4289             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4290                 if defined($conf->{$opt});
4291             $conf->{$opt} = $conf->{pending}->{$opt};
4292         } else {
4293             $conf->{$opt} = $conf->{pending}->{$opt};
4294         }
4295
4296         delete $conf->{pending}->{$opt};
4297         PVE::QemuConfig->write_config($vmid, $conf);
4298     }
4299 }
4300
4301 my $safe_num_ne = sub {
4302     my ($a, $b) = @_;
4303
4304     return 0 if !defined($a) && !defined($b);
4305     return 1 if !defined($a);
4306     return 1 if !defined($b);
4307
4308     return $a != $b;
4309 };
4310
4311 my $safe_string_ne = sub {
4312     my ($a, $b) = @_;
4313
4314     return 0 if !defined($a) && !defined($b);
4315     return 1 if !defined($a);
4316     return 1 if !defined($b);
4317
4318     return $a ne $b;
4319 };
4320
4321 sub vmconfig_update_net {
4322     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4323
4324     my $newnet = parse_net($value);
4325
4326     if ($conf->{$opt}) {
4327         my $oldnet = parse_net($conf->{$opt});
4328
4329         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4330             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4331             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4332             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4333
4334             # for non online change, we try to hot-unplug
4335             die "skip\n" if !$hotplug;
4336             vm_deviceunplug($vmid, $conf, $opt);
4337         } else {
4338
4339             die "internal error" if $opt !~ m/net(\d+)/;
4340             my $iface = "tap${vmid}i$1";
4341
4342             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4343                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4344                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4345                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4346                 PVE::Network::tap_unplug($iface);
4347                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4348             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4349                 # Rate can be applied on its own but any change above needs to
4350                 # include the rate in tap_plug since OVS resets everything.
4351                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4352             }
4353
4354             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4355                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4356             }
4357
4358             return 1;
4359         }
4360     }
4361
4362     if ($hotplug) {
4363         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4364     } else {
4365         die "skip\n";
4366     }
4367 }
4368
4369 sub vmconfig_update_disk {
4370     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4371
4372     # fixme: do we need force?
4373
4374     my $drive = parse_drive($opt, $value);
4375
4376     if ($conf->{$opt}) {
4377
4378         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4379
4380             my $media = $drive->{media} || 'disk';
4381             my $oldmedia = $old_drive->{media} || 'disk';
4382             die "unable to change media type\n" if $media ne $oldmedia;
4383
4384             if (!drive_is_cdrom($old_drive)) {
4385
4386                 if ($drive->{file} ne $old_drive->{file}) {
4387
4388                     die "skip\n" if !$hotplug;
4389
4390                     # unplug and register as unused
4391                     vm_deviceunplug($vmid, $conf, $opt);
4392                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4393
4394                 } else {
4395                     # update existing disk
4396
4397                     # skip non hotpluggable value
4398                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4399                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4400                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4401                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4402                         die "skip\n";
4403                     }
4404
4405                     # apply throttle
4406                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4407                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4408                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4409                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4410                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4411                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4412                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4413                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4414                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4415                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4416                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4417                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4418                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4419                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4420                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4421                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4422                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4423                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4424
4425                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4426                                                    ($drive->{mbps} || 0)*1024*1024,
4427                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4428                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4429                                                    $drive->{iops} || 0,
4430                                                    $drive->{iops_rd} || 0,
4431                                                    $drive->{iops_wr} || 0,
4432                                                    ($drive->{mbps_max} || 0)*1024*1024,
4433                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4434                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4435                                                    $drive->{iops_max} || 0,
4436                                                    $drive->{iops_rd_max} || 0,
4437                                                    $drive->{iops_wr_max} || 0,
4438                                                    $drive->{bps_max_length} || 1,
4439                                                    $drive->{bps_rd_max_length} || 1,
4440                                                    $drive->{bps_wr_max_length} || 1,
4441                                                    $drive->{iops_max_length} || 1,
4442                                                    $drive->{iops_rd_max_length} || 1,
4443                                                    $drive->{iops_wr_max_length} || 1);
4444
4445                     }
4446
4447                     return 1;
4448                 }
4449
4450             } else { # cdrom
4451
4452                 if ($drive->{file} eq 'none') {
4453                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4454                 } else {
4455                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4456                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4457                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4458                 }
4459
4460                 return 1;
4461             }
4462         }
4463     }
4464
4465     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4466     # hotplug new disks
4467     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4468     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4469 }
4470
4471 sub vm_start {
4472     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4473         $forcemachine, $spice_ticket, $migration_network, $migration_type) = @_;
4474
4475     PVE::QemuConfig->lock_config($vmid, sub {
4476         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4477
4478         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4479
4480         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4481
4482         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4483
4484         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4485             vmconfig_apply_pending($vmid, $conf, $storecfg);
4486             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4487         }
4488
4489         my $defaults = load_defaults();
4490
4491         # set environment variable useful inside network script
4492         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4493
4494         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4495
4496         my $migrate_port = 0;
4497         my $migrate_uri;
4498         if ($statefile) {
4499             if ($statefile eq 'tcp') {
4500                 my $localip = "localhost";
4501                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4502                 my $nodename = PVE::INotify::nodename();
4503
4504                 if ($migration_type eq 'insecure') {
4505                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4506                     if ($migrate_network_addr) {
4507                         $localip = $migrate_network_addr;
4508                     } else {
4509                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4510                     }
4511
4512                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4513                 }
4514
4515                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4516                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4517                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4518                 push @$cmd, '-incoming', $migrate_uri;
4519                 push @$cmd, '-S';
4520
4521             } elsif ($statefile eq 'unix') {
4522                 # should be default for secure migrations as a ssh TCP forward
4523                 # tunnel is not deterministic reliable ready and fails regurarly
4524                 # to set up in time, so use UNIX socket forwards
4525                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4526                 unlink $socket_addr;
4527
4528                 $migrate_uri = "unix:$socket_addr";
4529
4530                 push @$cmd, '-incoming', $migrate_uri;
4531                 push @$cmd, '-S';
4532
4533             } else {
4534                 push @$cmd, '-loadstate', $statefile;
4535             }
4536         } elsif ($paused) {
4537             push @$cmd, '-S';
4538         }
4539
4540         # host pci devices
4541         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4542           my $d = parse_hostpci($conf->{"hostpci$i"});
4543           next if !$d;
4544           my $pcidevices = $d->{pciid};
4545           foreach my $pcidevice (@$pcidevices) {
4546                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4547
4548                 my $info = pci_device_info("0000:$pciid");
4549                 die "IOMMU not present\n" if !check_iommu_support();
4550                 die "no pci device info for device '$pciid'\n" if !$info;
4551                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4552                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4553           }
4554         }
4555
4556         PVE::Storage::activate_volumes($storecfg, $vollist);
4557
4558         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4559             my $cmd = [];
4560             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4561             eval  { run_command($cmd); };
4562         }
4563
4564         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4565                                                   : $defaults->{cpuunits};
4566
4567         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4568
4569         my %properties = (
4570             Slice => 'qemu.slice',
4571             KillMode => 'none',
4572             CPUShares => $cpuunits
4573         );
4574
4575         if (my $cpulimit = $conf->{cpulimit}) {
4576             $properties{CPUQuota} = int($cpulimit * 100);
4577         }
4578         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4579
4580         if ($conf->{hugepages}) {
4581
4582             my $code = sub {
4583                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4584                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4585
4586                 PVE::QemuServer::Memory::hugepages_mount();
4587                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4588
4589                 eval  {
4590                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4591                     run_command($cmd, %run_params);
4592                 };
4593
4594                 if (my $err = $@) {
4595                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4596                     die $err;
4597                 }
4598
4599                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4600             };
4601             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4602
4603         } else {
4604             eval  {
4605                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4606                 run_command($cmd, %run_params);
4607             };
4608         }
4609
4610         if (my $err = $@) {
4611             # deactivate volumes if start fails
4612             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4613             die "start failed: $err";
4614         }
4615
4616         print "migration listens on $migrate_uri\n" if $migrate_uri;
4617
4618         if ($statefile && $statefile ne 'tcp')  {
4619             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4620             warn $@ if $@;
4621         }
4622
4623         if ($migratedfrom) {
4624
4625             eval {
4626                 set_migration_caps($vmid);
4627             };
4628             warn $@ if $@;
4629
4630             if ($spice_port) {
4631                 print "spice listens on port $spice_port\n";
4632                 if ($spice_ticket) {
4633                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4634                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4635                 }
4636             }
4637
4638         } else {
4639
4640             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4641                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4642                     if $conf->{balloon};
4643             }
4644
4645             foreach my $opt (keys %$conf) {
4646                 next if $opt !~  m/^net\d+$/;
4647                 my $nicconf = parse_net($conf->{$opt});
4648                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4649             }
4650         }
4651
4652         vm_mon_cmd_nocheck($vmid, 'qom-set',
4653                     path => "machine/peripheral/balloon0",
4654                     property => "guest-stats-polling-interval",
4655                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4656
4657     });
4658 }
4659
4660 sub vm_mon_cmd {
4661     my ($vmid, $execute, %params) = @_;
4662
4663     my $cmd = { execute => $execute, arguments => \%params };
4664     vm_qmp_command($vmid, $cmd);
4665 }
4666
4667 sub vm_mon_cmd_nocheck {
4668     my ($vmid, $execute, %params) = @_;
4669
4670     my $cmd = { execute => $execute, arguments => \%params };
4671     vm_qmp_command($vmid, $cmd, 1);
4672 }
4673
4674 sub vm_qmp_command {
4675     my ($vmid, $cmd, $nocheck) = @_;
4676
4677     my $res;
4678
4679     my $timeout;
4680     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4681         $timeout = $cmd->{arguments}->{timeout};
4682         delete $cmd->{arguments}->{timeout};
4683     }
4684
4685     eval {
4686         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4687         my $sname = qmp_socket($vmid);
4688         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4689             my $qmpclient = PVE::QMPClient->new();
4690
4691             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4692         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4693             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4694                 if scalar(%{$cmd->{arguments}});
4695             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4696         } else {
4697             die "unable to open monitor socket\n";
4698         }
4699     };
4700     if (my $err = $@) {
4701         syslog("err", "VM $vmid qmp command failed - $err");
4702         die $err;
4703     }
4704
4705     return $res;
4706 }
4707
4708 sub vm_human_monitor_command {
4709     my ($vmid, $cmdline) = @_;
4710
4711     my $res;
4712
4713     my $cmd = {
4714         execute => 'human-monitor-command',
4715         arguments => { 'command-line' => $cmdline},
4716     };
4717
4718     return vm_qmp_command($vmid, $cmd);
4719 }
4720
4721 sub vm_commandline {
4722     my ($storecfg, $vmid) = @_;
4723
4724     my $conf = PVE::QemuConfig->load_config($vmid);
4725
4726     my $defaults = load_defaults();
4727
4728     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4729
4730     return PVE::Tools::cmd2string($cmd);
4731 }
4732
4733 sub vm_reset {
4734     my ($vmid, $skiplock) = @_;
4735
4736     PVE::QemuConfig->lock_config($vmid, sub {
4737
4738         my $conf = PVE::QemuConfig->load_config($vmid);
4739
4740         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4741
4742         vm_mon_cmd($vmid, "system_reset");
4743     });
4744 }
4745
4746 sub get_vm_volumes {
4747     my ($conf) = @_;
4748
4749     my $vollist = [];
4750     foreach_volid($conf, sub {
4751         my ($volid, $is_cdrom) = @_;
4752
4753         return if $volid =~ m|^/|;
4754
4755         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4756         return if !$sid;
4757
4758         push @$vollist, $volid;
4759     });
4760
4761     return $vollist;
4762 }
4763
4764 sub vm_stop_cleanup {
4765     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4766
4767     eval {
4768
4769         if (!$keepActive) {
4770             my $vollist = get_vm_volumes($conf);
4771             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4772         }
4773
4774         foreach my $ext (qw(mon qmp pid vnc qga)) {
4775             unlink "/var/run/qemu-server/${vmid}.$ext";
4776         }
4777
4778         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4779     };
4780     warn $@ if $@; # avoid errors - just warn
4781 }
4782
4783 # Note: use $nockeck to skip tests if VM configuration file exists.
4784 # We need that when migration VMs to other nodes (files already moved)
4785 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4786 sub vm_stop {
4787     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4788
4789     $force = 1 if !defined($force) && !$shutdown;
4790
4791     if ($migratedfrom){
4792         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4793         kill 15, $pid if $pid;
4794         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4795         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4796         return;
4797     }
4798
4799     PVE::QemuConfig->lock_config($vmid, sub {
4800
4801         my $pid = check_running($vmid, $nocheck);
4802         return if !$pid;
4803
4804         my $conf;
4805         if (!$nocheck) {
4806             $conf = PVE::QemuConfig->load_config($vmid);
4807             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4808             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4809                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4810                 $timeout = $opts->{down} if $opts->{down};
4811             }
4812         }
4813
4814         $timeout = 60 if !defined($timeout);
4815
4816         eval {
4817             if ($shutdown) {
4818                 if (defined($conf) && $conf->{agent}) {
4819                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4820                 } else {
4821                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4822                 }
4823             } else {
4824                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4825             }
4826         };
4827         my $err = $@;
4828
4829         if (!$err) {
4830             my $count = 0;
4831             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4832                 $count++;
4833                 sleep 1;
4834             }
4835
4836             if ($count >= $timeout) {
4837                 if ($force) {
4838                     warn "VM still running - terminating now with SIGTERM\n";
4839                     kill 15, $pid;
4840                 } else {
4841                     die "VM quit/powerdown failed - got timeout\n";
4842                 }
4843             } else {
4844                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4845                 return;
4846             }
4847         } else {
4848             if ($force) {
4849                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4850                 kill 15, $pid;
4851             } else {
4852                 die "VM quit/powerdown failed\n";
4853             }
4854         }
4855
4856         # wait again
4857         $timeout = 10;
4858
4859         my $count = 0;
4860         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4861             $count++;
4862             sleep 1;
4863         }
4864
4865         if ($count >= $timeout) {
4866             warn "VM still running - terminating now with SIGKILL\n";
4867             kill 9, $pid;
4868             sleep 1;
4869         }
4870
4871         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4872    });
4873 }
4874
4875 sub vm_suspend {
4876     my ($vmid, $skiplock) = @_;
4877
4878     PVE::QemuConfig->lock_config($vmid, sub {
4879
4880         my $conf = PVE::QemuConfig->load_config($vmid);
4881
4882         PVE::QemuConfig->check_lock($conf)
4883             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4884
4885         vm_mon_cmd($vmid, "stop");
4886     });
4887 }
4888
4889 sub vm_resume {
4890     my ($vmid, $skiplock, $nocheck) = @_;
4891
4892     PVE::QemuConfig->lock_config($vmid, sub {
4893
4894         if (!$nocheck) {
4895
4896             my $conf = PVE::QemuConfig->load_config($vmid);
4897
4898             PVE::QemuConfig->check_lock($conf)
4899                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4900
4901             vm_mon_cmd($vmid, "cont");
4902
4903         } else {
4904             vm_mon_cmd_nocheck($vmid, "cont");
4905         }
4906     });
4907 }
4908
4909 sub vm_sendkey {
4910     my ($vmid, $skiplock, $key) = @_;
4911
4912     PVE::QemuConfig->lock_config($vmid, sub {
4913
4914         my $conf = PVE::QemuConfig->load_config($vmid);
4915
4916         # there is no qmp command, so we use the human monitor command
4917         vm_human_monitor_command($vmid, "sendkey $key");
4918     });
4919 }
4920
4921 sub vm_destroy {
4922     my ($storecfg, $vmid, $skiplock) = @_;
4923
4924     PVE::QemuConfig->lock_config($vmid, sub {
4925
4926         my $conf = PVE::QemuConfig->load_config($vmid);
4927
4928         if (!check_running($vmid)) {
4929             destroy_vm($storecfg, $vmid, undef, $skiplock);
4930         } else {
4931             die "VM $vmid is running - destroy failed\n";
4932         }
4933     });
4934 }
4935
4936 # pci helpers
4937
4938 sub file_write {
4939     my ($filename, $buf) = @_;
4940
4941     my $fh = IO::File->new($filename, "w");
4942     return undef if !$fh;
4943
4944     my $res = print $fh $buf;
4945
4946     $fh->close();
4947
4948     return $res;
4949 }
4950
4951 sub pci_device_info {
4952     my ($name) = @_;
4953
4954     my $res;
4955
4956     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4957     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4958
4959     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4960     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4961
4962     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4963     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4964
4965     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4966     return undef if !defined($product) || $product !~ s/^0x//;
4967
4968     $res = {
4969         name => $name,
4970         vendor => $vendor,
4971         product => $product,
4972         domain => $domain,
4973         bus => $bus,
4974         slot => $slot,
4975         func => $func,
4976         irq => $irq,
4977         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4978     };
4979
4980     return $res;
4981 }
4982
4983 sub pci_dev_reset {
4984     my ($dev) = @_;
4985
4986     my $name = $dev->{name};
4987
4988     my $fn = "$pcisysfs/devices/$name/reset";
4989
4990     return file_write($fn, "1");
4991 }
4992
4993 sub pci_dev_bind_to_vfio {
4994     my ($dev) = @_;
4995
4996     my $name = $dev->{name};
4997
4998     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4999
5000     if (!-d $vfio_basedir) {
5001         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5002     }
5003     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5004
5005     my $testdir = "$vfio_basedir/$name";
5006     return 1 if -d $testdir;
5007
5008     my $data = "$dev->{vendor} $dev->{product}";
5009     return undef if !file_write("$vfio_basedir/new_id", $data);
5010
5011     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5012     if (!file_write($fn, $name)) {
5013         return undef if -f $fn;
5014     }
5015
5016     $fn = "$vfio_basedir/bind";
5017     if (! -d $testdir) {
5018         return undef if !file_write($fn, $name);
5019     }
5020
5021     return -d $testdir;
5022 }
5023
5024 sub pci_dev_group_bind_to_vfio {
5025     my ($pciid) = @_;
5026
5027     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5028
5029     if (!-d $vfio_basedir) {
5030         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5031     }
5032     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5033
5034     # get IOMMU group devices
5035     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5036       my @devs = grep /^0000:/, readdir($D);
5037     closedir($D);
5038
5039     foreach my $pciid (@devs) {
5040         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5041
5042         # pci bridges, switches or root ports are not supported
5043         # they have a pci_bus subdirectory so skip them
5044         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5045
5046         my $info = pci_device_info($1);
5047         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5048     }
5049
5050     return 1;
5051 }
5052
5053 # vzdump restore implementaion
5054
5055 sub tar_archive_read_firstfile {
5056     my $archive = shift;
5057
5058     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5059
5060     # try to detect archive type first
5061     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5062         die "unable to open file '$archive'\n";
5063     my $firstfile = <$fh>;
5064     kill 15, $pid;
5065     close $fh;
5066
5067     die "ERROR: archive contaions no data\n" if !$firstfile;
5068     chomp $firstfile;
5069
5070     return $firstfile;
5071 }
5072
5073 sub tar_restore_cleanup {
5074     my ($storecfg, $statfile) = @_;
5075
5076     print STDERR "starting cleanup\n";
5077
5078     if (my $fd = IO::File->new($statfile, "r")) {
5079         while (defined(my $line = <$fd>)) {
5080             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5081                 my $volid = $2;
5082                 eval {
5083                     if ($volid =~ m|^/|) {
5084                         unlink $volid || die 'unlink failed\n';
5085                     } else {
5086                         PVE::Storage::vdisk_free($storecfg, $volid);
5087                     }
5088                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5089                 };
5090                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5091             } else {
5092                 print STDERR "unable to parse line in statfile - $line";
5093             }
5094         }
5095         $fd->close();
5096     }
5097 }
5098
5099 sub restore_archive {
5100     my ($archive, $vmid, $user, $opts) = @_;
5101
5102     my $format = $opts->{format};
5103     my $comp;
5104
5105     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5106         $format = 'tar' if !$format;
5107         $comp = 'gzip';
5108     } elsif ($archive =~ m/\.tar$/) {
5109         $format = 'tar' if !$format;
5110     } elsif ($archive =~ m/.tar.lzo$/) {
5111         $format = 'tar' if !$format;
5112         $comp = 'lzop';
5113     } elsif ($archive =~ m/\.vma$/) {
5114         $format = 'vma' if !$format;
5115     } elsif ($archive =~ m/\.vma\.gz$/) {
5116         $format = 'vma' if !$format;
5117         $comp = 'gzip';
5118     } elsif ($archive =~ m/\.vma\.lzo$/) {
5119         $format = 'vma' if !$format;
5120         $comp = 'lzop';
5121     } else {
5122         $format = 'vma' if !$format; # default
5123     }
5124
5125     # try to detect archive format
5126     if ($format eq 'tar') {
5127         return restore_tar_archive($archive, $vmid, $user, $opts);
5128     } else {
5129         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5130     }
5131 }
5132
5133 sub restore_update_config_line {
5134     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5135
5136     return if $line =~ m/^\#qmdump\#/;
5137     return if $line =~ m/^\#vzdump\#/;
5138     return if $line =~ m/^lock:/;
5139     return if $line =~ m/^unused\d+:/;
5140     return if $line =~ m/^parent:/;
5141     return if $line =~ m/^template:/; # restored VM is never a template
5142
5143     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5144     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5145         # try to convert old 1.X settings
5146         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5147         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5148             my ($model, $macaddr) = split(/\=/, $devconfig);
5149             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5150             my $net = {
5151                 model => $model,
5152                 bridge => "vmbr$ind",
5153                 macaddr => $macaddr,
5154             };
5155             my $netstr = print_net($net);
5156
5157             print $outfd "net$cookie->{netcount}: $netstr\n";
5158             $cookie->{netcount}++;
5159         }
5160     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5161         my ($id, $netstr) = ($1, $2);
5162         my $net = parse_net($netstr);
5163         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5164         $netstr = print_net($net);
5165         print $outfd "$id: $netstr\n";
5166     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5167         my $virtdev = $1;
5168         my $value = $3;
5169         my $di = parse_drive($virtdev, $value);
5170         if (defined($di->{backup}) && !$di->{backup}) {
5171             print $outfd "#$line";
5172         } elsif ($map->{$virtdev}) {
5173             delete $di->{format}; # format can change on restore
5174             $di->{file} = $map->{$virtdev};
5175             $value = print_drive($vmid, $di);
5176             print $outfd "$virtdev: $value\n";
5177         } else {
5178             print $outfd $line;
5179         }
5180     } else {
5181         print $outfd $line;
5182     }
5183 }
5184
5185 sub scan_volids {
5186     my ($cfg, $vmid) = @_;
5187
5188     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5189
5190     my $volid_hash = {};
5191     foreach my $storeid (keys %$info) {
5192         foreach my $item (@{$info->{$storeid}}) {
5193             next if !($item->{volid} && $item->{size});
5194             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5195             $volid_hash->{$item->{volid}} = $item;
5196         }
5197     }
5198
5199     return $volid_hash;
5200 }
5201
5202 sub is_volume_in_use {
5203     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5204
5205     my $path = PVE::Storage::path($storecfg, $volid);
5206
5207     my $scan_config = sub {
5208         my ($cref, $snapname) = @_;
5209
5210         foreach my $key (keys %$cref) {
5211             my $value = $cref->{$key};
5212             if (is_valid_drivename($key)) {
5213                 next if $skip_drive && $key eq $skip_drive;
5214                 my $drive = parse_drive($key, $value);
5215                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5216                 return 1 if $volid eq $drive->{file};
5217                 if ($drive->{file} =~ m!^/!) {
5218                     return 1 if $drive->{file} eq $path;
5219                 } else {
5220                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5221                     next if !$storeid;
5222                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5223                     next if !$scfg;
5224                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5225                 }
5226             }
5227         }
5228
5229         return 0;
5230     };
5231
5232     return 1 if &$scan_config($conf);
5233
5234     undef $skip_drive;
5235
5236     foreach my $snapname (keys %{$conf->{snapshots}}) {
5237         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5238     }
5239
5240     return 0;
5241 }
5242
5243 sub update_disksize {
5244     my ($vmid, $conf, $volid_hash) = @_;
5245
5246     my $changes;
5247
5248     my $used = {};
5249
5250     # Note: it is allowed to define multiple storages with same path (alias), so
5251     # we need to check both 'volid' and real 'path' (two different volid can point
5252     # to the same path).
5253
5254     my $usedpath = {};
5255
5256     # update size info
5257     foreach my $opt (keys %$conf) {
5258         if (is_valid_drivename($opt)) {
5259             my $drive = parse_drive($opt, $conf->{$opt});
5260             my $volid = $drive->{file};
5261             next if !$volid;
5262
5263             $used->{$volid} = 1;
5264             if ($volid_hash->{$volid} &&
5265                 (my $path = $volid_hash->{$volid}->{path})) {
5266                 $usedpath->{$path} = 1;
5267             }
5268
5269             next if drive_is_cdrom($drive);
5270             next if !$volid_hash->{$volid};
5271
5272             $drive->{size} = $volid_hash->{$volid}->{size};
5273             my $new = print_drive($vmid, $drive);
5274             if ($new ne $conf->{$opt}) {
5275                 $changes = 1;
5276                 $conf->{$opt} = $new;
5277             }
5278         }
5279     }
5280
5281     # remove 'unusedX' entry if volume is used
5282     foreach my $opt (keys %$conf) {
5283         next if $opt !~ m/^unused\d+$/;
5284         my $volid = $conf->{$opt};
5285         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5286         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5287             $changes = 1;
5288             delete $conf->{$opt};
5289         }
5290     }
5291
5292     foreach my $volid (sort keys %$volid_hash) {
5293         next if $volid =~ m/vm-$vmid-state-/;
5294         next if $used->{$volid};
5295         my $path = $volid_hash->{$volid}->{path};
5296         next if !$path; # just to be sure
5297         next if $usedpath->{$path};
5298         $changes = 1;
5299         PVE::QemuConfig->add_unused_volume($conf, $volid);
5300         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5301     }
5302
5303     return $changes;
5304 }
5305
5306 sub rescan {
5307     my ($vmid, $nolock) = @_;
5308
5309     my $cfg = PVE::Storage::config();
5310
5311     my $volid_hash = scan_volids($cfg, $vmid);
5312
5313     my $updatefn =  sub {
5314         my ($vmid) = @_;
5315
5316         my $conf = PVE::QemuConfig->load_config($vmid);
5317
5318         PVE::QemuConfig->check_lock($conf);
5319
5320         my $vm_volids = {};
5321         foreach my $volid (keys %$volid_hash) {
5322             my $info = $volid_hash->{$volid};
5323             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5324         }
5325
5326         my $changes = update_disksize($vmid, $conf, $vm_volids);
5327
5328         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5329     };
5330
5331     if (defined($vmid)) {
5332         if ($nolock) {
5333             &$updatefn($vmid);
5334         } else {
5335             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5336         }
5337     } else {
5338         my $vmlist = config_list();
5339         foreach my $vmid (keys %$vmlist) {
5340             if ($nolock) {
5341                 &$updatefn($vmid);
5342             } else {
5343                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5344             }
5345         }
5346     }
5347 }
5348
5349 sub restore_vma_archive {
5350     my ($archive, $vmid, $user, $opts, $comp) = @_;
5351
5352     my $input = $archive eq '-' ? "<&STDIN" : undef;
5353     my $readfrom = $archive;
5354
5355     my $uncomp = '';
5356     if ($comp) {
5357         $readfrom = '-';
5358         my $qarchive = PVE::Tools::shellquote($archive);
5359         if ($comp eq 'gzip') {
5360             $uncomp = "zcat $qarchive|";
5361         } elsif ($comp eq 'lzop') {
5362             $uncomp = "lzop -d -c $qarchive|";
5363         } else {
5364             die "unknown compression method '$comp'\n";
5365         }
5366
5367     }
5368
5369     my $tmpdir = "/var/tmp/vzdumptmp$$";
5370     rmtree $tmpdir;
5371
5372     # disable interrupts (always do cleanups)
5373     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5374         warn "got interrupt - ignored\n";
5375     };
5376
5377     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5378     POSIX::mkfifo($mapfifo, 0600);
5379     my $fifofh;
5380
5381     my $openfifo = sub {
5382         open($fifofh, '>', $mapfifo) || die $!;
5383     };
5384
5385     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5386
5387     my $oldtimeout;
5388     my $timeout = 5;
5389
5390     my $devinfo = {};
5391
5392     my $rpcenv = PVE::RPCEnvironment::get();
5393
5394     my $conffile = PVE::QemuConfig->config_file($vmid);
5395     my $tmpfn = "$conffile.$$.tmp";
5396
5397     # Note: $oldconf is undef if VM does not exists
5398     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5399     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5400
5401     my $print_devmap = sub {
5402         my $virtdev_hash = {};
5403
5404         my $cfgfn = "$tmpdir/qemu-server.conf";
5405
5406         # we can read the config - that is already extracted
5407         my $fh = IO::File->new($cfgfn, "r") ||
5408             "unable to read qemu-server.conf - $!\n";
5409
5410         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5411         if (-f $fwcfgfn) {
5412             my $pve_firewall_dir = '/etc/pve/firewall';
5413             mkdir $pve_firewall_dir; # make sure the dir exists
5414             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5415         }
5416
5417         while (defined(my $line = <$fh>)) {
5418             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5419                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5420                 die "archive does not contain data for drive '$virtdev'\n"
5421                     if !$devinfo->{$devname};
5422                 if (defined($opts->{storage})) {
5423                     $storeid = $opts->{storage} || 'local';
5424                 } elsif (!$storeid) {
5425                     $storeid = 'local';
5426                 }
5427                 $format = 'raw' if !$format;
5428                 $devinfo->{$devname}->{devname} = $devname;
5429                 $devinfo->{$devname}->{virtdev} = $virtdev;
5430                 $devinfo->{$devname}->{format} = $format;
5431                 $devinfo->{$devname}->{storeid} = $storeid;
5432
5433                 # check permission on storage
5434                 my $pool = $opts->{pool}; # todo: do we need that?
5435                 if ($user ne 'root@pam') {
5436                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5437                 }
5438
5439                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5440             }
5441         }
5442
5443         foreach my $devname (keys %$devinfo) {
5444             die "found no device mapping information for device '$devname'\n"
5445                 if !$devinfo->{$devname}->{virtdev};
5446         }
5447
5448         my $cfg = PVE::Storage::config();
5449
5450         # create empty/temp config
5451         if ($oldconf) {
5452             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5453             foreach_drive($oldconf, sub {
5454                 my ($ds, $drive) = @_;
5455
5456                 return if drive_is_cdrom($drive);
5457
5458                 my $volid = $drive->{file};
5459
5460                 return if !$volid || $volid =~ m|^/|;
5461
5462                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5463                 return if !$path || !$owner || ($owner != $vmid);
5464
5465                 # Note: only delete disk we want to restore
5466                 # other volumes will become unused
5467                 if ($virtdev_hash->{$ds}) {
5468                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5469                     if (my $err = $@) {
5470                         warn $err;
5471                     }
5472                 }
5473             });
5474
5475             # delete vmstate files
5476             # since after the restore we have no snapshots anymore
5477             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5478                 my $snap = $oldconf->{snapshots}->{$snapname};
5479                 if ($snap->{vmstate}) {
5480                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5481                     if (my $err = $@) {
5482                         warn $err;
5483                     }
5484                 }
5485             }
5486         }
5487
5488         my $map = {};
5489         foreach my $virtdev (sort keys %$virtdev_hash) {
5490             my $d = $virtdev_hash->{$virtdev};
5491             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5492             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5493
5494             # test if requested format is supported
5495             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5496             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5497             $d->{format} = $defFormat if !$supported;
5498
5499             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5500                                                   $d->{format}, undef, $alloc_size);
5501             print STDERR "new volume ID is '$volid'\n";
5502             $d->{volid} = $volid;
5503             my $path = PVE::Storage::path($cfg, $volid);
5504
5505             PVE::Storage::activate_volumes($cfg,[$volid]);
5506
5507             my $write_zeros = 1;
5508             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5509                 $write_zeros = 0;
5510             }
5511
5512             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5513
5514             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5515             $map->{$virtdev} = $volid;
5516         }
5517
5518         $fh->seek(0, 0) || die "seek failed - $!\n";
5519
5520         my $outfd = new IO::File ($tmpfn, "w") ||
5521             die "unable to write config for VM $vmid\n";
5522
5523         my $cookie = { netcount => 0 };
5524         while (defined(my $line = <$fh>)) {
5525             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5526         }
5527
5528         $fh->close();
5529         $outfd->close();
5530     };
5531
5532     eval {
5533         # enable interrupts
5534         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5535             die "interrupted by signal\n";
5536         };
5537         local $SIG{ALRM} = sub { die "got timeout\n"; };
5538
5539         $oldtimeout = alarm($timeout);
5540
5541         my $parser = sub {
5542             my $line = shift;
5543
5544             print "$line\n";
5545
5546             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5547                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5548                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5549             } elsif ($line =~ m/^CTIME: /) {
5550                 # we correctly received the vma config, so we can disable
5551                 # the timeout now for disk allocation (set to 10 minutes, so
5552                 # that we always timeout if something goes wrong)
5553                 alarm(600);
5554                 &$print_devmap();
5555                 print $fifofh "done\n";
5556                 my $tmp = $oldtimeout || 0;
5557                 $oldtimeout = undef;
5558                 alarm($tmp);
5559                 close($fifofh);
5560             }
5561         };
5562
5563         print "restore vma archive: $cmd\n";
5564         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5565     };
5566     my $err = $@;
5567
5568     alarm($oldtimeout) if $oldtimeout;
5569
5570     my $vollist = [];
5571     foreach my $devname (keys %$devinfo) {
5572         my $volid = $devinfo->{$devname}->{volid};
5573         push @$vollist, $volid if $volid;
5574     }
5575
5576     my $cfg = PVE::Storage::config();
5577     PVE::Storage::deactivate_volumes($cfg, $vollist);
5578
5579     unlink $mapfifo;
5580
5581     if ($err) {
5582         rmtree $tmpdir;
5583         unlink $tmpfn;
5584
5585         foreach my $devname (keys %$devinfo) {
5586             my $volid = $devinfo->{$devname}->{volid};
5587             next if !$volid;
5588             eval {
5589                 if ($volid =~ m|^/|) {
5590                     unlink $volid || die 'unlink failed\n';
5591                 } else {
5592                     PVE::Storage::vdisk_free($cfg, $volid);
5593                 }
5594                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5595             };
5596             print STDERR "unable to cleanup '$volid' - $@" if $@;
5597         }
5598         die $err;
5599     }
5600
5601     rmtree $tmpdir;
5602
5603     rename($tmpfn, $conffile) ||
5604         die "unable to commit configuration file '$conffile'\n";
5605
5606     PVE::Cluster::cfs_update(); # make sure we read new file
5607
5608     eval { rescan($vmid, 1); };
5609     warn $@ if $@;
5610 }
5611
5612 sub restore_tar_archive {
5613     my ($archive, $vmid, $user, $opts) = @_;
5614
5615     if ($archive ne '-') {
5616         my $firstfile = tar_archive_read_firstfile($archive);
5617         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5618             if $firstfile ne 'qemu-server.conf';
5619     }
5620
5621     my $storecfg = PVE::Storage::config();
5622
5623     # destroy existing data - keep empty config
5624     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5625     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5626
5627     my $tocmd = "/usr/lib/qemu-server/qmextract";
5628
5629     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5630     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5631     $tocmd .= ' --prealloc' if $opts->{prealloc};
5632     $tocmd .= ' --info' if $opts->{info};
5633
5634     # tar option "xf" does not autodetect compression when read from STDIN,
5635     # so we pipe to zcat
5636     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5637         PVE::Tools::shellquote("--to-command=$tocmd");
5638
5639     my $tmpdir = "/var/tmp/vzdumptmp$$";
5640     mkpath $tmpdir;
5641
5642     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5643     local $ENV{VZDUMP_VMID} = $vmid;
5644     local $ENV{VZDUMP_USER} = $user;
5645
5646     my $conffile = PVE::QemuConfig->config_file($vmid);
5647     my $tmpfn = "$conffile.$$.tmp";
5648
5649     # disable interrupts (always do cleanups)
5650     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5651         print STDERR "got interrupt - ignored\n";
5652     };
5653
5654     eval {
5655         # enable interrupts
5656         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5657             die "interrupted by signal\n";
5658         };
5659
5660         if ($archive eq '-') {
5661             print "extracting archive from STDIN\n";
5662             run_command($cmd, input => "<&STDIN");
5663         } else {
5664             print "extracting archive '$archive'\n";
5665             run_command($cmd);
5666         }
5667
5668         return if $opts->{info};
5669
5670         # read new mapping
5671         my $map = {};
5672         my $statfile = "$tmpdir/qmrestore.stat";
5673         if (my $fd = IO::File->new($statfile, "r")) {
5674             while (defined (my $line = <$fd>)) {
5675                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5676                     $map->{$1} = $2 if $1;
5677                 } else {
5678                     print STDERR "unable to parse line in statfile - $line\n";
5679                 }
5680             }
5681             $fd->close();
5682         }
5683
5684         my $confsrc = "$tmpdir/qemu-server.conf";
5685
5686         my $srcfd = new IO::File($confsrc, "r") ||
5687             die "unable to open file '$confsrc'\n";
5688
5689         my $outfd = new IO::File ($tmpfn, "w") ||
5690             die "unable to write config for VM $vmid\n";
5691
5692         my $cookie = { netcount => 0 };
5693         while (defined (my $line = <$srcfd>)) {
5694             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5695         }
5696
5697         $srcfd->close();
5698         $outfd->close();
5699     };
5700     my $err = $@;
5701
5702     if ($err) {
5703
5704         unlink $tmpfn;
5705
5706         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5707
5708         die $err;
5709     }
5710
5711     rmtree $tmpdir;
5712
5713     rename $tmpfn, $conffile ||
5714         die "unable to commit configuration file '$conffile'\n";
5715
5716     PVE::Cluster::cfs_update(); # make sure we read new file
5717
5718     eval { rescan($vmid, 1); };
5719     warn $@ if $@;
5720 };
5721
5722 sub foreach_writable_storage {
5723     my ($conf, $func) = @_;
5724
5725     my $sidhash = {};
5726
5727     foreach my $ds (keys %$conf) {
5728         next if !is_valid_drivename($ds);
5729
5730         my $drive = parse_drive($ds, $conf->{$ds});
5731         next if !$drive;
5732         next if drive_is_cdrom($drive);
5733
5734         my $volid = $drive->{file};
5735
5736         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5737         $sidhash->{$sid} = $sid if $sid;
5738     }
5739
5740     foreach my $sid (sort keys %$sidhash) {
5741         &$func($sid);
5742     }
5743 }
5744
5745 sub do_snapshots_with_qemu {
5746     my ($storecfg, $volid) = @_;
5747
5748     my $storage_name = PVE::Storage::parse_volume_id($volid);
5749
5750     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5751         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5752         return 1;
5753     }
5754
5755     if ($volid =~ m/\.(qcow2|qed)$/){
5756         return 1;
5757     }
5758
5759     return undef;
5760 }
5761
5762 sub qga_check_running {
5763     my ($vmid) = @_;
5764
5765     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5766     if ($@) {
5767         warn "Qemu Guest Agent are not running - $@";
5768         return 0;
5769     }
5770     return 1;
5771 }
5772
5773 sub template_create {
5774     my ($vmid, $conf, $disk) = @_;
5775
5776     my $storecfg = PVE::Storage::config();
5777
5778     foreach_drive($conf, sub {
5779         my ($ds, $drive) = @_;
5780
5781         return if drive_is_cdrom($drive);
5782         return if $disk && $ds ne $disk;
5783
5784         my $volid = $drive->{file};
5785         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5786
5787         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5788         $drive->{file} = $voliddst;
5789         $conf->{$ds} = print_drive($vmid, $drive);
5790         PVE::QemuConfig->write_config($vmid, $conf);
5791     });
5792 }
5793
5794 sub qemu_img_convert {
5795     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5796
5797     my $storecfg = PVE::Storage::config();
5798     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5799     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5800
5801     if ($src_storeid && $dst_storeid) {
5802
5803         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5804
5805         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5806         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5807
5808         my $src_format = qemu_img_format($src_scfg, $src_volname);
5809         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5810
5811         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5812         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5813
5814         my $cmd = [];
5815         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5816         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5817         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5818         if ($is_zero_initialized) {
5819             push @$cmd, "zeroinit:$dst_path";
5820         } else {
5821             push @$cmd, $dst_path;
5822         }
5823
5824         my $parser = sub {
5825             my $line = shift;
5826             if($line =~ m/\((\S+)\/100\%\)/){
5827                 my $percent = $1;
5828                 my $transferred = int($size * $percent / 100);
5829                 my $remaining = $size - $transferred;
5830
5831                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5832             }
5833
5834         };
5835
5836         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5837         my $err = $@;
5838         die "copy failed: $err" if $err;
5839     }
5840 }
5841
5842 sub qemu_img_format {
5843     my ($scfg, $volname) = @_;
5844
5845     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5846         return $1;
5847     } else {
5848         return "raw";
5849     }
5850 }
5851
5852 sub qemu_drive_mirror {
5853     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5854
5855     my $storecfg = PVE::Storage::config();
5856     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5857
5858     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5859
5860     my $format = qemu_img_format($dst_scfg, $dst_volname);
5861
5862     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5863
5864     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5865
5866     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5867     $opts->{format} = $format if $format;
5868
5869     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5870
5871     my $finish_job = sub {
5872         while (1) {
5873             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5874             my $stat = @$stats[0];
5875             last if !$stat;
5876             sleep 1;
5877         }
5878     };
5879
5880     eval {
5881     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5882         while (1) {
5883             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5884             my $stat = @$stats[0];
5885             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5886             die "error job is not mirroring" if $stat->{type} ne "mirror";
5887
5888             my $busy = $stat->{busy};
5889             my $ready = $stat->{ready};
5890
5891             if (my $total = $stat->{len}) {
5892                 my $transferred = $stat->{offset} || 0;
5893                 my $remaining = $total - $transferred;
5894                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5895
5896                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5897             }
5898
5899
5900             if ($stat->{ready} eq 'true') {
5901
5902                 last if $vmiddst != $vmid;
5903
5904                 # try to switch the disk if source and destination are on the same guest
5905                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5906                 if (!$@) {
5907                     &$finish_job();
5908                     last;
5909                 }
5910                 die $@ if $@ !~ m/cannot be completed/;
5911             }
5912             sleep 1;
5913         }
5914
5915
5916     };
5917     my $err = $@;
5918
5919     my $cancel_job = sub {
5920         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5921         &$finish_job();
5922     };
5923
5924     if ($err) {
5925         eval { &$cancel_job(); };
5926         die "mirroring error: $err";
5927     }
5928
5929     if ($vmiddst != $vmid) {
5930         # if we clone a disk for a new target vm, we don't switch the disk
5931         &$cancel_job(); # so we call block-job-cancel
5932     }
5933 }
5934
5935 sub clone_disk {
5936     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5937         $newvmid, $storage, $format, $full, $newvollist) = @_;
5938
5939     my $newvolid;
5940
5941     if (!$full) {
5942         print "create linked clone of drive $drivename ($drive->{file})\n";
5943         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5944         push @$newvollist, $newvolid;
5945     } else {
5946         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5947         $storeid = $storage if $storage;
5948
5949         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5950         if (!$format) {
5951             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5952             $format = qemu_img_format($scfg, $volname);
5953         }
5954
5955         # test if requested format is supported - else use default
5956         my $supported = grep { $_ eq $format } @$validFormats;
5957         $format = $defFormat if !$supported;
5958
5959         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5960
5961         print "create full clone of drive $drivename ($drive->{file})\n";
5962         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5963         push @$newvollist, $newvolid;
5964
5965         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
5966
5967         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5968         if (!$running || $snapname) {
5969             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5970         } else {
5971
5972             my $kvmver = get_running_qemu_version ($vmid);
5973             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
5974                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
5975                     if $drive->{iothread};
5976             }
5977
5978             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5979         }
5980     }
5981
5982     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5983
5984     my $disk = $drive;
5985     $disk->{format} = undef;
5986     $disk->{file} = $newvolid;
5987     $disk->{size} = $size;
5988
5989     return $disk;
5990 }
5991
5992 # this only works if VM is running
5993 sub get_current_qemu_machine {
5994     my ($vmid) = @_;
5995
5996     my $cmd = { execute => 'query-machines', arguments => {} };
5997     my $res = vm_qmp_command($vmid, $cmd);
5998
5999     my ($current, $default);
6000     foreach my $e (@$res) {
6001         $default = $e->{name} if $e->{'is-default'};
6002         $current = $e->{name} if $e->{'is-current'};
6003     }
6004
6005     # fallback to the default machine if current is not supported by qemu
6006     return $current || $default || 'pc';
6007 }
6008
6009 sub get_running_qemu_version {
6010     my ($vmid) = @_;
6011     my $cmd = { execute => 'query-version', arguments => {} };
6012     my $res = vm_qmp_command($vmid, $cmd);
6013     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6014 }
6015
6016 sub qemu_machine_feature_enabled {
6017     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6018
6019     my $current_major;
6020     my $current_minor;
6021
6022     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6023
6024         $current_major = $3;
6025         $current_minor = $4;
6026
6027     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6028
6029         $current_major = $1;
6030         $current_minor = $2;
6031     }
6032
6033     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6034
6035
6036 }
6037
6038 sub qemu_machine_pxe {
6039     my ($vmid, $conf, $machine) = @_;
6040
6041     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6042
6043     foreach my $opt (keys %$conf) {
6044         next if $opt !~ m/^net(\d+)$/;
6045         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6046         next if !$net;
6047         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6048         return $machine.".pxe" if $romfile =~ m/pxe/;
6049         last;
6050     }
6051
6052     return $machine;
6053 }
6054
6055 sub qemu_use_old_bios_files {
6056     my ($machine_type) = @_;
6057
6058     return if !$machine_type;
6059
6060     my $use_old_bios_files = undef;
6061
6062     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6063         $machine_type = $1;
6064         $use_old_bios_files = 1;
6065     } else {
6066         my $kvmver = kvm_user_version();
6067         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6068         # load new efi bios files on migration. So this hack is required to allow
6069         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6070         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6071         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6072     }
6073
6074     return ($use_old_bios_files, $machine_type);
6075 }
6076
6077 sub lspci {
6078
6079     my $devices = {};
6080
6081     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6082             my (undef, $id, $function) = @_;
6083             my $res = { id => $id, function => $function};
6084             push @{$devices->{$id}}, $res;
6085     });
6086
6087     return $devices;
6088 }
6089
6090 sub vm_iothreads_list {
6091     my ($vmid) = @_;
6092
6093     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6094
6095     my $iothreads = {};
6096     foreach my $iothread (@$res) {
6097         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6098     }
6099
6100     return $iothreads;
6101 }
6102
6103 sub scsihw_infos {
6104     my ($conf, $drive) = @_;
6105
6106     my $maxdev = 0;
6107
6108     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6109         $maxdev = 7;
6110     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6111         $maxdev = 1;
6112     } else {
6113         $maxdev = 256;
6114     }
6115
6116     my $controller = int($drive->{index} / $maxdev);
6117     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6118
6119     return ($maxdev, $controller, $controller_prefix);
6120 }
6121
6122 # bash completion helper
6123
6124 sub complete_backup_archives {
6125     my ($cmdname, $pname, $cvalue) = @_;
6126
6127     my $cfg = PVE::Storage::config();
6128
6129     my $storeid;
6130
6131     if ($cvalue =~ m/^([^:]+):/) {
6132         $storeid = $1;
6133     }
6134
6135     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6136
6137     my $res = [];
6138     foreach my $id (keys %$data) {
6139         foreach my $item (@{$data->{$id}}) {
6140             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6141             push @$res, $item->{volid} if defined($item->{volid});
6142         }
6143     }
6144
6145     return $res;
6146 }
6147
6148 my $complete_vmid_full = sub {
6149     my ($running) = @_;
6150
6151     my $idlist = vmstatus();
6152
6153     my $res = [];
6154
6155     foreach my $id (keys %$idlist) {
6156         my $d = $idlist->{$id};
6157         if (defined($running)) {
6158             next if $d->{template};
6159             next if $running && $d->{status} ne 'running';
6160             next if !$running && $d->{status} eq 'running';
6161         }
6162         push @$res, $id;
6163
6164     }
6165     return $res;
6166 };
6167
6168 sub complete_vmid {
6169     return &$complete_vmid_full();
6170 }
6171
6172 sub complete_vmid_stopped {
6173     return &$complete_vmid_full(0);
6174 }
6175
6176 sub complete_vmid_running {
6177     return &$complete_vmid_full(1);
6178 }
6179
6180 sub complete_storage {
6181
6182     my $cfg = PVE::Storage::config();
6183     my $ids = $cfg->{ids};
6184
6185     my $res = [];
6186     foreach my $sid (keys %$ids) {
6187         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6188         next if !$ids->{$sid}->{content}->{images};
6189         push @$res, $sid;
6190     }
6191
6192     return $res;
6193 }
6194
6195 1;