PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('skiplock', {
  60     description => "Ignore locks - only root is allowed to use this option.",
  61     type => 'boolean',
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  66     description => "Some command save/restore state from this location.",
  67     type => 'string',
  68     maxLength => 128,
  69     optional => 1,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  73     description => "The name of the snapshot.",
  74     type => 'string', format => 'pve-configid',
  75     maxLength => 40,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120
 121     # AMD CPUs
 122     athlon => 'AuthenticAMD',
 123     phenom  => 'AuthenticAMD',
 124     Opteron_G1  => 'AuthenticAMD',
 125     Opteron_G2  => 'AuthenticAMD',
 126     Opteron_G3  => 'AuthenticAMD',
 127     Opteron_G4  => 'AuthenticAMD',
 128     Opteron_G5  => 'AuthenticAMD',
 129
 130     # generic types, use vendor from host node
 131     host => 'default',
 132     kvm32 => 'default',
 133     kvm64 => 'default',
 134     qemu32 => 'default',
 135     qemu64 => 'default',
 136 };
 137
 138 my $cpu_fmt = {
 139     cputype => {
 140         description => "Emulated CPU type.",
 141         type => 'string',
 142         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 143         format_description => 'cputype',
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 218         minimum => 0,
 219         maximum => 500000,
 220         default => 1024,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 247         enum => PVE::Tools::kvmkeymaplist(),
 248         default => 'en-us',
 249     },
 250     name => {
 251         optional => 1,
 252         type => 'string', format => 'dns-name',
 253         description => "Set a name for the VM. Only used on the configuration web interface.",
 254     },
 255     scsihw => {
 256         optional => 1,
 257         type => 'string',
 258         description => "SCSI controller model",
 259         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 260         default => 'lsi',
 261     },
 262     description => {
 263         optional => 1,
 264         type => 'string',
 265         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 266     },
 267     ostype => {
 268         optional => 1,
 269         type => 'string',
 270         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 271         description => "Specify guest operating system.",
 272         verbose_description => <<EODESC,
 273 Specify guest operating system. This is used to enable special
 274 optimization/features for specific operating systems:
 275
 276 [horizontal]
 277 other;; unspecified OS
 278 wxp;; Microsoft Windows XP
 279 w2k;; Microsoft Windows 2000
 280 w2k3;; Microsoft Windows 2003
 281 w2k8;; Microsoft Windows 2008
 282 wvista;; Microsoft Windows Vista
 283 win7;; Microsoft Windows 7
 284 win8;; Microsoft Windows 8/2012
 285 l24;; Linux 2.4 Kernel
 286 l26;; Linux 2.6/3.X Kernel
 287 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 288 EODESC
 289     },
 290     boot => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 294         pattern => '[acdn]{1,4}',
 295         default => 'cdn',
 296     },
 297     bootdisk => {
 298         optional => 1,
 299         type => 'string', format => 'pve-qm-bootdisk',
 300         description => "Enable booting from specified disk.",
 301         pattern => '(ide|sata|scsi|virtio)\d+',
 302     },
 303     smp => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "The number of CPUs. Please use option -sockets instead.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     sockets => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "The number of CPU sockets.",
 314         minimum => 1,
 315         default => 1,
 316     },
 317     cores => {
 318         optional => 1,
 319         type => 'integer',
 320         description => "The number of cores per socket.",
 321         minimum => 1,
 322         default => 1,
 323     },
 324     numa => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Enable/disable NUMA.",
 328         default => 0,
 329     },
 330     hugepages => {
 331         optional => 1,
 332         type => 'string',
 333         description => "Enable/disable hugepages memory.",
 334         enum => [qw(any 2 1024)],
 335     },
 336     vcpus => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Number of hotplugged vcpus.",
 340         minimum => 1,
 341         default => 0,
 342     },
 343     acpi => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable ACPI.",
 347         default => 1,
 348     },
 349     agent => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable Qemu GuestAgent.",
 353         default => 0,
 354     },
 355     kvm => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable KVM hardware virtualization.",
 359         default => 1,
 360     },
 361     tdf => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable time drift fix.",
 365         default => 0,
 366     },
 367     localtime => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 371     },
 372     freeze => {
 373         optional => 1,
 374         type => 'boolean',
 375         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 376     },
 377     vga => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Select the VGA type.",
 381         verbose_description => "Select the VGA type. If you want to use high resolution" .
 382             " modes (>= 1280x1024x16) then you should use the options " .
 383             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 384             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 385             "display sever. For win* OS you can select how many independent " .
 386             "displays you want, Linux guests can add displays them self. " .
 387             "You can also run without any graphic card, using a serial device" .
 388             " as terminal.",
 389         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 390     },
 391     watchdog => {
 392         optional => 1,
 393         type => 'string', format => 'pve-qm-watchdog',
 394         description => "Create a virtual hardware watchdog device.",
 395         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 396             " (by a guest action), the watchdog must be periodically polled " .
 397             "by an agent inside the guest or else the watchdog will reset " .
 398             "the guest (or execute the respective action specified)",
 399     },
 400     startdate => {
 401         optional => 1,
 402         type => 'string',
 403         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 404         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 405         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 406         default => 'now',
 407     },
 408     startup =>  get_standard_option('pve-startup-order'),
 409     template => {
 410         optional => 1,
 411         type => 'boolean',
 412         description => "Enable/disable Template.",
 413         default => 0,
 414     },
 415     args => {
 416         optional => 1,
 417         type => 'string',
 418         description => "Arbitrary arguments passed to kvm.",
 419         verbose_description => <<EODESCR,
 420 Arbitrary arguments passed to kvm, for example:
 421
 422 args: -no-reboot -no-hpet
 423
 424 NOTE: this option is for experts only.
 425 EODESCR
 426     },
 427     tablet => {
 428         optional => 1,
 429         type => 'boolean',
 430         default => 1,
 431         description => "Enable/disable the USB tablet device.",
 432         verbose_description => "Enable/disable the USB tablet device. This device is " .
 433             "usually needed to allow absolute mouse positioning with VNC. " .
 434             "Else the mouse runs out of sync with normal VNC clients. " .
 435             "If you're running lots of console-only guests on one host, " .
 436             "you may consider disabling this to save some context switches. " .
 437             "This is turned off by default if you use spice (-vga=qxl).",
 438     },
 439     migrate_speed => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 443         minimum => 0,
 444         default => 0,
 445     },
 446     migrate_downtime => {
 447         optional => 1,
 448         type => 'number',
 449         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 450         minimum => 0,
 451         default => 0.1,
 452     },
 453     cdrom => {
 454         optional => 1,
 455         type => 'string', format => 'pve-qm-ide',
 456         typetext => 'volume',
 457         description => "This is an alias for option -ide2",
 458     },
 459     cpu => {
 460         optional => 1,
 461         description => "Emulated CPU type.",
 462         type => 'string',
 463         format => $cpu_fmt,
 464     },
 465     parent => get_standard_option('pve-snapshot-name', {
 466         optional => 1,
 467         description => "Parent snapshot name. This is used internally, and should not be modified.",
 468     }),
 469     snaptime => {
 470         optional => 1,
 471         description => "Timestamp for snapshots.",
 472         type => 'integer',
 473         minimum => 0,
 474     },
 475     vmstate => {
 476         optional => 1,
 477         type => 'string', format => 'pve-volume-id',
 478         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 479     },
 480     machine => {
 481         description => "Specific the Qemu machine type.",
 482         type => 'string',
 483         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 484         maxLength => 40,
 485         optional => 1,
 486     },
 487     smbios1 => {
 488         description => "Specify SMBIOS type 1 fields.",
 489         type => 'string', format => 'pve-qm-smbios1',
 490         maxLength => 256,
 491         optional => 1,
 492     },
 493     protection => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 497         default => 0,
 498     },
 499     bios => {
 500         optional => 1,
 501         type => 'string',
 502         enum => [ qw(seabios ovmf) ],
 503         description => "Select BIOS implementation.",
 504         default => 'seabios',
 505     },
 506 };
 507
 508 # what about other qemu settings ?
 509 #cpu => 'string',
 510 #machine => 'string',
 511 #fda => 'file',
 512 #fdb => 'file',
 513 #mtdblock => 'file',
 514 #sd => 'file',
 515 #pflash => 'file',
 516 #snapshot => 'bool',
 517 #bootp => 'file',
 518 ##tftp => 'dir',
 519 ##smb => 'dir',
 520 #kernel => 'file',
 521 #append => 'string',
 522 #initrd => 'file',
 523 ##soundhw => 'string',
 524
 525 while (my ($k, $v) = each %$confdesc) {
 526     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 527 }
 528
 529 my $MAX_IDE_DISKS = 4;
 530 my $MAX_SCSI_DISKS = 14;
 531 my $MAX_VIRTIO_DISKS = 16;
 532 my $MAX_SATA_DISKS = 6;
 533 my $MAX_USB_DEVICES = 5;
 534 my $MAX_NETS = 32;
 535 my $MAX_UNUSED_DISKS = 8;
 536 my $MAX_HOSTPCI_DEVICES = 4;
 537 my $MAX_SERIAL_PORTS = 4;
 538 my $MAX_PARALLEL_PORTS = 3;
 539 my $MAX_NUMA = 8;
 540
 541 my $numa_fmt = {
 542     cpus => {
 543         type => "string",
 544         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 545         description => "CPUs accessing this NUMA node.",
 546         format_description => "id[-id];...",
 547     },
 548     memory => {
 549         type => "number",
 550         description => "Amount of memory this NUMA node provides.",
 551         optional => 1,
 552     },
 553     hostnodes => {
 554         type => "string",
 555         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 556         description => "Host NUMA nodes to use.",
 557         format_description => "id[-id];...",
 558         optional => 1,
 559     },
 560     policy => {
 561         type => 'string',
 562         enum => [qw(preferred bind interleave)],
 563         description => "NUMA allocation policy.",
 564         optional => 1,
 565     },
 566 };
 567 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 568 my $numadesc = {
 569     optional => 1,
 570     type => 'string', format => $numa_fmt,
 571     description => "NUMA topology.",
 572 };
 573 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 574
 575 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 576     $confdesc->{"numa$i"} = $numadesc;
 577 }
 578
 579 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 580                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 581                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 582 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 583
 584 my $net_fmt_bridge_descr = <<__EOD__;
 585 Bridge to attach the network device to. The Proxmox VE standard bridge
 586 is called 'vmbr0'.
 587
 588 If you do not specify a bridge, we create a kvm user (NATed) network
 589 device, which provides DHCP and DNS services. The following addresses
 590 are used:
 591
 592  10.0.2.2   Gateway
 593  10.0.2.3   DNS Server
 594  10.0.2.4   SMB Server
 595
 596 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 597 __EOD__
 598
 599 my $net_fmt = {
 600     macaddr => {
 601         type => 'string',
 602         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 603         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 604         format_description => "XX:XX:XX:XX:XX:XX",
 605         optional => 1,
 606     },
 607     model => {
 608         type => 'string',
 609         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 610         format_description => 'model',
 611         enum => $nic_model_list,
 612         default_key => 1,
 613     },
 614     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 615     bridge => {
 616         type => 'string',
 617         description => $net_fmt_bridge_descr,
 618         format_description => 'bridge',
 619         optional => 1,
 620     },
 621     queues => {
 622         type => 'integer',
 623         minimum => 0, maximum => 16,
 624         description => 'Number of packet queues to be used on the device.',
 625         optional => 1,
 626     },
 627     rate => {
 628         type => 'number',
 629         minimum => 0,
 630         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 631         optional => 1,
 632     },
 633     tag => {
 634         type => 'integer',
 635         minimum => 1, maximum => 4094,
 636         description => 'VLAN tag to apply to packets on this interface.',
 637         optional => 1,
 638     },
 639     trunks => {
 640         type => 'string',
 641         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 642         description => 'VLAN trunks to pass through this interface.',
 643         format_description => 'vlanid[;vlanid...]',
 644         optional => 1,
 645     },
 646     firewall => {
 647         type => 'boolean',
 648         description => 'Whether this interface should be protected by the firewall.',
 649         optional => 1,
 650     },
 651     link_down => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be disconnected (like pulling the plug).',
 654         optional => 1,
 655     },
 656 };
 657
 658 my $netdesc = {
 659     optional => 1,
 660     type => 'string', format => $net_fmt,
 661     description => "Specify network devices.",
 662 };
 663
 664 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 665
 666 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 667     $confdesc->{"net$i"} = $netdesc;
 668 }
 669
 670 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 671 sub verify_volume_id_or_qm_path {
 672     my ($volid, $noerr) = @_;
 673
 674     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 675         return $volid;
 676     }
 677
 678     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 679     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 680     if ($@) {
 681         return undef if $noerr;
 682         die $@;
 683     }
 684     return $volid;
 685 }
 686
 687 my $drivename_hash;
 688
 689 my %drivedesc_base = (
 690     volume => { alias => 'file' },
 691     file => {
 692         type => 'string',
 693         format => 'pve-volume-id-or-qm-path',
 694         default_key => 1,
 695         format_description => 'volume',
 696         description => "The drive's backing volume.",
 697     },
 698     media => {
 699         type => 'string',
 700         enum => [qw(cdrom disk)],
 701         description => "The drive's media type.",
 702         default => 'disk',
 703         optional => 1
 704     },
 705     cyls => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific cylinder count.",
 708         optional => 1
 709     },
 710     heads => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific head count.",
 713         optional => 1
 714     },
 715     secs => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific sector count.",
 718         optional => 1
 719     },
 720     trans => {
 721         type => 'string',
 722         enum => [qw(none lba auto)],
 723         description => "Force disk geometry bios translation mode.",
 724         optional => 1,
 725     },
 726     snapshot => {
 727         type => 'boolean',
 728         description => "Whether the drive should be included when making snapshots.",
 729         optional => 1,
 730     },
 731     cache => {
 732         type => 'string',
 733         enum => [qw(none writethrough writeback unsafe directsync)],
 734         description => "The drive's cache mode",
 735         optional => 1,
 736     },
 737     format => {
 738         type => 'string',
 739         format_description => 'image format',
 740         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 741         description => "The drive's backing file's data format.",
 742         optional => 1,
 743     },
 744     size => {
 745         type => 'string',
 746         format => 'disk-size',
 747         format_description => 'DiskSize',
 748         description => "Disk size. This is purely informational and has no effect.",
 749         optional => 1,
 750     },
 751     backup => {
 752         type => 'boolean',
 753         description => "Whether the drive should be included when making backups.",
 754         optional => 1,
 755     },
 756     werror => {
 757         type => 'string',
 758         enum => [qw(enospc ignore report stop)],
 759         description => 'Write error action.',
 760         optional => 1,
 761     },
 762     aio => {
 763         type => 'string',
 764         enum => [qw(native threads)],
 765         description => 'AIO type to use.',
 766         optional => 1,
 767     },
 768     discard => {
 769         type => 'string',
 770         enum => [qw(ignore on)],
 771         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 772         optional => 1,
 773     },
 774     detect_zeroes => {
 775         type => 'boolean',
 776         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 777         optional => 1,
 778     },
 779     serial => {
 780         type => 'string',
 781         format => 'urlencoded',
 782         format_description => 'serial',
 783         maxLength => 20*3, # *3 since it's %xx url enoded
 784         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 785         optional => 1,
 786     }
 787 );
 788
 789 my %rerror_fmt = (
 790     rerror => {
 791         type => 'string',
 792         enum => [qw(ignore report stop)],
 793         description => 'Read error action.',
 794         optional => 1,
 795     },
 796 );
 797
 798 my %iothread_fmt = ( iothread => {
 799         type => 'boolean',
 800         description => "Whether to use iothreads for this drive",
 801         optional => 1,
 802 });
 803
 804 my %model_fmt = (
 805     model => {
 806         type => 'string',
 807         format => 'urlencoded',
 808         format_description => 'model',
 809         maxLength => 40*3, # *3 since it's %xx url enoded
 810         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 811         optional => 1,
 812     },
 813 );
 814
 815 my %queues_fmt = (
 816     queues => {
 817         type => 'integer',
 818         description => "Number of queues.",
 819         minimum => 2,
 820         optional => 1
 821     }
 822 );
 823
 824 my $add_throttle_desc = sub {
 825     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 826     my $d = {
 827         type => $type,
 828         format_description => $unit,
 829         description => "Maximum $what in $longunit.",
 830         optional => 1,
 831     };
 832     $d->{minimum} = $minimum if defined($minimum);
 833     $drivedesc_base{$key} = $d;
 834 };
 835 # throughput: (leaky bucket)
 836 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 837 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 838 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 839 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 840 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 841 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 842 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 843 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 844 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 845
 846 # pools: (pool of IO before throttling starts taking effect)
 847 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 848 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 849 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 850 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 851 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 852 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 853
 854 # burst lengths
 855 $add_throttle_desc->('bps_max_length',  'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 856 $add_throttle_desc->('bps_rd_length',   'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 857 $add_throttle_desc->('bps_wr_length',   'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 858 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 859 $add_throttle_desc->('iops_rd_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 860 $add_throttle_desc->('iops_wr_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 861
 862 my $ide_fmt = {
 863     %drivedesc_base,
 864     %rerror_fmt,
 865     %model_fmt,
 866 };
 867 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 868
 869 my $idedesc = {
 870     optional => 1,
 871     type => 'string', format => $ide_fmt,
 872     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 873 };
 874 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 875
 876 my $scsi_fmt = {
 877     %drivedesc_base,
 878     %iothread_fmt,
 879     %queues_fmt,
 880 };
 881 my $scsidesc = {
 882     optional => 1,
 883     type => 'string', format => $scsi_fmt,
 884     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 885 };
 886 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 887
 888 my $sata_fmt = {
 889     %drivedesc_base,
 890     %rerror_fmt,
 891 };
 892 my $satadesc = {
 893     optional => 1,
 894     type => 'string', format => $sata_fmt,
 895     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 896 };
 897 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 898
 899 my $virtio_fmt = {
 900     %drivedesc_base,
 901     %iothread_fmt,
 902     %rerror_fmt,
 903 };
 904 my $virtiodesc = {
 905     optional => 1,
 906     type => 'string', format => $virtio_fmt,
 907     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 908 };
 909 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 910
 911 my $alldrive_fmt = {
 912     %drivedesc_base,
 913     %rerror_fmt,
 914     %iothread_fmt,
 915     %model_fmt,
 916     %queues_fmt,
 917 };
 918
 919 my $efidisk_fmt = {
 920     volume => { alias => 'file' },
 921     file => {
 922         type => 'string',
 923         format => 'pve-volume-id-or-qm-path',
 924         default_key => 1,
 925         format_description => 'volume',
 926         description => "The drive's backing volume.",
 927     },
 928     format => {
 929         type => 'string',
 930         format_description => 'image format',
 931         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 932         description => "The drive's backing file's data format.",
 933         optional => 1,
 934     },
 935     size => {
 936         type => 'string',
 937         format => 'disk-size',
 938         format_description => 'DiskSize',
 939         description => "Disk size. This is purely informational and has no effect.",
 940         optional => 1,
 941     },
 942 };
 943
 944 my $efidisk_desc = {
 945     optional => 1,
 946     type => 'string', format => $efidisk_fmt,
 947     description => "Configure a Disk for storing EFI vars",
 948 };
 949
 950 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 951
 952 my $usb_fmt = {
 953     host => {
 954         default_key => 1,
 955         type => 'string', format => 'pve-qm-usb-device',
 956         format_description => 'HOSTUSBDEVICE|spice',
 957         description => <<EODESCR,
 958 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 959
 960  'bus-port(.port)*' (decimal numbers) or
 961  'vendor_id:product_id' (hexadeciaml numbers) or
 962  'spice'
 963
 964 You can use the 'lsusb -t' command to list existing usb devices.
 965
 966 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 967
 968 The value 'spice' can be used to add a usb redirection devices for spice.
 969 EODESCR
 970     },
 971     usb3 => {
 972         optional => 1,
 973         type => 'boolean',
 974         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 975         default => 0,
 976     },
 977 };
 978
 979 my $usbdesc = {
 980     optional => 1,
 981     type => 'string', format => $usb_fmt,
 982     description => "Configure an USB device (n is 0 to 4).",
 983 };
 984 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 985
 986 # NOTE: the match-groups of this regex are used in parse_hostpci
 987 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 988 my $hostpci_fmt = {
 989     host => {
 990         default_key => 1,
 991         type => 'string',
 992         pattern => qr/$PCIRE(;$PCIRE)*/,
 993         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 994         description => <<EODESCR,
 995 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 996 of PCI virtual functions of the host. HOSTPCIID syntax is:
 997
 998 'bus:dev.func' (hexadecimal numbers)
 999
1000 You can us the 'lspci' command to list existing PCI devices.
1001 EODESCR
1002     },
1003     rombar => {
1004         type => 'boolean',
1005         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1006         optional => 1,
1007         default => 1,
1008     },
1009     pcie => {
1010         type => 'boolean',
1011         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1012         optional => 1,
1013         default => 0,
1014     },
1015     'x-vga' => {
1016         type => 'boolean',
1017         description =>  "Enable vfio-vga device support.",
1018         optional => 1,
1019         default => 0,
1020     },
1021 };
1022 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1023
1024 my $hostpcidesc = {
1025         optional => 1,
1026         type => 'string', format => 'pve-qm-hostpci',
1027         description => "Map host PCI devices into guest.",
1028         verbose_description =>  <<EODESCR,
1029 Map host PCI devices into guest.
1030
1031 NOTE: This option allows direct access to host hardware. So it is no longer
1032 possible to migrate such machines - use with special care.
1033
1034 CAUTION: Experimental! User reported problems with this option.
1035 EODESCR
1036 };
1037 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1038
1039 my $serialdesc = {
1040         optional => 1,
1041         type => 'string',
1042         pattern => '(/dev/.+|socket)',
1043         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1044         verbose_description =>  <<EODESCR,
1045 Create a serial device inside the VM (n is 0 to 3), and pass through a
1046 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1047 host side (use 'qm terminal' to open a terminal connection).
1048
1049 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1050
1051 CAUTION: Experimental! User reported problems with this option.
1052 EODESCR
1053 };
1054
1055 my $paralleldesc= {
1056         optional => 1,
1057         type => 'string',
1058         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1059         description =>  "Map host parallel devices (n is 0 to 2).",
1060         verbose_description =>  <<EODESCR,
1061 Map host parallel devices (n is 0 to 2).
1062
1063 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1064
1065 CAUTION: Experimental! User reported problems with this option.
1066 EODESCR
1067 };
1068
1069 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1070     $confdesc->{"parallel$i"} = $paralleldesc;
1071 }
1072
1073 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1074     $confdesc->{"serial$i"} = $serialdesc;
1075 }
1076
1077 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1078     $confdesc->{"hostpci$i"} = $hostpcidesc;
1079 }
1080
1081 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1082     $drivename_hash->{"ide$i"} = 1;
1083     $confdesc->{"ide$i"} = $idedesc;
1084 }
1085
1086 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1087     $drivename_hash->{"sata$i"} = 1;
1088     $confdesc->{"sata$i"} = $satadesc;
1089 }
1090
1091 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1092     $drivename_hash->{"scsi$i"} = 1;
1093     $confdesc->{"scsi$i"} = $scsidesc ;
1094 }
1095
1096 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1097     $drivename_hash->{"virtio$i"} = 1;
1098     $confdesc->{"virtio$i"} = $virtiodesc;
1099 }
1100
1101 $drivename_hash->{efidisk0} = 1;
1102 $confdesc->{efidisk0} = $efidisk_desc;
1103
1104 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1105     $confdesc->{"usb$i"} = $usbdesc;
1106 }
1107
1108 my $unuseddesc = {
1109     optional => 1,
1110     type => 'string', format => 'pve-volume-id',
1111     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1112 };
1113
1114 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1115     $confdesc->{"unused$i"} = $unuseddesc;
1116 }
1117
1118 my $kvm_api_version = 0;
1119
1120 sub kvm_version {
1121
1122     return $kvm_api_version if $kvm_api_version;
1123
1124     my $fh = IO::File->new("</dev/kvm") ||
1125         return 0;
1126
1127     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1128         $kvm_api_version = $v;
1129     }
1130
1131     $fh->close();
1132
1133     return  $kvm_api_version;
1134 }
1135
1136 my $kvm_user_version;
1137
1138 sub kvm_user_version {
1139
1140     return $kvm_user_version if $kvm_user_version;
1141
1142     $kvm_user_version = 'unknown';
1143
1144     my $code = sub {
1145         my $line = shift;
1146         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1147             $kvm_user_version = $2;
1148         }
1149     };
1150
1151     eval { run_command("kvm -version", outfunc => $code); };
1152     warn $@ if $@;
1153
1154     return $kvm_user_version;
1155
1156 }
1157
1158 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1159
1160 sub valid_drive_names {
1161     # order is important - used to autoselect boot disk
1162     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1163             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1164             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1165             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1166             'efidisk0');
1167 }
1168
1169 sub is_valid_drivename {
1170     my $dev = shift;
1171
1172     return defined($drivename_hash->{$dev});
1173 }
1174
1175 sub option_exists {
1176     my $key = shift;
1177     return defined($confdesc->{$key});
1178 }
1179
1180 sub nic_models {
1181     return $nic_model_list;
1182 }
1183
1184 sub os_list_description {
1185
1186     return {
1187         other => 'Other',
1188         wxp => 'Windows XP',
1189         w2k => 'Windows 2000',
1190         w2k3 =>, 'Windows 2003',
1191         w2k8 => 'Windows 2008',
1192         wvista => 'Windows Vista',
1193         win7 => 'Windows 7',
1194         win8 => 'Windows 8/2012',
1195         l24 => 'Linux 2.4',
1196         l26 => 'Linux 2.6',
1197     };
1198 }
1199
1200 my $cdrom_path;
1201
1202 sub get_cdrom_path {
1203
1204     return  $cdrom_path if $cdrom_path;
1205
1206     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1207     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1208     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1209 }
1210
1211 sub get_iso_path {
1212     my ($storecfg, $vmid, $cdrom) = @_;
1213
1214     if ($cdrom eq 'cdrom') {
1215         return get_cdrom_path();
1216     } elsif ($cdrom eq 'none') {
1217         return '';
1218     } elsif ($cdrom =~ m|^/|) {
1219         return $cdrom;
1220     } else {
1221         return PVE::Storage::path($storecfg, $cdrom);
1222     }
1223 }
1224
1225 # try to convert old style file names to volume IDs
1226 sub filename_to_volume_id {
1227     my ($vmid, $file, $media) = @_;
1228
1229     if (!($file eq 'none' || $file eq 'cdrom' ||
1230           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1231
1232         return undef if $file =~ m|/|;
1233
1234         if ($media && $media eq 'cdrom') {
1235             $file = "local:iso/$file";
1236         } else {
1237             $file = "local:$vmid/$file";
1238         }
1239     }
1240
1241     return $file;
1242 }
1243
1244 sub verify_media_type {
1245     my ($opt, $vtype, $media) = @_;
1246
1247     return if !$media;
1248
1249     my $etype;
1250     if ($media eq 'disk') {
1251         $etype = 'images';
1252     } elsif ($media eq 'cdrom') {
1253         $etype = 'iso';
1254     } else {
1255         die "internal error";
1256     }
1257
1258     return if ($vtype eq $etype);
1259
1260     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1261 }
1262
1263 sub cleanup_drive_path {
1264     my ($opt, $storecfg, $drive) = @_;
1265
1266     # try to convert filesystem paths to volume IDs
1267
1268     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1269         ($drive->{file} !~ m|^/dev/.+|) &&
1270         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1271         ($drive->{file} !~ m/^\d+$/)) {
1272         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1273         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1274         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1275         verify_media_type($opt, $vtype, $drive->{media});
1276         $drive->{file} = $volid;
1277     }
1278
1279     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1280 }
1281
1282 sub parse_hotplug_features {
1283     my ($data) = @_;
1284
1285     my $res = {};
1286
1287     return $res if $data eq '0';
1288
1289     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1290
1291     foreach my $feature (PVE::Tools::split_list($data)) {
1292         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1293             $res->{$1} = 1;
1294         } else {
1295             die "invalid hotplug feature '$feature'\n";
1296         }
1297     }
1298     return $res;
1299 }
1300
1301 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1302 sub pve_verify_hotplug_features {
1303     my ($value, $noerr) = @_;
1304
1305     return $value if parse_hotplug_features($value);
1306
1307     return undef if $noerr;
1308
1309     die "unable to parse hotplug option\n";
1310 }
1311
1312 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1313 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1314 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1315 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1316 #        [,iothread=on][,serial=serial][,model=model]
1317
1318 sub parse_drive {
1319     my ($key, $data) = @_;
1320
1321     my ($interface, $index);
1322
1323     if ($key =~ m/^([^\d]+)(\d+)$/) {
1324         $interface = $1;
1325         $index = $2;
1326     } else {
1327         return undef;
1328     }
1329
1330     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1331                                      : $confdesc->{$key}->{format};
1332     if (!$desc) {
1333         warn "invalid drive key: $key\n";
1334         return undef;
1335     }
1336     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1337     return undef if !$res;
1338     $res->{interface} = $interface;
1339     $res->{index} = $index;
1340
1341     my $error = 0;
1342     foreach my $opt (qw(bps bps_rd bps_wr)) {
1343         if (my $bps = defined(delete $res->{$opt})) {
1344             if (defined($res->{"m$opt"})) {
1345                 warn "both $opt and m$opt specified\n";
1346                 ++$error;
1347                 next;
1348             }
1349             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1350         }
1351     }
1352
1353     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1354     for my $requirement (
1355         [bps_max_length => 'mbps_max'],
1356         [bps_rd_max_length => 'mbps_rd_max'],
1357         [bps_wr_max_length => 'mbps_wr_max'],
1358         [iops_max_length => 'iops_max'],
1359         [iops_rd_max_length => 'iops_rd_max'],
1360         [iops_wr_max_length => 'iops_wr_max']) {
1361         my ($option, $requires) = @$requirement;
1362         if ($res->{$option} && !$res->{$requires}) {
1363             warn "$option requires $requires\n";
1364             ++$error;
1365         }
1366     }
1367
1368     return undef if $error;
1369
1370     return undef if $res->{mbps_rd} && $res->{mbps};
1371     return undef if $res->{mbps_wr} && $res->{mbps};
1372     return undef if $res->{iops_rd} && $res->{iops};
1373     return undef if $res->{iops_wr} && $res->{iops};
1374
1375     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1376         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1377         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1378         return undef if $res->{interface} eq 'virtio';
1379     }
1380
1381     if (my $size = $res->{size}) {
1382         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1383     }
1384
1385     return $res;
1386 }
1387
1388 sub print_drive {
1389     my ($vmid, $drive) = @_;
1390     my $data = { %$drive };
1391     delete $data->{$_} for qw(index interface);
1392     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1393 }
1394
1395 sub scsi_inquiry {
1396     my($fh, $noerr) = @_;
1397
1398     my $SG_IO = 0x2285;
1399     my $SG_GET_VERSION_NUM = 0x2282;
1400
1401     my $versionbuf = "\x00" x 8;
1402     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1403     if (!$ret) {
1404         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1405         return undef;
1406     }
1407     my $version = unpack("I", $versionbuf);
1408     if ($version < 30000) {
1409         die "scsi generic interface too old\n"  if !$noerr;
1410         return undef;
1411     }
1412
1413     my $buf = "\x00" x 36;
1414     my $sensebuf = "\x00" x 8;
1415     my $cmd = pack("C x3 C x1", 0x12, 36);
1416
1417     # see /usr/include/scsi/sg.h
1418     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1419
1420     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1421                       length($sensebuf), 0, length($buf), $buf,
1422                       $cmd, $sensebuf, 6000);
1423
1424     $ret = ioctl($fh, $SG_IO, $packet);
1425     if (!$ret) {
1426         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1427         return undef;
1428     }
1429
1430     my @res = unpack($sg_io_hdr_t, $packet);
1431     if ($res[17] || $res[18]) {
1432         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1433         return undef;
1434     }
1435
1436     my $res = {};
1437     (my $byte0, my $byte1, $res->{vendor},
1438      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1439
1440     $res->{removable} = $byte1 & 128 ? 1 : 0;
1441     $res->{type} = $byte0 & 31;
1442
1443     return $res;
1444 }
1445
1446 sub path_is_scsi {
1447     my ($path) = @_;
1448
1449     my $fh = IO::File->new("+<$path") || return undef;
1450     my $res = scsi_inquiry($fh, 1);
1451     close($fh);
1452
1453     return $res;
1454 }
1455
1456 sub machine_type_is_q35 {
1457     my ($conf) = @_;
1458
1459     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1460 }
1461
1462 sub print_tabletdevice_full {
1463     my ($conf) = @_;
1464
1465     my $q35 = machine_type_is_q35($conf);
1466
1467     # we use uhci for old VMs because tablet driver was buggy in older qemu
1468     my $usbbus = $q35 ? "ehci" : "uhci";
1469
1470     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1471 }
1472
1473 sub print_drivedevice_full {
1474     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1475
1476     my $device = '';
1477     my $maxdev = 0;
1478
1479     if ($drive->{interface} eq 'virtio') {
1480         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1481         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1482         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1483     } elsif ($drive->{interface} eq 'scsi') {
1484
1485         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1486         my $unit = $drive->{index} % $maxdev;
1487         my $devicetype = 'hd';
1488         my $path = '';
1489         if (drive_is_cdrom($drive)) {
1490             $devicetype = 'cd';
1491         } else {
1492             if ($drive->{file} =~ m|^/|) {
1493                 $path = $drive->{file};
1494                 if (my $info = path_is_scsi($path)) {
1495                     if ($info->{type} == 0) {
1496                         $devicetype = 'block';
1497                     } elsif ($info->{type} == 1) { # tape
1498                         $devicetype = 'generic';
1499                     }
1500                 }
1501             } else {
1502                  $path = PVE::Storage::path($storecfg, $drive->{file});
1503             }
1504
1505             if($path =~ m/^iscsi\:\/\//){
1506                 $devicetype = 'generic';
1507             }
1508         }
1509
1510         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1511             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1512         } else {
1513             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1514         }
1515
1516     } elsif ($drive->{interface} eq 'ide'){
1517         $maxdev = 2;
1518         my $controller = int($drive->{index} / $maxdev);
1519         my $unit = $drive->{index} % $maxdev;
1520         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1521
1522         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1523         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1524             $model = URI::Escape::uri_unescape($model);
1525             $device .= ",model=$model";
1526         }
1527     } elsif ($drive->{interface} eq 'sata'){
1528         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1529         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1530         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1531     } elsif ($drive->{interface} eq 'usb') {
1532         die "implement me";
1533         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1534     } else {
1535         die "unsupported interface type";
1536     }
1537
1538     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1539
1540     return $device;
1541 }
1542
1543 sub get_initiator_name {
1544     my $initiator;
1545
1546     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1547     while (defined(my $line = <$fh>)) {
1548         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1549         $initiator = $1;
1550         last;
1551     }
1552     $fh->close();
1553
1554     return $initiator;
1555 }
1556
1557 sub print_drive_full {
1558     my ($storecfg, $vmid, $drive) = @_;
1559
1560     my $path;
1561     my $volid = $drive->{file};
1562     my $format;
1563
1564     if (drive_is_cdrom($drive)) {
1565         $path = get_iso_path($storecfg, $vmid, $volid);
1566     } else {
1567         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1568         if ($storeid) {
1569             $path = PVE::Storage::path($storecfg, $volid);
1570             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1571             $format = qemu_img_format($scfg, $volname);
1572         } else {
1573             $path = $volid;
1574             $format = "raw";
1575         }
1576    }
1577
1578     my $opts = '';
1579     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1580     foreach my $o (@qemu_drive_options) {
1581         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1582     }
1583     if (my $serial = $drive->{serial}) {
1584         $serial = URI::Escape::uri_unescape($serial);
1585         $opts .= ",serial=$serial";
1586     }
1587
1588     $opts .= ",format=$format" if $format && !$drive->{format};
1589
1590     foreach my $o (qw(bps bps_rd bps_wr)) {
1591         my $v = $drive->{"m$o"};
1592         $opts .= ",$o=" . int($v*1024*1024) if $v;
1593     }
1594
1595     my $cache_direct = 0;
1596
1597     if (my $cache = $drive->{cache}) {
1598         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1599     } elsif (!drive_is_cdrom($drive)) {
1600         $opts .= ",cache=none";
1601         $cache_direct = 1;
1602     }
1603
1604     # aio native works only with O_DIRECT
1605     if (!$drive->{aio}) {
1606         if($cache_direct) {
1607             $opts .= ",aio=native";
1608         } else {
1609             $opts .= ",aio=threads";
1610         }
1611     }
1612
1613     if (!drive_is_cdrom($drive)) {
1614         my $detectzeroes;
1615         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1616             $detectzeroes = 'off';
1617         } elsif ($drive->{discard}) {
1618             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1619         } else {
1620             # This used to be our default with discard not being specified:
1621             $detectzeroes = 'on';
1622         }
1623         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1624     }
1625
1626     my $pathinfo = $path ? "file=$path," : '';
1627
1628     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1629 }
1630
1631 sub print_netdevice_full {
1632     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1633
1634     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1635
1636     my $device = $net->{model};
1637     if ($net->{model} eq 'virtio') {
1638          $device = 'virtio-net-pci';
1639      };
1640
1641     my $pciaddr = print_pci_addr("$netid", $bridges);
1642     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1643     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1644         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1645         my $vectors = $net->{queues} * 2 + 2;
1646         $tmpstr .= ",vectors=$vectors,mq=on";
1647     }
1648     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1649
1650     if ($use_old_bios_files) {
1651         my $romfile;
1652         if ($device eq 'virtio-net-pci') {
1653             $romfile = 'pxe-virtio.rom';
1654         } elsif ($device eq 'e1000') {
1655             $romfile = 'pxe-e1000.rom';
1656         } elsif ($device eq 'ne2k') {
1657             $romfile = 'pxe-ne2k_pci.rom';
1658         } elsif ($device eq 'pcnet') {
1659             $romfile = 'pxe-pcnet.rom';
1660         } elsif ($device eq 'rtl8139') {
1661             $romfile = 'pxe-rtl8139.rom';
1662         }
1663         $tmpstr .= ",romfile=$romfile" if $romfile;
1664     }
1665
1666     return $tmpstr;
1667 }
1668
1669 sub print_netdev_full {
1670     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1671
1672     my $i = '';
1673     if ($netid =~ m/^net(\d+)$/) {
1674         $i = int($1);
1675     }
1676
1677     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1678
1679     my $ifname = "tap${vmid}i$i";
1680
1681     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1682     die "interface name '$ifname' is too long (max 15 character)\n"
1683         if length($ifname) >= 16;
1684
1685     my $vhostparam = '';
1686     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1687
1688     my $vmname = $conf->{name} || "vm$vmid";
1689
1690     my $netdev = "";
1691     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1692
1693     if ($net->{bridge}) {
1694         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1695     } else {
1696         $netdev = "type=user,id=$netid,hostname=$vmname";
1697     }
1698
1699     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1700
1701     return $netdev;
1702 }
1703
1704
1705 sub print_cpu_device {
1706     my ($conf, $id) = @_;
1707
1708     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1709     my $cpu = $nokvm ? "qemu64" : "kvm64";
1710     if (my $cputype = $conf->{cpu}) {
1711         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1712             or die "Cannot parse cpu description: $cputype\n";
1713         $cpu = $cpuconf->{cputype};
1714     }
1715
1716     my $sockets = 1;
1717     $sockets = $conf->{sockets} if  $conf->{sockets};
1718     my $cores = $conf->{cores} || 1;
1719
1720     my $current_core = ($id - 1) % $cores;
1721     my $current_socket = int(($id - $current_core)/$cores);
1722
1723     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1724 }
1725
1726 sub drive_is_cdrom {
1727     my ($drive) = @_;
1728
1729     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1730
1731 }
1732
1733 sub parse_number_sets {
1734     my ($set) = @_;
1735     my $res = [];
1736     foreach my $part (split(/;/, $set)) {
1737         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1738             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1739             push @$res, [ $1, $2 ];
1740         } else {
1741             die "invalid range: $part\n";
1742         }
1743     }
1744     return $res;
1745 }
1746
1747 sub parse_numa {
1748     my ($data) = @_;
1749
1750     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1751     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1752     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1753     return $res;
1754 }
1755
1756 sub parse_hostpci {
1757     my ($value) = @_;
1758
1759     return undef if !$value;
1760
1761     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1762
1763     my @idlist = split(/;/, $res->{host});
1764     delete $res->{host};
1765     foreach my $id (@idlist) {
1766         if ($id =~ /^$PCIRE$/) {
1767             if (defined($2)) {
1768                 push @{$res->{pciid}}, { id => $1, function => $2 };
1769             } else {
1770                 my $pcidevices = lspci($1);
1771                 $res->{pciid} = $pcidevices->{$1};
1772             }
1773         } else {
1774             # should have been caught by parse_property_string already
1775             die "failed to parse PCI id: $id\n";
1776         }
1777     }
1778     return $res;
1779 }
1780
1781 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1782 sub parse_net {
1783     my ($data) = @_;
1784
1785     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1786     if ($@) {
1787         warn $@;
1788         return undef;
1789     }
1790     if (!defined($res->{macaddr})) {
1791         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1792         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1793     }
1794     return $res;
1795 }
1796
1797 sub print_net {
1798     my $net = shift;
1799
1800     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1801 }
1802
1803 sub add_random_macs {
1804     my ($settings) = @_;
1805
1806     foreach my $opt (keys %$settings) {
1807         next if $opt !~ m/^net(\d+)$/;
1808         my $net = parse_net($settings->{$opt});
1809         next if !$net;
1810         $settings->{$opt} = print_net($net);
1811     }
1812 }
1813
1814 sub vm_is_volid_owner {
1815     my ($storecfg, $vmid, $volid) = @_;
1816
1817     if ($volid !~  m|^/|) {
1818         my ($path, $owner);
1819         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1820         if ($owner && ($owner == $vmid)) {
1821             return 1;
1822         }
1823     }
1824
1825     return undef;
1826 }
1827
1828 sub split_flagged_list {
1829     my $text = shift || '';
1830     $text =~ s/[,;]/ /g;
1831     $text =~ s/^\s+//;
1832     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1833 }
1834
1835 sub join_flagged_list {
1836     my ($how, $lst) = @_;
1837     join $how, map { $lst->{$_} . $_ } keys %$lst;
1838 }
1839
1840 sub vmconfig_delete_pending_option {
1841     my ($conf, $key, $force) = @_;
1842
1843     delete $conf->{pending}->{$key};
1844     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1845     $pending_delete_hash->{$key} = $force ? '!' : '';
1846     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1847 }
1848
1849 sub vmconfig_undelete_pending_option {
1850     my ($conf, $key) = @_;
1851
1852     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1853     delete $pending_delete_hash->{$key};
1854
1855     if (%$pending_delete_hash) {
1856         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1857     } else {
1858         delete $conf->{pending}->{delete};
1859     }
1860 }
1861
1862 sub vmconfig_register_unused_drive {
1863     my ($storecfg, $vmid, $conf, $drive) = @_;
1864
1865     if (!drive_is_cdrom($drive)) {
1866         my $volid = $drive->{file};
1867         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1868             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1869         }
1870     }
1871 }
1872
1873 sub vmconfig_cleanup_pending {
1874     my ($conf) = @_;
1875
1876     # remove pending changes when nothing changed
1877     my $changes;
1878     foreach my $opt (keys %{$conf->{pending}}) {
1879         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1880             $changes = 1;
1881             delete $conf->{pending}->{$opt};
1882         }
1883     }
1884
1885     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1886     my $pending_delete_hash = {};
1887     while (my ($opt, $force) = each %$current_delete_hash) {
1888         if (defined($conf->{$opt})) {
1889             $pending_delete_hash->{$opt} = $force;
1890         } else {
1891             $changes = 1;
1892         }
1893     }
1894
1895     if (%$pending_delete_hash) {
1896         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1897     } else {
1898         delete $conf->{pending}->{delete};
1899     }
1900
1901     return $changes;
1902 }
1903
1904 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1905 my $smbios1_fmt = {
1906     uuid => {
1907         type => 'string',
1908         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1909         format_description => 'UUID',
1910         description => "Set SMBIOS1 UUID.",
1911         optional => 1,
1912     },
1913     version => {
1914         type => 'string',
1915         pattern => '\S+',
1916         format_description => 'string',
1917         description => "Set SMBIOS1 version.",
1918         optional => 1,
1919     },
1920     serial => {
1921         type => 'string',
1922         pattern => '\S+',
1923         format_description => 'string',
1924         description => "Set SMBIOS1 serial number.",
1925         optional => 1,
1926     },
1927     manufacturer => {
1928         type => 'string',
1929         pattern => '\S+',
1930         format_description => 'string',
1931         description => "Set SMBIOS1 manufacturer.",
1932         optional => 1,
1933     },
1934     product => {
1935         type => 'string',
1936         pattern => '\S+',
1937         format_description => 'string',
1938         description => "Set SMBIOS1 product ID.",
1939         optional => 1,
1940     },
1941     sku => {
1942         type => 'string',
1943         pattern => '\S+',
1944         format_description => 'string',
1945         description => "Set SMBIOS1 SKU string.",
1946         optional => 1,
1947     },
1948     family => {
1949         type => 'string',
1950         pattern => '\S+',
1951         format_description => 'string',
1952         description => "Set SMBIOS1 family string.",
1953         optional => 1,
1954     },
1955 };
1956
1957 sub parse_smbios1 {
1958     my ($data) = @_;
1959
1960     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1961     warn $@ if $@;
1962     return $res;
1963 }
1964
1965 sub print_smbios1 {
1966     my ($smbios1) = @_;
1967     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1968 }
1969
1970 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1971
1972 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1973 sub verify_bootdisk {
1974     my ($value, $noerr) = @_;
1975
1976     return $value if is_valid_drivename($value);
1977
1978     return undef if $noerr;
1979
1980     die "invalid boot disk '$value'\n";
1981 }
1982
1983 sub parse_watchdog {
1984     my ($value) = @_;
1985
1986     return undef if !$value;
1987
1988     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1989     warn $@ if $@;
1990     return $res;
1991 }
1992
1993 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1994 sub verify_usb_device {
1995     my ($value, $noerr) = @_;
1996
1997     return $value if parse_usb_device($value);
1998
1999     return undef if $noerr;
2000
2001     die "unable to parse usb device\n";
2002 }
2003
2004 # add JSON properties for create and set function
2005 sub json_config_properties {
2006     my $prop = shift;
2007
2008     foreach my $opt (keys %$confdesc) {
2009         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2010         $prop->{$opt} = $confdesc->{$opt};
2011     }
2012
2013     return $prop;
2014 }
2015
2016 sub check_type {
2017     my ($key, $value) = @_;
2018
2019     die "unknown setting '$key'\n" if !$confdesc->{$key};
2020
2021     my $type = $confdesc->{$key}->{type};
2022
2023     if (!defined($value)) {
2024         die "got undefined value\n";
2025     }
2026
2027     if ($value =~ m/[\n\r]/) {
2028         die "property contains a line feed\n";
2029     }
2030
2031     if ($type eq 'boolean') {
2032         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2033         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2034         die "type check ('boolean') failed - got '$value'\n";
2035     } elsif ($type eq 'integer') {
2036         return int($1) if $value =~ m/^(\d+)$/;
2037         die "type check ('integer') failed - got '$value'\n";
2038     } elsif ($type eq 'number') {
2039         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2040         die "type check ('number') failed - got '$value'\n";
2041     } elsif ($type eq 'string') {
2042         if (my $fmt = $confdesc->{$key}->{format}) {
2043             PVE::JSONSchema::check_format($fmt, $value);
2044             return $value;
2045         }
2046         $value =~ s/^\"(.*)\"$/$1/;
2047         return $value;
2048     } else {
2049         die "internal error"
2050     }
2051 }
2052
2053 sub check_iommu_support{
2054     #fixme : need to check IOMMU support
2055     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2056
2057     my $iommu=1;
2058     return $iommu;
2059
2060 }
2061
2062 sub touch_config {
2063     my ($vmid) = @_;
2064
2065     my $conf = PVE::QemuConfig->config_file($vmid);
2066     utime undef, undef, $conf;
2067 }
2068
2069 sub destroy_vm {
2070     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2071
2072     my $conffile = PVE::QemuConfig->config_file($vmid);
2073
2074     my $conf = PVE::QemuConfig->load_config($vmid);
2075
2076     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2077
2078     # only remove disks owned by this VM
2079     foreach_drive($conf, sub {
2080         my ($ds, $drive) = @_;
2081
2082         return if drive_is_cdrom($drive);
2083
2084         my $volid = $drive->{file};
2085
2086         return if !$volid || $volid =~ m|^/|;
2087
2088         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2089         return if !$path || !$owner || ($owner != $vmid);
2090
2091         PVE::Storage::vdisk_free($storecfg, $volid);
2092     });
2093
2094     if ($keep_empty_config) {
2095         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2096     } else {
2097         unlink $conffile;
2098     }
2099
2100     # also remove unused disk
2101     eval {
2102         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2103
2104         eval {
2105             PVE::Storage::foreach_volid($dl, sub {
2106                 my ($volid, $sid, $volname, $d) = @_;
2107                 PVE::Storage::vdisk_free($storecfg, $volid);
2108             });
2109         };
2110         warn $@ if $@;
2111
2112     };
2113     warn $@ if $@;
2114 }
2115
2116 sub parse_vm_config {
2117     my ($filename, $raw) = @_;
2118
2119     return undef if !defined($raw);
2120
2121     my $res = {
2122         digest => Digest::SHA::sha1_hex($raw),
2123         snapshots => {},
2124         pending => {},
2125     };
2126
2127     $filename =~ m|/qemu-server/(\d+)\.conf$|
2128         || die "got strange filename '$filename'";
2129
2130     my $vmid = $1;
2131
2132     my $conf = $res;
2133     my $descr;
2134     my $section = '';
2135
2136     my @lines = split(/\n/, $raw);
2137     foreach my $line (@lines) {
2138         next if $line =~ m/^\s*$/;
2139
2140         if ($line =~ m/^\[PENDING\]\s*$/i) {
2141             $section = 'pending';
2142             if (defined($descr)) {
2143                 $descr =~ s/\s+$//;
2144                 $conf->{description} = $descr;
2145             }
2146             $descr = undef;
2147             $conf = $res->{$section} = {};
2148             next;
2149
2150         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2151             $section = $1;
2152             if (defined($descr)) {
2153                 $descr =~ s/\s+$//;
2154                 $conf->{description} = $descr;
2155             }
2156             $descr = undef;
2157             $conf = $res->{snapshots}->{$section} = {};
2158             next;
2159         }
2160
2161         if ($line =~ m/^\#(.*)\s*$/) {
2162             $descr = '' if !defined($descr);
2163             $descr .= PVE::Tools::decode_text($1) . "\n";
2164             next;
2165         }
2166
2167         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2168             $descr = '' if !defined($descr);
2169             $descr .= PVE::Tools::decode_text($2);
2170         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2171             $conf->{snapstate} = $1;
2172         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2173             my $key = $1;
2174             my $value = $2;
2175             $conf->{$key} = $value;
2176         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2177             my $value = $1;
2178             if ($section eq 'pending') {
2179                 $conf->{delete} = $value; # we parse this later
2180             } else {
2181                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2182             }
2183         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2184             my $key = $1;
2185             my $value = $2;
2186             eval { $value = check_type($key, $value); };
2187             if ($@) {
2188                 warn "vm $vmid - unable to parse value of '$key' - $@";
2189             } else {
2190                 $key = 'ide2' if $key eq 'cdrom';
2191                 my $fmt = $confdesc->{$key}->{format};
2192                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2193                     my $v = parse_drive($key, $value);
2194                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2195                         $v->{file} = $volid;
2196                         $value = print_drive($vmid, $v);
2197                     } else {
2198                         warn "vm $vmid - unable to parse value of '$key'\n";
2199                         next;
2200                     }
2201                 }
2202
2203                 $conf->{$key} = $value;
2204             }
2205         }
2206     }
2207
2208     if (defined($descr)) {
2209         $descr =~ s/\s+$//;
2210         $conf->{description} = $descr;
2211     }
2212     delete $res->{snapstate}; # just to be sure
2213
2214     return $res;
2215 }
2216
2217 sub write_vm_config {
2218     my ($filename, $conf) = @_;
2219
2220     delete $conf->{snapstate}; # just to be sure
2221
2222     if ($conf->{cdrom}) {
2223         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2224         $conf->{ide2} = $conf->{cdrom};
2225         delete $conf->{cdrom};
2226     }
2227
2228     # we do not use 'smp' any longer
2229     if ($conf->{sockets}) {
2230         delete $conf->{smp};
2231     } elsif ($conf->{smp}) {
2232         $conf->{sockets} = $conf->{smp};
2233         delete $conf->{cores};
2234         delete $conf->{smp};
2235     }
2236
2237     my $used_volids = {};
2238
2239     my $cleanup_config = sub {
2240         my ($cref, $pending, $snapname) = @_;
2241
2242         foreach my $key (keys %$cref) {
2243             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2244                 $key eq 'snapstate' || $key eq 'pending';
2245             my $value = $cref->{$key};
2246             if ($key eq 'delete') {
2247                 die "propertry 'delete' is only allowed in [PENDING]\n"
2248                     if !$pending;
2249                 # fixme: check syntax?
2250                 next;
2251             }
2252             eval { $value = check_type($key, $value); };
2253             die "unable to parse value of '$key' - $@" if $@;
2254
2255             $cref->{$key} = $value;
2256
2257             if (!$snapname && is_valid_drivename($key)) {
2258                 my $drive = parse_drive($key, $value);
2259                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2260             }
2261         }
2262     };
2263
2264     &$cleanup_config($conf);
2265
2266     &$cleanup_config($conf->{pending}, 1);
2267
2268     foreach my $snapname (keys %{$conf->{snapshots}}) {
2269         die "internal error" if $snapname eq 'pending';
2270         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2271     }
2272
2273     # remove 'unusedX' settings if we re-add a volume
2274     foreach my $key (keys %$conf) {
2275         my $value = $conf->{$key};
2276         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2277             delete $conf->{$key};
2278         }
2279     }
2280
2281     my $generate_raw_config = sub {
2282         my ($conf, $pending) = @_;
2283
2284         my $raw = '';
2285
2286         # add description as comment to top of file
2287         if (defined(my $descr = $conf->{description})) {
2288             if ($descr) {
2289                 foreach my $cl (split(/\n/, $descr)) {
2290                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2291                 }
2292             } else {
2293                 $raw .= "#\n" if $pending;
2294             }
2295         }
2296
2297         foreach my $key (sort keys %$conf) {
2298             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2299             $raw .= "$key: $conf->{$key}\n";
2300         }
2301         return $raw;
2302     };
2303
2304     my $raw = &$generate_raw_config($conf);
2305
2306     if (scalar(keys %{$conf->{pending}})){
2307         $raw .= "\n[PENDING]\n";
2308         $raw .= &$generate_raw_config($conf->{pending}, 1);
2309     }
2310
2311     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2312         $raw .= "\n[$snapname]\n";
2313         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2314     }
2315
2316     return $raw;
2317 }
2318
2319 sub load_defaults {
2320
2321     my $res = {};
2322
2323     # we use static defaults from our JSON schema configuration
2324     foreach my $key (keys %$confdesc) {
2325         if (defined(my $default = $confdesc->{$key}->{default})) {
2326             $res->{$key} = $default;
2327         }
2328     }
2329
2330     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2331     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2332
2333     return $res;
2334 }
2335
2336 sub config_list {
2337     my $vmlist = PVE::Cluster::get_vmlist();
2338     my $res = {};
2339     return $res if !$vmlist || !$vmlist->{ids};
2340     my $ids = $vmlist->{ids};
2341
2342     foreach my $vmid (keys %$ids) {
2343         my $d = $ids->{$vmid};
2344         next if !$d->{node} || $d->{node} ne $nodename;
2345         next if !$d->{type} || $d->{type} ne 'qemu';
2346         $res->{$vmid}->{exists} = 1;
2347     }
2348     return $res;
2349 }
2350
2351 # test if VM uses local resources (to prevent migration)
2352 sub check_local_resources {
2353     my ($conf, $noerr) = @_;
2354
2355     my $loc_res = 0;
2356
2357     $loc_res = 1 if $conf->{hostusb}; # old syntax
2358     $loc_res = 1 if $conf->{hostpci}; # old syntax
2359
2360     foreach my $k (keys %$conf) {
2361         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2362         # sockets are safe: they will recreated be on the target side post-migrate
2363         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2364         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2365     }
2366
2367     die "VM uses local resources\n" if $loc_res && !$noerr;
2368
2369     return $loc_res;
2370 }
2371
2372 # check if used storages are available on all nodes (use by migrate)
2373 sub check_storage_availability {
2374     my ($storecfg, $conf, $node) = @_;
2375
2376     foreach_drive($conf, sub {
2377         my ($ds, $drive) = @_;
2378
2379         my $volid = $drive->{file};
2380         return if !$volid;
2381
2382         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2383         return if !$sid;
2384
2385         # check if storage is available on both nodes
2386         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2387         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2388    });
2389 }
2390
2391 # list nodes where all VM images are available (used by has_feature API)
2392 sub shared_nodes {
2393     my ($conf, $storecfg) = @_;
2394
2395     my $nodelist = PVE::Cluster::get_nodelist();
2396     my $nodehash = { map { $_ => 1 } @$nodelist };
2397     my $nodename = PVE::INotify::nodename();
2398
2399     foreach_drive($conf, sub {
2400         my ($ds, $drive) = @_;
2401
2402         my $volid = $drive->{file};
2403         return if !$volid;
2404
2405         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2406         if ($storeid) {
2407             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2408             if ($scfg->{disable}) {
2409                 $nodehash = {};
2410             } elsif (my $avail = $scfg->{nodes}) {
2411                 foreach my $node (keys %$nodehash) {
2412                     delete $nodehash->{$node} if !$avail->{$node};
2413                 }
2414             } elsif (!$scfg->{shared}) {
2415                 foreach my $node (keys %$nodehash) {
2416                     delete $nodehash->{$node} if $node ne $nodename
2417                 }
2418             }
2419         }
2420     });
2421
2422     return $nodehash
2423 }
2424
2425 sub check_cmdline {
2426     my ($pidfile, $pid) = @_;
2427
2428     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2429     if (defined($fh)) {
2430         my $line = <$fh>;
2431         $fh->close;
2432         return undef if !$line;
2433         my @param = split(/\0/, $line);
2434
2435         my $cmd = $param[0];
2436         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2437
2438         for (my $i = 0; $i < scalar (@param); $i++) {
2439             my $p = $param[$i];
2440             next if !$p;
2441             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2442                 my $p = $param[$i+1];
2443                 return 1 if $p && ($p eq $pidfile);
2444                 return undef;
2445             }
2446         }
2447     }
2448     return undef;
2449 }
2450
2451 sub check_running {
2452     my ($vmid, $nocheck, $node) = @_;
2453
2454     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2455
2456     die "unable to find configuration file for VM $vmid - no such machine\n"
2457         if !$nocheck && ! -f $filename;
2458
2459     my $pidfile = pidfile_name($vmid);
2460
2461     if (my $fd = IO::File->new("<$pidfile")) {
2462         my $st = stat($fd);
2463         my $line = <$fd>;
2464         close($fd);
2465
2466         my $mtime = $st->mtime;
2467         if ($mtime > time()) {
2468             warn "file '$filename' modified in future\n";
2469         }
2470
2471         if ($line =~ m/^(\d+)$/) {
2472             my $pid = $1;
2473             if (check_cmdline($pidfile, $pid)) {
2474                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2475                     return $pid;
2476                 }
2477             }
2478         }
2479     }
2480
2481     return undef;
2482 }
2483
2484 sub vzlist {
2485
2486     my $vzlist = config_list();
2487
2488     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2489
2490     while (defined(my $de = $fd->read)) {
2491         next if $de !~ m/^(\d+)\.pid$/;
2492         my $vmid = $1;
2493         next if !defined($vzlist->{$vmid});
2494         if (my $pid = check_running($vmid)) {
2495             $vzlist->{$vmid}->{pid} = $pid;
2496         }
2497     }
2498
2499     return $vzlist;
2500 }
2501
2502 sub disksize {
2503     my ($storecfg, $conf) = @_;
2504
2505     my $bootdisk = $conf->{bootdisk};
2506     return undef if !$bootdisk;
2507     return undef if !is_valid_drivename($bootdisk);
2508
2509     return undef if !$conf->{$bootdisk};
2510
2511     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2512     return undef if !defined($drive);
2513
2514     return undef if drive_is_cdrom($drive);
2515
2516     my $volid = $drive->{file};
2517     return undef if !$volid;
2518
2519     return $drive->{size};
2520 }
2521
2522 my $last_proc_pid_stat;
2523
2524 # get VM status information
2525 # This must be fast and should not block ($full == false)
2526 # We only query KVM using QMP if $full == true (this can be slow)
2527 sub vmstatus {
2528     my ($opt_vmid, $full) = @_;
2529
2530     my $res = {};
2531
2532     my $storecfg = PVE::Storage::config();
2533
2534     my $list = vzlist();
2535     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2536
2537     my $cpucount = $cpuinfo->{cpus} || 1;
2538
2539     foreach my $vmid (keys %$list) {
2540         next if $opt_vmid && ($vmid ne $opt_vmid);
2541
2542         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2543         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2544
2545         my $d = {};
2546         $d->{pid} = $list->{$vmid}->{pid};
2547
2548         # fixme: better status?
2549         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2550
2551         my $size = disksize($storecfg, $conf);
2552         if (defined($size)) {
2553             $d->{disk} = 0; # no info available
2554             $d->{maxdisk} = $size;
2555         } else {
2556             $d->{disk} = 0;
2557             $d->{maxdisk} = 0;
2558         }
2559
2560         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2561         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2562         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2563
2564         $d->{name} = $conf->{name} || "VM $vmid";
2565         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2566
2567         if ($conf->{balloon}) {
2568             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2569             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2570         }
2571
2572         $d->{uptime} = 0;
2573         $d->{cpu} = 0;
2574         $d->{mem} = 0;
2575
2576         $d->{netout} = 0;
2577         $d->{netin} = 0;
2578
2579         $d->{diskread} = 0;
2580         $d->{diskwrite} = 0;
2581
2582         $d->{template} = PVE::QemuConfig->is_template($conf);
2583
2584         $res->{$vmid} = $d;
2585     }
2586
2587     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2588     foreach my $dev (keys %$netdev) {
2589         next if $dev !~ m/^tap([1-9]\d*)i/;
2590         my $vmid = $1;
2591         my $d = $res->{$vmid};
2592         next if !$d;
2593
2594         $d->{netout} += $netdev->{$dev}->{receive};
2595         $d->{netin} += $netdev->{$dev}->{transmit};
2596
2597         if ($full) {
2598             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2599             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2600         }
2601
2602     }
2603
2604     my $ctime = gettimeofday;
2605
2606     foreach my $vmid (keys %$list) {
2607
2608         my $d = $res->{$vmid};
2609         my $pid = $d->{pid};
2610         next if !$pid;
2611
2612         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2613         next if !$pstat; # not running
2614
2615         my $used = $pstat->{utime} + $pstat->{stime};
2616
2617         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2618
2619         if ($pstat->{vsize}) {
2620             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2621         }
2622
2623         my $old = $last_proc_pid_stat->{$pid};
2624         if (!$old) {
2625             $last_proc_pid_stat->{$pid} = {
2626                 time => $ctime,
2627                 used => $used,
2628                 cpu => 0,
2629             };
2630             next;
2631         }
2632
2633         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2634
2635         if ($dtime > 1000) {
2636             my $dutime = $used -  $old->{used};
2637
2638             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2639             $last_proc_pid_stat->{$pid} = {
2640                 time => $ctime,
2641                 used => $used,
2642                 cpu => $d->{cpu},
2643             };
2644         } else {
2645             $d->{cpu} = $old->{cpu};
2646         }
2647     }
2648
2649     return $res if !$full;
2650
2651     my $qmpclient = PVE::QMPClient->new();
2652
2653     my $ballooncb = sub {
2654         my ($vmid, $resp) = @_;
2655
2656         my $info = $resp->{'return'};
2657         return if !$info->{max_mem};
2658
2659         my $d = $res->{$vmid};
2660
2661         # use memory assigned to VM
2662         $d->{maxmem} = $info->{max_mem};
2663         $d->{balloon} = $info->{actual};
2664
2665         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2666             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2667             $d->{freemem} = $info->{free_mem};
2668         }
2669
2670         $d->{ballooninfo} = $info;
2671     };
2672
2673     my $blockstatscb = sub {
2674         my ($vmid, $resp) = @_;
2675         my $data = $resp->{'return'} || [];
2676         my $totalrdbytes = 0;
2677         my $totalwrbytes = 0;
2678
2679         for my $blockstat (@$data) {
2680             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2681             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2682
2683             $blockstat->{device} =~ s/drive-//;
2684             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2685         }
2686         $res->{$vmid}->{diskread} = $totalrdbytes;
2687         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2688     };
2689
2690     my $statuscb = sub {
2691         my ($vmid, $resp) = @_;
2692
2693         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2694         # this fails if ballon driver is not loaded, so this must be
2695         # the last commnand (following command are aborted if this fails).
2696         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2697
2698         my $status = 'unknown';
2699         if (!defined($status = $resp->{'return'}->{status})) {
2700             warn "unable to get VM status\n";
2701             return;
2702         }
2703
2704         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2705     };
2706
2707     foreach my $vmid (keys %$list) {
2708         next if $opt_vmid && ($vmid ne $opt_vmid);
2709         next if !$res->{$vmid}->{pid}; # not running
2710         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2711     }
2712
2713     $qmpclient->queue_execute(undef, 1);
2714
2715     foreach my $vmid (keys %$list) {
2716         next if $opt_vmid && ($vmid ne $opt_vmid);
2717         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2718     }
2719
2720     return $res;
2721 }
2722
2723 sub foreach_drive {
2724     my ($conf, $func, @param) = @_;
2725
2726     foreach my $ds (valid_drive_names()) {
2727         next if !defined($conf->{$ds});
2728
2729         my $drive = parse_drive($ds, $conf->{$ds});
2730         next if !$drive;
2731
2732         &$func($ds, $drive, @param);
2733     }
2734 }
2735
2736 sub foreach_volid {
2737     my ($conf, $func, @param) = @_;
2738
2739     my $volhash = {};
2740
2741     my $test_volid = sub {
2742         my ($volid, $is_cdrom) = @_;
2743
2744         return if !$volid;
2745
2746         $volhash->{$volid} = $is_cdrom || 0;
2747     };
2748
2749     foreach_drive($conf, sub {
2750         my ($ds, $drive) = @_;
2751         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2752     });
2753
2754     foreach my $snapname (keys %{$conf->{snapshots}}) {
2755         my $snap = $conf->{snapshots}->{$snapname};
2756         &$test_volid($snap->{vmstate}, 0);
2757         foreach_drive($snap, sub {
2758             my ($ds, $drive) = @_;
2759             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2760         });
2761     }
2762
2763     foreach my $volid (keys %$volhash) {
2764         &$func($volid, $volhash->{$volid}, @param);
2765     }
2766 }
2767
2768 sub vga_conf_has_spice {
2769     my ($vga) = @_;
2770
2771     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2772
2773     return $1 || 1;
2774 }
2775
2776 sub config_to_command {
2777     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2778
2779     my $cmd = [];
2780     my $globalFlags = [];
2781     my $machineFlags = [];
2782     my $rtcFlags = [];
2783     my $cpuFlags = [];
2784     my $devices = [];
2785     my $pciaddr = '';
2786     my $bridges = {};
2787     my $kvmver = kvm_user_version();
2788     my $vernum = 0; # unknown
2789     my $ostype = $conf->{ostype};
2790     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2791         $vernum = $1*1000000+$2*1000;
2792     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2793         $vernum = $1*1000000+$2*1000+$3;
2794     }
2795
2796     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2797
2798     my $have_ovz = -f '/proc/vz/vestat';
2799
2800     my $q35 = machine_type_is_q35($conf);
2801     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2802     my $machine_type = $forcemachine || $conf->{machine};
2803     my $use_old_bios_files = undef;
2804     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2805
2806     my $cpuunits = defined($conf->{cpuunits}) ?
2807             $conf->{cpuunits} : $defaults->{cpuunits};
2808
2809     push @$cmd, '/usr/bin/kvm';
2810
2811     push @$cmd, '-id', $vmid;
2812
2813     my $use_virtio = 0;
2814
2815     my $qmpsocket = qmp_socket($vmid);
2816     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2817     push @$cmd, '-mon', "chardev=qmp,mode=control";
2818
2819
2820     push @$cmd, '-pidfile' , pidfile_name($vmid);
2821
2822     push @$cmd, '-daemonize';
2823
2824     if ($conf->{smbios1}) {
2825         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2826     }
2827
2828     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2829         my $ovmfbase;
2830
2831         # prefer the OVMF_CODE variant
2832         if (-f $OVMF_CODE) {
2833             $ovmfbase = $OVMF_CODE;
2834         } elsif (-f $OVMF_IMG) {
2835             $ovmfbase = $OVMF_IMG;
2836         }
2837
2838         die "no uefi base img found\n" if !$ovmfbase;
2839         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2840
2841         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2842             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2843             my $format = $d->{format} // 'raw';
2844             my $path;
2845             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2846             if ($storeid) {
2847                 $path = PVE::Storage::path($storecfg, $d->{file});
2848                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2849                 $format = qemu_img_format($scfg, $volname);
2850             } else {
2851                 $path = $d->{file};
2852                 $format = "raw";
2853             }
2854             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2855         } elsif ($ovmfbase eq $OVMF_CODE) {
2856             warn "using uefi without permanent efivars disk\n";
2857             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2858             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2859             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2860         } else {
2861             # if the base img is not OVMF_CODE, we do not have to bother
2862             # to create/use a vars image, since it will not be used anyway
2863             # this can only happen if someone manually deletes the OVMF_CODE image
2864             # or has an old pve-qemu-kvm version installed.
2865             # both should not happen, but we ignore it here
2866         }
2867     }
2868
2869
2870     # add usb controllers
2871     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2872     push @$devices, @usbcontrollers if @usbcontrollers;
2873     my $vga = $conf->{vga};
2874
2875     my $qxlnum = vga_conf_has_spice($vga);
2876     $vga = 'qxl' if $qxlnum;
2877
2878     if (!$vga) {
2879         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2880                                 $conf->{ostype} eq 'win7' ||
2881                                 $conf->{ostype} eq 'w2k8')) {
2882             $vga = 'std';
2883         } else {
2884             $vga = 'cirrus';
2885         }
2886     }
2887
2888     # enable absolute mouse coordinates (needed by vnc)
2889     my $tablet;
2890     if (defined($conf->{tablet})) {
2891         $tablet = $conf->{tablet};
2892     } else {
2893         $tablet = $defaults->{tablet};
2894         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2895         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2896     }
2897
2898     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2899
2900     my $kvm_off = 0;
2901     # host pci devices
2902     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2903         my $d = parse_hostpci($conf->{"hostpci$i"});
2904         next if !$d;
2905
2906         my $pcie = $d->{pcie};
2907         if($pcie){
2908             die "q35 machine model is not enabled" if !$q35;
2909             $pciaddr = print_pcie_addr("hostpci$i");
2910         }else{
2911             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2912         }
2913
2914         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2915         my $xvga = '';
2916         if ($d->{'x-vga'}) {
2917             $xvga = ',x-vga=on';
2918             $kvm_off = 1;
2919             $vga = 'none';
2920             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2921                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2922             }
2923             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2924                 $xvga = "";
2925             }
2926         }
2927         my $pcidevices = $d->{pciid};
2928         my $multifunction = 1 if @$pcidevices > 1;
2929
2930         my $j=0;
2931         foreach my $pcidevice (@$pcidevices) {
2932
2933             my $id = "hostpci$i";
2934             $id .= ".$j" if $multifunction;
2935             my $addr = $pciaddr;
2936             $addr .= ".$j" if $multifunction;
2937             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2938
2939             if($j == 0){
2940                 $devicestr .= "$rombar$xvga";
2941                 $devicestr .= ",multifunction=on" if $multifunction;
2942             }
2943
2944             push @$devices, '-device', $devicestr;
2945             $j++;
2946         }
2947     }
2948
2949     # usb devices
2950     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2951     push @$devices, @usbdevices if @usbdevices;
2952     # serial devices
2953     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2954         if (my $path = $conf->{"serial$i"}) {
2955             if ($path eq 'socket') {
2956                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2957                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2958                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2959             } else {
2960                 die "no such serial device\n" if ! -c $path;
2961                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2962                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2963             }
2964         }
2965     }
2966
2967     # parallel devices
2968     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2969         if (my $path = $conf->{"parallel$i"}) {
2970             die "no such parallel device\n" if ! -c $path;
2971             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2972             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2973             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2974         }
2975     }
2976
2977     my $vmname = $conf->{name} || "vm$vmid";
2978
2979     push @$cmd, '-name', $vmname;
2980
2981     my $sockets = 1;
2982     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2983     $sockets = $conf->{sockets} if  $conf->{sockets};
2984
2985     my $cores = $conf->{cores} || 1;
2986
2987     my $maxcpus = $sockets * $cores;
2988
2989     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2990
2991     my $allowed_vcpus = $cpuinfo->{cpus};
2992
2993     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2994         if ($allowed_vcpus < $maxcpus);
2995
2996     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
2997
2998         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2999         for (my $i = 2; $i <= $vcpus; $i++)  {
3000             my $cpustr = print_cpu_device($conf,$i);
3001             push @$cmd, '-device', $cpustr;
3002         }
3003
3004     } else {
3005
3006         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3007     }
3008     push @$cmd, '-nodefaults';
3009
3010     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3011
3012     my $bootindex_hash = {};
3013     my $i = 1;
3014     foreach my $o (split(//, $bootorder)) {
3015         $bootindex_hash->{$o} = $i*100;
3016         $i++;
3017     }
3018
3019     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3020
3021     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3022
3023     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3024
3025     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3026
3027     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3028         my $socket = vnc_socket($vmid);
3029         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3030     } else {
3031         push @$cmd, '-nographic';
3032     }
3033
3034     # time drift fix
3035     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3036
3037     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3038     my $useLocaltime = $conf->{localtime};
3039
3040     if ($ostype) {
3041         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3042
3043         if ($ostype =~ m/^w/) { # windows
3044             $useLocaltime = 1 if !defined($conf->{localtime});
3045
3046             # use time drift fix when acpi is enabled
3047             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3048                 $tdf = 1 if !defined($conf->{tdf});
3049             }
3050         }
3051
3052         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3053             $ostype eq 'wvista') {
3054             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3055             push @$cmd, '-no-hpet';
3056             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3057                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3058                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3059                 push @$cpuFlags , 'hv_time' if !$nokvm;
3060
3061                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
3062                     push @$cpuFlags , 'hv_reset' if !$nokvm;
3063                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
3064                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
3065                 }
3066
3067             } else {
3068                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3069             }
3070         }
3071
3072         if ($ostype eq 'win7' || $ostype eq 'win8') {
3073             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3074         }
3075     }
3076
3077     push @$rtcFlags, 'driftfix=slew' if $tdf;
3078
3079     if ($nokvm) {
3080         push @$machineFlags, 'accel=tcg';
3081     } else {
3082         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3083     }
3084
3085     if ($machine_type) {
3086         push @$machineFlags, "type=${machine_type}";
3087     }
3088
3089     if ($conf->{startdate}) {
3090         push @$rtcFlags, "base=$conf->{startdate}";
3091     } elsif ($useLocaltime) {
3092         push @$rtcFlags, 'base=localtime';
3093     }
3094
3095     my $cpu = $nokvm ? "qemu64" : "kvm64";
3096     if (my $cputype = $conf->{cpu}) {
3097         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3098             or die "Cannot parse cpu description: $cputype\n";
3099         $cpu = $cpuconf->{cputype};
3100         $kvm_off = 1 if $cpuconf->{hidden};
3101     }
3102
3103     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3104
3105     push @$cpuFlags , '-x2apic'
3106         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3107
3108     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3109
3110     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3111
3112     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3113
3114         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3115         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3116     }
3117
3118     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3119
3120     push @$cpuFlags, 'kvm=off' if $kvm_off;
3121
3122     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3123         die "internal error"; # should not happen
3124
3125     push @$cpuFlags, "vendor=${cpu_vendor}"
3126         if $cpu_vendor ne 'default';
3127
3128     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3129
3130     push @$cmd, '-cpu', $cpu;
3131
3132     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3133
3134     push @$cmd, '-S' if $conf->{freeze};
3135
3136     # set keyboard layout
3137     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3138     push @$cmd, '-k', $kb if $kb;
3139
3140     # enable sound
3141     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3142     #push @$cmd, '-soundhw', 'es1370';
3143     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3144
3145     if($conf->{agent}) {
3146         my $qgasocket = qmp_socket($vmid, 1);
3147         my $pciaddr = print_pci_addr("qga0", $bridges);
3148         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3149         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3150         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3151     }
3152
3153     my $spice_port;
3154
3155     if ($qxlnum) {
3156         if ($qxlnum > 1) {
3157             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3158                 for(my $i = 1; $i < $qxlnum; $i++){
3159                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3160                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3161                 }
3162             } else {
3163                 # assume other OS works like Linux
3164                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3165                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3166             }
3167         }
3168
3169         my $pciaddr = print_pci_addr("spice", $bridges);
3170
3171         my $nodename = PVE::INotify::nodename();
3172         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3173         $spice_port = PVE::Tools::next_spice_port($pfamily);
3174
3175         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3176
3177         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3178         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3179         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3180     }
3181
3182     # enable balloon by default, unless explicitly disabled
3183     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3184         $pciaddr = print_pci_addr("balloon0", $bridges);
3185         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3186     }
3187
3188     if ($conf->{watchdog}) {
3189         my $wdopts = parse_watchdog($conf->{watchdog});
3190         $pciaddr = print_pci_addr("watchdog", $bridges);
3191         my $watchdog = $wdopts->{model} || 'i6300esb';
3192         push @$devices, '-device', "$watchdog$pciaddr";
3193         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3194     }
3195
3196     my $vollist = [];
3197     my $scsicontroller = {};
3198     my $ahcicontroller = {};
3199     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3200
3201     # Add iscsi initiator name if available
3202     if (my $initiator = get_initiator_name()) {
3203         push @$devices, '-iscsi', "initiator-name=$initiator";
3204     }
3205
3206     foreach_drive($conf, sub {
3207         my ($ds, $drive) = @_;
3208
3209         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3210             push @$vollist, $drive->{file};
3211         }
3212
3213         $use_virtio = 1 if $ds =~ m/^virtio/;
3214
3215         if (drive_is_cdrom ($drive)) {
3216             if ($bootindex_hash->{d}) {
3217                 $drive->{bootindex} = $bootindex_hash->{d};
3218                 $bootindex_hash->{d} += 1;
3219             }
3220         } else {
3221             if ($bootindex_hash->{c}) {
3222                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3223                 $bootindex_hash->{c} += 1;
3224             }
3225         }
3226
3227         if($drive->{interface} eq 'virtio'){
3228            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3229         }
3230
3231         if ($drive->{interface} eq 'scsi') {
3232
3233             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3234
3235             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3236             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3237
3238             my $iothread = '';
3239             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3240                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3241                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3242             } elsif ($drive->{iothread}) {
3243                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3244             }
3245
3246             my $queues = '';
3247             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3248                 $queues = ",num_queues=$drive->{queues}";
3249             }
3250
3251             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3252             $scsicontroller->{$controller}=1;
3253         }
3254
3255         if ($drive->{interface} eq 'sata') {
3256            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3257            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3258            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3259            $ahcicontroller->{$controller}=1;
3260         }
3261
3262         if ($drive->{interface} eq 'efidisk') {
3263             # this will be added somewhere else
3264             return;
3265         }
3266
3267         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3268         push @$devices, '-drive',$drive_cmd;
3269         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3270     });
3271
3272     for (my $i = 0; $i < $MAX_NETS; $i++) {
3273          next if !$conf->{"net$i"};
3274          my $d = parse_net($conf->{"net$i"});
3275          next if !$d;
3276
3277          $use_virtio = 1 if $d->{model} eq 'virtio';
3278
3279          if ($bootindex_hash->{n}) {
3280             $d->{bootindex} = $bootindex_hash->{n};
3281             $bootindex_hash->{n} += 1;
3282          }
3283
3284          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3285          push @$devices, '-netdev', $netdevfull;
3286
3287          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3288          push @$devices, '-device', $netdevicefull;
3289     }
3290
3291     if (!$q35) {
3292         # add pci bridges
3293         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3294            $bridges->{1} = 1;
3295            $bridges->{2} = 1;
3296         }
3297
3298         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3299
3300         while (my ($k, $v) = each %$bridges) {
3301             $pciaddr = print_pci_addr("pci.$k");
3302             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3303         }
3304     }
3305
3306     # add custom args
3307     if ($conf->{args}) {
3308         my $aa = PVE::Tools::split_args($conf->{args});
3309         push @$cmd, @$aa;
3310     }
3311
3312     push @$cmd, @$devices;
3313     push @$cmd, '-rtc', join(',', @$rtcFlags)
3314         if scalar(@$rtcFlags);
3315     push @$cmd, '-machine', join(',', @$machineFlags)
3316         if scalar(@$machineFlags);
3317     push @$cmd, '-global', join(',', @$globalFlags)
3318         if scalar(@$globalFlags);
3319
3320     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3321 }
3322
3323 sub vnc_socket {
3324     my ($vmid) = @_;
3325     return "${var_run_tmpdir}/$vmid.vnc";
3326 }
3327
3328 sub spice_port {
3329     my ($vmid) = @_;
3330
3331     my $res = vm_mon_cmd($vmid, 'query-spice');
3332
3333     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3334 }
3335
3336 sub qmp_socket {
3337     my ($vmid, $qga) = @_;
3338     my $sockettype = $qga ? 'qga' : 'qmp';
3339     return "${var_run_tmpdir}/$vmid.$sockettype";
3340 }
3341
3342 sub pidfile_name {
3343     my ($vmid) = @_;
3344     return "${var_run_tmpdir}/$vmid.pid";
3345 }
3346
3347 sub vm_devices_list {
3348     my ($vmid) = @_;
3349
3350     my $res = vm_mon_cmd($vmid, 'query-pci');
3351     my $devices = {};
3352     foreach my $pcibus (@$res) {
3353         foreach my $device (@{$pcibus->{devices}}) {
3354             next if !$device->{'qdev_id'};
3355             if ($device->{'pci_bridge'}) {
3356                 $devices->{$device->{'qdev_id'}} = 1;
3357                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3358                     next if !$bridge_device->{'qdev_id'};
3359                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3360                     $devices->{$device->{'qdev_id'}}++;
3361                 }
3362             } else {
3363                 $devices->{$device->{'qdev_id'}} = 1;
3364             }
3365         }
3366     }
3367
3368     my $resblock = vm_mon_cmd($vmid, 'query-block');
3369     foreach my $block (@$resblock) {
3370         if($block->{device} =~ m/^drive-(\S+)/){
3371                 $devices->{$1} = 1;
3372         }
3373     }
3374
3375     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3376     foreach my $mice (@$resmice) {
3377         if ($mice->{name} eq 'QEMU HID Tablet') {
3378             $devices->{tablet} = 1;
3379             last;
3380         }
3381     }
3382
3383     # for usb devices there is no query-usb
3384     # but we can iterate over the entries in
3385     # qom-list path=/machine/peripheral
3386     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3387     foreach my $per (@$resperipheral) {
3388         if ($per->{name} =~ m/^usb\d+$/) {
3389             $devices->{$per->{name}} = 1;
3390         }
3391     }
3392
3393     return $devices;
3394 }
3395
3396 sub vm_deviceplug {
3397     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3398
3399     my $q35 = machine_type_is_q35($conf);
3400
3401     my $devices_list = vm_devices_list($vmid);
3402     return 1 if defined($devices_list->{$deviceid});
3403
3404     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3405
3406     if ($deviceid eq 'tablet') {
3407
3408         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3409
3410     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3411
3412         die "usb hotplug currently not reliable\n";
3413         # since we can't reliably hot unplug all added usb devices
3414         # and usb passthrough disables live migration
3415         # we disable usb hotplugging for now
3416         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3417
3418     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3419
3420         qemu_iothread_add($vmid, $deviceid, $device);
3421
3422         qemu_driveadd($storecfg, $vmid, $device);
3423         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3424
3425         qemu_deviceadd($vmid, $devicefull);
3426         eval { qemu_deviceaddverify($vmid, $deviceid); };
3427         if (my $err = $@) {
3428             eval { qemu_drivedel($vmid, $deviceid); };
3429             warn $@ if $@;
3430             die $err;
3431         }
3432
3433     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3434
3435
3436         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3437         my $pciaddr = print_pci_addr($deviceid);
3438         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3439
3440         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3441
3442         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3443             qemu_iothread_add($vmid, $deviceid, $device);
3444             $devicefull .= ",iothread=iothread-$deviceid";
3445         }
3446
3447         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3448             $devicefull .= ",num_queues=$device->{queues}";
3449         }
3450
3451         qemu_deviceadd($vmid, $devicefull);
3452         qemu_deviceaddverify($vmid, $deviceid);
3453
3454     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3455
3456         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3457         qemu_driveadd($storecfg, $vmid, $device);
3458
3459         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3460         eval { qemu_deviceadd($vmid, $devicefull); };
3461         if (my $err = $@) {
3462             eval { qemu_drivedel($vmid, $deviceid); };
3463             warn $@ if $@;
3464             die $err;
3465         }
3466
3467     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3468
3469         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3470
3471         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3472         my $use_old_bios_files = undef;
3473         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3474
3475         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3476         qemu_deviceadd($vmid, $netdevicefull);
3477         eval { qemu_deviceaddverify($vmid, $deviceid); };
3478         if (my $err = $@) {
3479             eval { qemu_netdevdel($vmid, $deviceid); };
3480             warn $@ if $@;
3481             die $err;
3482         }
3483
3484     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3485
3486         my $bridgeid = $2;
3487         my $pciaddr = print_pci_addr($deviceid);
3488         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3489
3490         qemu_deviceadd($vmid, $devicefull);
3491         qemu_deviceaddverify($vmid, $deviceid);
3492
3493     } else {
3494         die "can't hotplug device '$deviceid'\n";
3495     }
3496
3497     return 1;
3498 }
3499
3500 # fixme: this should raise exceptions on error!
3501 sub vm_deviceunplug {
3502     my ($vmid, $conf, $deviceid) = @_;
3503
3504     my $devices_list = vm_devices_list($vmid);
3505     return 1 if !defined($devices_list->{$deviceid});
3506
3507     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3508
3509     if ($deviceid eq 'tablet') {
3510
3511         qemu_devicedel($vmid, $deviceid);
3512
3513     } elsif ($deviceid =~ m/^usb\d+$/) {
3514
3515         die "usb hotplug currently not reliable\n";
3516         # when unplugging usb devices this way,
3517         # there may be remaining usb controllers/hubs
3518         # so we disable it for now
3519         qemu_devicedel($vmid, $deviceid);
3520         qemu_devicedelverify($vmid, $deviceid);
3521
3522     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3523
3524         qemu_devicedel($vmid, $deviceid);
3525         qemu_devicedelverify($vmid, $deviceid);
3526         qemu_drivedel($vmid, $deviceid);
3527         qemu_iothread_del($conf, $vmid, $deviceid);
3528
3529     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3530
3531         qemu_devicedel($vmid, $deviceid);
3532         qemu_devicedelverify($vmid, $deviceid);
3533         qemu_iothread_del($conf, $vmid, $deviceid);
3534
3535     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3536
3537         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3538         my $device = parse_drive($deviceid, $conf->{$deviceid});
3539         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3540
3541         qemu_devicedel($vmid, $deviceid);
3542         qemu_drivedel($vmid, $deviceid);
3543         qemu_deletescsihw($conf, $vmid, $deviceid);
3544
3545     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3546
3547         qemu_devicedel($vmid, $deviceid);
3548         qemu_devicedelverify($vmid, $deviceid);
3549         qemu_netdevdel($vmid, $deviceid);
3550
3551     } else {
3552         die "can't unplug device '$deviceid'\n";
3553     }
3554
3555     return 1;
3556 }
3557
3558 sub qemu_deviceadd {
3559     my ($vmid, $devicefull) = @_;
3560
3561     $devicefull = "driver=".$devicefull;
3562     my %options =  split(/[=,]/, $devicefull);
3563
3564     vm_mon_cmd($vmid, "device_add" , %options);
3565 }
3566
3567 sub qemu_devicedel {
3568     my ($vmid, $deviceid) = @_;
3569
3570     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3571 }
3572
3573 sub qemu_iothread_add {
3574     my($vmid, $deviceid, $device) = @_;
3575
3576     if ($device->{iothread}) {
3577         my $iothreads = vm_iothreads_list($vmid);
3578         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3579     }
3580 }
3581
3582 sub qemu_iothread_del {
3583     my($conf, $vmid, $deviceid) = @_;
3584
3585     my $device = parse_drive($deviceid, $conf->{$deviceid});
3586     if ($device->{iothread}) {
3587         my $iothreads = vm_iothreads_list($vmid);
3588         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3589     }
3590 }
3591
3592 sub qemu_objectadd {
3593     my($vmid, $objectid, $qomtype) = @_;
3594
3595     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3596
3597     return 1;
3598 }
3599
3600 sub qemu_objectdel {
3601     my($vmid, $objectid) = @_;
3602
3603     vm_mon_cmd($vmid, "object-del", id => $objectid);
3604
3605     return 1;
3606 }
3607
3608 sub qemu_driveadd {
3609     my ($storecfg, $vmid, $device) = @_;
3610
3611     my $drive = print_drive_full($storecfg, $vmid, $device);
3612     $drive =~ s/\\/\\\\/g;
3613     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3614
3615     # If the command succeeds qemu prints: "OK"
3616     return 1 if $ret =~ m/OK/s;
3617
3618     die "adding drive failed: $ret\n";
3619 }
3620
3621 sub qemu_drivedel {
3622     my($vmid, $deviceid) = @_;
3623
3624     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3625     $ret =~ s/^\s+//;
3626
3627     return 1 if $ret eq "";
3628
3629     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3630     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3631
3632     die "deleting drive $deviceid failed : $ret\n";
3633 }
3634
3635 sub qemu_deviceaddverify {
3636     my ($vmid, $deviceid) = @_;
3637
3638     for (my $i = 0; $i <= 5; $i++) {
3639          my $devices_list = vm_devices_list($vmid);
3640          return 1 if defined($devices_list->{$deviceid});
3641          sleep 1;
3642     }
3643
3644     die "error on hotplug device '$deviceid'\n";
3645 }
3646
3647
3648 sub qemu_devicedelverify {
3649     my ($vmid, $deviceid) = @_;
3650
3651     # need to verify that the device is correctly removed as device_del
3652     # is async and empty return is not reliable
3653
3654     for (my $i = 0; $i <= 5; $i++) {
3655          my $devices_list = vm_devices_list($vmid);
3656          return 1 if !defined($devices_list->{$deviceid});
3657          sleep 1;
3658     }
3659
3660     die "error on hot-unplugging device '$deviceid'\n";
3661 }
3662
3663 sub qemu_findorcreatescsihw {
3664     my ($storecfg, $conf, $vmid, $device) = @_;
3665
3666     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3667
3668     my $scsihwid="$controller_prefix$controller";
3669     my $devices_list = vm_devices_list($vmid);
3670
3671     if(!defined($devices_list->{$scsihwid})) {
3672         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3673     }
3674
3675     return 1;
3676 }
3677
3678 sub qemu_deletescsihw {
3679     my ($conf, $vmid, $opt) = @_;
3680
3681     my $device = parse_drive($opt, $conf->{$opt});
3682
3683     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3684         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3685         return 1;
3686     }
3687
3688     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3689
3690     my $devices_list = vm_devices_list($vmid);
3691     foreach my $opt (keys %{$devices_list}) {
3692         if (PVE::QemuServer::is_valid_drivename($opt)) {
3693             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3694             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3695                 return 1;
3696             }
3697         }
3698     }
3699
3700     my $scsihwid="scsihw$controller";
3701
3702     vm_deviceunplug($vmid, $conf, $scsihwid);
3703
3704     return 1;
3705 }
3706
3707 sub qemu_add_pci_bridge {
3708     my ($storecfg, $conf, $vmid, $device) = @_;
3709
3710     my $bridges = {};
3711
3712     my $bridgeid;
3713
3714     print_pci_addr($device, $bridges);
3715
3716     while (my ($k, $v) = each %$bridges) {
3717         $bridgeid = $k;
3718     }
3719     return 1 if !defined($bridgeid) || $bridgeid < 1;
3720
3721     my $bridge = "pci.$bridgeid";
3722     my $devices_list = vm_devices_list($vmid);
3723
3724     if (!defined($devices_list->{$bridge})) {
3725         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3726     }
3727
3728     return 1;
3729 }
3730
3731 sub qemu_set_link_status {
3732     my ($vmid, $device, $up) = @_;
3733
3734     vm_mon_cmd($vmid, "set_link", name => $device,
3735                up => $up ? JSON::true : JSON::false);
3736 }
3737
3738 sub qemu_netdevadd {
3739     my ($vmid, $conf, $device, $deviceid) = @_;
3740
3741     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3742     my %options =  split(/[=,]/, $netdev);
3743
3744     vm_mon_cmd($vmid, "netdev_add",  %options);
3745     return 1;
3746 }
3747
3748 sub qemu_netdevdel {
3749     my ($vmid, $deviceid) = @_;
3750
3751     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3752 }
3753
3754 sub qemu_usb_hotplug {
3755     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3756
3757     return if !$device;
3758
3759     # remove the old one first
3760     vm_deviceunplug($vmid, $conf, $deviceid);
3761
3762     # check if xhci controller is necessary and available
3763     if ($device->{usb3}) {
3764
3765         my $devicelist = vm_devices_list($vmid);
3766
3767         if (!$devicelist->{xhci}) {
3768             my $pciaddr = print_pci_addr("xhci");
3769             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3770         }
3771     }
3772     my $d = parse_usb_device($device->{host});
3773     $d->{usb3} = $device->{usb3};
3774
3775     # add the new one
3776     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3777 }
3778
3779 sub qemu_cpu_hotplug {
3780     my ($vmid, $conf, $vcpus) = @_;
3781
3782     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3783
3784     my $sockets = 1;
3785     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3786     $sockets = $conf->{sockets} if  $conf->{sockets};
3787     my $cores = $conf->{cores} || 1;
3788     my $maxcpus = $sockets * $cores;
3789
3790     $vcpus = $maxcpus if !$vcpus;
3791
3792     die "you can't add more vcpus than maxcpus\n"
3793         if $vcpus > $maxcpus;
3794
3795     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3796
3797     if ($vcpus < $currentvcpus) {
3798
3799         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3800
3801             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3802                 qemu_devicedel($vmid, "cpu$i");
3803                 my $retry = 0;
3804                 my $currentrunningvcpus = undef;
3805                 while (1) {
3806                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3807                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3808                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3809                     $retry++;
3810                     sleep 1;
3811                 }
3812                 #update conf after each succesfull cpu unplug
3813                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3814                 PVE::QemuConfig->write_config($vmid, $conf);
3815             }
3816         } else {
3817             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3818         }
3819
3820         return;
3821     }
3822
3823     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3824     die "vcpus in running vm does not match its configuration\n"
3825         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3826
3827     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3828
3829         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3830             my $cpustr = print_cpu_device($conf, $i);
3831             qemu_deviceadd($vmid, $cpustr);
3832
3833             my $retry = 0;
3834             my $currentrunningvcpus = undef;
3835             while (1) {
3836                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3837                 last if scalar(@{$currentrunningvcpus}) == $i;
3838                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3839                 sleep 1;
3840                 $retry++;
3841             }
3842             #update conf after each succesfull cpu hotplug
3843             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3844             PVE::QemuConfig->write_config($vmid, $conf);
3845         }
3846     } else {
3847
3848         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3849             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3850         }
3851     }
3852 }
3853
3854 sub qemu_block_set_io_throttle {
3855     my ($vmid, $deviceid,
3856         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3857         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3858         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3859         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3860
3861     return if !check_running($vmid) ;
3862
3863     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3864         bps => int($bps),
3865         bps_rd => int($bps_rd),
3866         bps_wr => int($bps_wr),
3867         iops => int($iops),
3868         iops_rd => int($iops_rd),
3869         iops_wr => int($iops_wr),
3870         bps_max => int($bps_max),
3871         bps_rd_max => int($bps_rd_max),
3872         bps_wr_max => int($bps_wr_max),
3873         iops_max => int($iops_max),
3874         iops_rd_max => int($iops_rd_max),
3875         iops_wr_max => int($iops_wr_max),
3876         bps_max_length => int($bps_max_length),
3877         bps_rd_max_length => int($bps_rd_max_length),
3878         bps_wr_max_length => int($bps_wr_max_length),
3879         iops_max_length => int($iops_max_length),
3880         iops_rd_max_length => int($iops_rd_max_length),
3881         iops_wr_max_length => int($iops_wr_max_length),
3882     );
3883
3884 }
3885
3886 # old code, only used to shutdown old VM after update
3887 sub __read_avail {
3888     my ($fh, $timeout) = @_;
3889
3890     my $sel = new IO::Select;
3891     $sel->add($fh);
3892
3893     my $res = '';
3894     my $buf;
3895
3896     my @ready;
3897     while (scalar (@ready = $sel->can_read($timeout))) {
3898         my $count;
3899         if ($count = $fh->sysread($buf, 8192)) {
3900             if ($buf =~ /^(.*)\(qemu\) $/s) {
3901                 $res .= $1;
3902                 last;
3903             } else {
3904                 $res .= $buf;
3905             }
3906         } else {
3907             if (!defined($count)) {
3908                 die "$!\n";
3909             }
3910             last;
3911         }
3912     }
3913
3914     die "monitor read timeout\n" if !scalar(@ready);
3915
3916     return $res;
3917 }
3918
3919 # old code, only used to shutdown old VM after update
3920 sub vm_monitor_command {
3921     my ($vmid, $cmdstr, $nocheck) = @_;
3922
3923     my $res;
3924
3925     eval {
3926         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3927
3928         my $sname = "${var_run_tmpdir}/$vmid.mon";
3929
3930         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3931             die "unable to connect to VM $vmid socket - $!\n";
3932
3933         my $timeout = 3;
3934
3935         # hack: migrate sometime blocks the monitor (when migrate_downtime
3936         # is set)
3937         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3938             $timeout = 60*60; # 1 hour
3939         }
3940
3941         # read banner;
3942         my $data = __read_avail($sock, $timeout);
3943
3944         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3945             die "got unexpected qemu monitor banner\n";
3946         }
3947
3948         my $sel = new IO::Select;
3949         $sel->add($sock);
3950
3951         if (!scalar(my @ready = $sel->can_write($timeout))) {
3952             die "monitor write error - timeout";
3953         }
3954
3955         my $fullcmd = "$cmdstr\r";
3956
3957         # syslog('info', "VM $vmid monitor command: $cmdstr");
3958
3959         my $b;
3960         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3961             die "monitor write error - $!";
3962         }
3963
3964         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3965
3966         $timeout = 20;
3967
3968         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3969             $timeout = 60*60; # 1 hour
3970         } elsif ($cmdstr =~ m/^(eject|change)/) {
3971             $timeout = 60; # note: cdrom mount command is slow
3972         }
3973         if ($res = __read_avail($sock, $timeout)) {
3974
3975             my @lines = split("\r?\n", $res);
3976
3977             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3978
3979             $res = join("\n", @lines);
3980             $res .= "\n";
3981         }
3982     };
3983
3984     my $err = $@;
3985
3986     if ($err) {
3987         syslog("err", "VM $vmid monitor command failed - $err");
3988         die $err;
3989     }
3990
3991     return $res;
3992 }
3993
3994 sub qemu_block_resize {
3995     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3996
3997     my $running = check_running($vmid);
3998
3999     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4000
4001     return if !$running;
4002
4003     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4004
4005 }
4006
4007 sub qemu_volume_snapshot {
4008     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4009
4010     my $running = check_running($vmid);
4011
4012     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4013         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4014     } else {
4015         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4016     }
4017 }
4018
4019 sub qemu_volume_snapshot_delete {
4020     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4021
4022     my $running = check_running($vmid);
4023
4024     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4025
4026     return if !$running;
4027
4028     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4029 }
4030
4031 sub set_migration_caps {
4032     my ($vmid) = @_;
4033
4034     my $cap_ref = [];
4035
4036     my $enabled_cap = {
4037         "auto-converge" => 1,
4038         "xbzrle" => 1,
4039         "x-rdma-pin-all" => 0,
4040         "zero-blocks" => 0,
4041         "compress" => 0
4042     };
4043
4044     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4045
4046     for my $supported_capability (@$supported_capabilities) {
4047         push @$cap_ref, {
4048             capability => $supported_capability->{capability},
4049             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4050         };
4051     }
4052
4053     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4054 }
4055
4056 my $fast_plug_option = {
4057     'lock' => 1,
4058     'name' => 1,
4059     'onboot' => 1,
4060     'shares' => 1,
4061     'startup' => 1,
4062     'description' => 1,
4063     'protection' => 1,
4064 };
4065
4066 # hotplug changes in [PENDING]
4067 # $selection hash can be used to only apply specified options, for
4068 # example: { cores => 1 } (only apply changed 'cores')
4069 # $errors ref is used to return error messages
4070 sub vmconfig_hotplug_pending {
4071     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4072
4073     my $defaults = load_defaults();
4074
4075     # commit values which do not have any impact on running VM first
4076     # Note: those option cannot raise errors, we we do not care about
4077     # $selection and always apply them.
4078
4079     my $add_error = sub {
4080         my ($opt, $msg) = @_;
4081         $errors->{$opt} = "hotplug problem - $msg";
4082     };
4083
4084     my $changes = 0;
4085     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4086         if ($fast_plug_option->{$opt}) {
4087             $conf->{$opt} = $conf->{pending}->{$opt};
4088             delete $conf->{pending}->{$opt};
4089             $changes = 1;
4090         }
4091     }
4092
4093     if ($changes) {
4094         PVE::QemuConfig->write_config($vmid, $conf);
4095         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4096     }
4097
4098     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4099
4100     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4101     while (my ($opt, $force) = each %$pending_delete_hash) {
4102         next if $selection && !$selection->{$opt};
4103         eval {
4104             if ($opt eq 'hotplug') {
4105                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4106             } elsif ($opt eq 'tablet') {
4107                 die "skip\n" if !$hotplug_features->{usb};
4108                 if ($defaults->{tablet}) {
4109                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4110                 } else {
4111                     vm_deviceunplug($vmid, $conf, $opt);
4112                 }
4113             } elsif ($opt =~ m/^usb\d+/) {
4114                 die "skip\n";
4115                 # since we cannot reliably hot unplug usb devices
4116                 # we are disabling it
4117                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4118                 vm_deviceunplug($vmid, $conf, $opt);
4119             } elsif ($opt eq 'vcpus') {
4120                 die "skip\n" if !$hotplug_features->{cpu};
4121                 qemu_cpu_hotplug($vmid, $conf, undef);
4122             } elsif ($opt eq 'balloon') {
4123                 # enable balloon device is not hotpluggable
4124                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4125             } elsif ($fast_plug_option->{$opt}) {
4126                 # do nothing
4127             } elsif ($opt =~ m/^net(\d+)$/) {
4128                 die "skip\n" if !$hotplug_features->{network};
4129                 vm_deviceunplug($vmid, $conf, $opt);
4130             } elsif (is_valid_drivename($opt)) {
4131                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4132                 vm_deviceunplug($vmid, $conf, $opt);
4133                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4134             } elsif ($opt =~ m/^memory$/) {
4135                 die "skip\n" if !$hotplug_features->{memory};
4136                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4137             } elsif ($opt eq 'cpuunits') {
4138                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4139             } elsif ($opt eq 'cpulimit') {
4140                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4141             } else {
4142                 die "skip\n";
4143             }
4144         };
4145         if (my $err = $@) {
4146             &$add_error($opt, $err) if $err ne "skip\n";
4147         } else {
4148             # save new config if hotplug was successful
4149             delete $conf->{$opt};
4150             vmconfig_undelete_pending_option($conf, $opt);
4151             PVE::QemuConfig->write_config($vmid, $conf);
4152             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4153         }
4154     }
4155
4156     foreach my $opt (keys %{$conf->{pending}}) {
4157         next if $selection && !$selection->{$opt};
4158         my $value = $conf->{pending}->{$opt};
4159         eval {
4160             if ($opt eq 'hotplug') {
4161                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4162             } elsif ($opt eq 'tablet') {
4163                 die "skip\n" if !$hotplug_features->{usb};
4164                 if ($value == 1) {
4165                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4166                 } elsif ($value == 0) {
4167                     vm_deviceunplug($vmid, $conf, $opt);
4168                 }
4169             } elsif ($opt =~ m/^usb\d+$/) {
4170                 die "skip\n";
4171                 # since we cannot reliably hot unplug usb devices
4172                 # we are disabling it
4173                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4174                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4175                 die "skip\n" if !$d;
4176                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4177             } elsif ($opt eq 'vcpus') {
4178                 die "skip\n" if !$hotplug_features->{cpu};
4179                 qemu_cpu_hotplug($vmid, $conf, $value);
4180             } elsif ($opt eq 'balloon') {
4181                 # enable/disable balloning device is not hotpluggable
4182                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4183                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4184                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4185
4186                 # allow manual ballooning if shares is set to zero
4187                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4188                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4189                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4190                 }
4191             } elsif ($opt =~ m/^net(\d+)$/) {
4192                 # some changes can be done without hotplug
4193                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4194                                     $vmid, $opt, $value);
4195             } elsif (is_valid_drivename($opt)) {
4196                 # some changes can be done without hotplug
4197                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4198                                      $vmid, $opt, $value, 1);
4199             } elsif ($opt =~ m/^memory$/) { #dimms
4200                 die "skip\n" if !$hotplug_features->{memory};
4201                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4202             } elsif ($opt eq 'cpuunits') {
4203                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4204             } elsif ($opt eq 'cpulimit') {
4205                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4206                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4207             } else {
4208                 die "skip\n";  # skip non-hot-pluggable options
4209             }
4210         };
4211         if (my $err = $@) {
4212             &$add_error($opt, $err) if $err ne "skip\n";
4213         } else {
4214             # save new config if hotplug was successful
4215             $conf->{$opt} = $value;
4216             delete $conf->{pending}->{$opt};
4217             PVE::QemuConfig->write_config($vmid, $conf);
4218             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4219         }
4220     }
4221 }
4222
4223 sub try_deallocate_drive {
4224     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4225
4226     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4227         my $volid = $drive->{file};
4228         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4229             my $sid = PVE::Storage::parse_volume_id($volid);
4230             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4231
4232             # check if the disk is really unused
4233             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4234                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4235             PVE::Storage::vdisk_free($storecfg, $volid);
4236             return 1;
4237         } else {
4238             # If vm is not owner of this disk remove from config
4239             return 1;
4240         }
4241     }
4242
4243     return undef;
4244 }
4245
4246 sub vmconfig_delete_or_detach_drive {
4247     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4248
4249     my $drive = parse_drive($opt, $conf->{$opt});
4250
4251     my $rpcenv = PVE::RPCEnvironment::get();
4252     my $authuser = $rpcenv->get_user();
4253
4254     if ($force) {
4255         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4256         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4257     } else {
4258         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4259     }
4260 }
4261
4262 sub vmconfig_apply_pending {
4263     my ($vmid, $conf, $storecfg) = @_;
4264
4265     # cold plug
4266
4267     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4268     while (my ($opt, $force) = each %$pending_delete_hash) {
4269         die "internal error" if $opt =~ m/^unused/;
4270         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4271         if (!defined($conf->{$opt})) {
4272             vmconfig_undelete_pending_option($conf, $opt);
4273             PVE::QemuConfig->write_config($vmid, $conf);
4274         } elsif (is_valid_drivename($opt)) {
4275             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4276             vmconfig_undelete_pending_option($conf, $opt);
4277             delete $conf->{$opt};
4278             PVE::QemuConfig->write_config($vmid, $conf);
4279         } else {
4280             vmconfig_undelete_pending_option($conf, $opt);
4281             delete $conf->{$opt};
4282             PVE::QemuConfig->write_config($vmid, $conf);
4283         }
4284     }
4285
4286     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4287
4288     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4289         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4290
4291         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4292             # skip if nothing changed
4293         } elsif (is_valid_drivename($opt)) {
4294             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4295                 if defined($conf->{$opt});
4296             $conf->{$opt} = $conf->{pending}->{$opt};
4297         } else {
4298             $conf->{$opt} = $conf->{pending}->{$opt};
4299         }
4300
4301         delete $conf->{pending}->{$opt};
4302         PVE::QemuConfig->write_config($vmid, $conf);
4303     }
4304 }
4305
4306 my $safe_num_ne = sub {
4307     my ($a, $b) = @_;
4308
4309     return 0 if !defined($a) && !defined($b);
4310     return 1 if !defined($a);
4311     return 1 if !defined($b);
4312
4313     return $a != $b;
4314 };
4315
4316 my $safe_string_ne = sub {
4317     my ($a, $b) = @_;
4318
4319     return 0 if !defined($a) && !defined($b);
4320     return 1 if !defined($a);
4321     return 1 if !defined($b);
4322
4323     return $a ne $b;
4324 };
4325
4326 sub vmconfig_update_net {
4327     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4328
4329     my $newnet = parse_net($value);
4330
4331     if ($conf->{$opt}) {
4332         my $oldnet = parse_net($conf->{$opt});
4333
4334         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4335             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4336             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4337             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4338
4339             # for non online change, we try to hot-unplug
4340             die "skip\n" if !$hotplug;
4341             vm_deviceunplug($vmid, $conf, $opt);
4342         } else {
4343
4344             die "internal error" if $opt !~ m/net(\d+)/;
4345             my $iface = "tap${vmid}i$1";
4346
4347             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4348                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4349                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4350                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4351                 PVE::Network::tap_unplug($iface);
4352                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4353             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4354                 # Rate can be applied on its own but any change above needs to
4355                 # include the rate in tap_plug since OVS resets everything.
4356                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4357             }
4358
4359             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4360                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4361             }
4362
4363             return 1;
4364         }
4365     }
4366
4367     if ($hotplug) {
4368         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4369     } else {
4370         die "skip\n";
4371     }
4372 }
4373
4374 sub vmconfig_update_disk {
4375     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4376
4377     # fixme: do we need force?
4378
4379     my $drive = parse_drive($opt, $value);
4380
4381     if ($conf->{$opt}) {
4382
4383         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4384
4385             my $media = $drive->{media} || 'disk';
4386             my $oldmedia = $old_drive->{media} || 'disk';
4387             die "unable to change media type\n" if $media ne $oldmedia;
4388
4389             if (!drive_is_cdrom($old_drive)) {
4390
4391                 if ($drive->{file} ne $old_drive->{file}) {
4392
4393                     die "skip\n" if !$hotplug;
4394
4395                     # unplug and register as unused
4396                     vm_deviceunplug($vmid, $conf, $opt);
4397                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4398
4399                 } else {
4400                     # update existing disk
4401
4402                     # skip non hotpluggable value
4403                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4404                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4405                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4406                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4407                         die "skip\n";
4408                     }
4409
4410                     # apply throttle
4411                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4412                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4413                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4414                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4415                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4416                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4417                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4418                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4419                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4420                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4421                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4422                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4423                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4424                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4425                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4426                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4427                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4428                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4429
4430                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4431                                                    ($drive->{mbps} || 0)*1024*1024,
4432                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4433                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4434                                                    $drive->{iops} || 0,
4435                                                    $drive->{iops_rd} || 0,
4436                                                    $drive->{iops_wr} || 0,
4437                                                    ($drive->{mbps_max} || 0)*1024*1024,
4438                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4439                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4440                                                    $drive->{iops_max} || 0,
4441                                                    $drive->{iops_rd_max} || 0,
4442                                                    $drive->{iops_wr_max} || 0,
4443                                                    $drive->{bps_max_length} || 1,
4444                                                    $drive->{bps_rd_max_length} || 1,
4445                                                    $drive->{bps_wr_max_length} || 1,
4446                                                    $drive->{iops_max_length} || 1,
4447                                                    $drive->{iops_rd_max_length} || 1,
4448                                                    $drive->{iops_wr_max_length} || 1);
4449
4450                     }
4451
4452                     return 1;
4453                 }
4454
4455             } else { # cdrom
4456
4457                 if ($drive->{file} eq 'none') {
4458                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4459                 } else {
4460                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4461                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4462                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4463                 }
4464
4465                 return 1;
4466             }
4467         }
4468     }
4469
4470     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4471     # hotplug new disks
4472     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4473     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4474 }
4475
4476 sub vm_start {
4477     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4478         $forcemachine, $spice_ticket, $migration_network, $migration_type) = @_;
4479
4480     PVE::QemuConfig->lock_config($vmid, sub {
4481         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4482
4483         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4484
4485         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4486
4487         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4488
4489         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4490             vmconfig_apply_pending($vmid, $conf, $storecfg);
4491             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4492         }
4493
4494         my $defaults = load_defaults();
4495
4496         # set environment variable useful inside network script
4497         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4498
4499         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4500
4501         my $migrate_port = 0;
4502         my $migrate_uri;
4503         if ($statefile) {
4504             if ($statefile eq 'tcp') {
4505                 my $localip = "localhost";
4506                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4507                 my $nodename = PVE::INotify::nodename();
4508
4509                 if ($migration_type eq 'insecure') {
4510                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4511                     if ($migrate_network_addr) {
4512                         $localip = $migrate_network_addr;
4513                     } else {
4514                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4515                     }
4516
4517                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4518                 }
4519
4520                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4521                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4522                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4523                 push @$cmd, '-incoming', $migrate_uri;
4524                 push @$cmd, '-S';
4525
4526             } elsif ($statefile eq 'unix') {
4527                 # should be default for secure migrations as a ssh TCP forward
4528                 # tunnel is not deterministic reliable ready and fails regurarly
4529                 # to set up in time, so use UNIX socket forwards
4530                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4531                 unlink $socket_addr;
4532
4533                 $migrate_uri = "unix:$socket_addr";
4534
4535                 push @$cmd, '-incoming', $migrate_uri;
4536                 push @$cmd, '-S';
4537
4538             } else {
4539                 push @$cmd, '-loadstate', $statefile;
4540             }
4541         } elsif ($paused) {
4542             push @$cmd, '-S';
4543         }
4544
4545         # host pci devices
4546         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4547           my $d = parse_hostpci($conf->{"hostpci$i"});
4548           next if !$d;
4549           my $pcidevices = $d->{pciid};
4550           foreach my $pcidevice (@$pcidevices) {
4551                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4552
4553                 my $info = pci_device_info("0000:$pciid");
4554                 die "IOMMU not present\n" if !check_iommu_support();
4555                 die "no pci device info for device '$pciid'\n" if !$info;
4556                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4557                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4558           }
4559         }
4560
4561         PVE::Storage::activate_volumes($storecfg, $vollist);
4562
4563         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4564             my $cmd = [];
4565             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4566             eval  { run_command($cmd); };
4567         }
4568
4569         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4570                                                   : $defaults->{cpuunits};
4571
4572         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4573
4574         my %properties = (
4575             Slice => 'qemu.slice',
4576             KillMode => 'none',
4577             CPUShares => $cpuunits
4578         );
4579
4580         if (my $cpulimit = $conf->{cpulimit}) {
4581             $properties{CPUQuota} = int($cpulimit * 100);
4582         }
4583         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4584
4585         if ($conf->{hugepages}) {
4586
4587             my $code = sub {
4588                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4589                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4590
4591                 PVE::QemuServer::Memory::hugepages_mount();
4592                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4593
4594                 eval  {
4595                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4596                     run_command($cmd, %run_params);
4597                 };
4598
4599                 if (my $err = $@) {
4600                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4601                     die $err;
4602                 }
4603
4604                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4605             };
4606             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4607
4608         } else {
4609             eval  {
4610                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4611                 run_command($cmd, %run_params);
4612             };
4613         }
4614
4615         if (my $err = $@) {
4616             # deactivate volumes if start fails
4617             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4618             die "start failed: $err";
4619         }
4620
4621         print "migration listens on $migrate_uri\n" if $migrate_uri;
4622
4623         if ($statefile && $statefile ne 'tcp')  {
4624             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4625             warn $@ if $@;
4626         }
4627
4628         if ($migratedfrom) {
4629
4630             eval {
4631                 set_migration_caps($vmid);
4632             };
4633             warn $@ if $@;
4634
4635             if ($spice_port) {
4636                 print "spice listens on port $spice_port\n";
4637                 if ($spice_ticket) {
4638                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4639                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4640                 }
4641             }
4642
4643         } else {
4644
4645             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4646                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4647                     if $conf->{balloon};
4648             }
4649
4650             foreach my $opt (keys %$conf) {
4651                 next if $opt !~  m/^net\d+$/;
4652                 my $nicconf = parse_net($conf->{$opt});
4653                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4654             }
4655         }
4656
4657         vm_mon_cmd_nocheck($vmid, 'qom-set',
4658                     path => "machine/peripheral/balloon0",
4659                     property => "guest-stats-polling-interval",
4660                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4661
4662     });
4663 }
4664
4665 sub vm_mon_cmd {
4666     my ($vmid, $execute, %params) = @_;
4667
4668     my $cmd = { execute => $execute, arguments => \%params };
4669     vm_qmp_command($vmid, $cmd);
4670 }
4671
4672 sub vm_mon_cmd_nocheck {
4673     my ($vmid, $execute, %params) = @_;
4674
4675     my $cmd = { execute => $execute, arguments => \%params };
4676     vm_qmp_command($vmid, $cmd, 1);
4677 }
4678
4679 sub vm_qmp_command {
4680     my ($vmid, $cmd, $nocheck) = @_;
4681
4682     my $res;
4683
4684     my $timeout;
4685     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4686         $timeout = $cmd->{arguments}->{timeout};
4687         delete $cmd->{arguments}->{timeout};
4688     }
4689
4690     eval {
4691         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4692         my $sname = qmp_socket($vmid);
4693         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4694             my $qmpclient = PVE::QMPClient->new();
4695
4696             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4697         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4698             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4699                 if scalar(%{$cmd->{arguments}});
4700             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4701         } else {
4702             die "unable to open monitor socket\n";
4703         }
4704     };
4705     if (my $err = $@) {
4706         syslog("err", "VM $vmid qmp command failed - $err");
4707         die $err;
4708     }
4709
4710     return $res;
4711 }
4712
4713 sub vm_human_monitor_command {
4714     my ($vmid, $cmdline) = @_;
4715
4716     my $res;
4717
4718     my $cmd = {
4719         execute => 'human-monitor-command',
4720         arguments => { 'command-line' => $cmdline},
4721     };
4722
4723     return vm_qmp_command($vmid, $cmd);
4724 }
4725
4726 sub vm_commandline {
4727     my ($storecfg, $vmid) = @_;
4728
4729     my $conf = PVE::QemuConfig->load_config($vmid);
4730
4731     my $defaults = load_defaults();
4732
4733     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4734
4735     return PVE::Tools::cmd2string($cmd);
4736 }
4737
4738 sub vm_reset {
4739     my ($vmid, $skiplock) = @_;
4740
4741     PVE::QemuConfig->lock_config($vmid, sub {
4742
4743         my $conf = PVE::QemuConfig->load_config($vmid);
4744
4745         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4746
4747         vm_mon_cmd($vmid, "system_reset");
4748     });
4749 }
4750
4751 sub get_vm_volumes {
4752     my ($conf) = @_;
4753
4754     my $vollist = [];
4755     foreach_volid($conf, sub {
4756         my ($volid, $is_cdrom) = @_;
4757
4758         return if $volid =~ m|^/|;
4759
4760         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4761         return if !$sid;
4762
4763         push @$vollist, $volid;
4764     });
4765
4766     return $vollist;
4767 }
4768
4769 sub vm_stop_cleanup {
4770     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4771
4772     eval {
4773
4774         if (!$keepActive) {
4775             my $vollist = get_vm_volumes($conf);
4776             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4777         }
4778
4779         foreach my $ext (qw(mon qmp pid vnc qga)) {
4780             unlink "/var/run/qemu-server/${vmid}.$ext";
4781         }
4782
4783         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4784     };
4785     warn $@ if $@; # avoid errors - just warn
4786 }
4787
4788 # Note: use $nockeck to skip tests if VM configuration file exists.
4789 # We need that when migration VMs to other nodes (files already moved)
4790 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4791 sub vm_stop {
4792     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4793
4794     $force = 1 if !defined($force) && !$shutdown;
4795
4796     if ($migratedfrom){
4797         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4798         kill 15, $pid if $pid;
4799         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4800         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4801         return;
4802     }
4803
4804     PVE::QemuConfig->lock_config($vmid, sub {
4805
4806         my $pid = check_running($vmid, $nocheck);
4807         return if !$pid;
4808
4809         my $conf;
4810         if (!$nocheck) {
4811             $conf = PVE::QemuConfig->load_config($vmid);
4812             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4813             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4814                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4815                 $timeout = $opts->{down} if $opts->{down};
4816             }
4817         }
4818
4819         $timeout = 60 if !defined($timeout);
4820
4821         eval {
4822             if ($shutdown) {
4823                 if (defined($conf) && $conf->{agent}) {
4824                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4825                 } else {
4826                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4827                 }
4828             } else {
4829                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4830             }
4831         };
4832         my $err = $@;
4833
4834         if (!$err) {
4835             my $count = 0;
4836             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4837                 $count++;
4838                 sleep 1;
4839             }
4840
4841             if ($count >= $timeout) {
4842                 if ($force) {
4843                     warn "VM still running - terminating now with SIGTERM\n";
4844                     kill 15, $pid;
4845                 } else {
4846                     die "VM quit/powerdown failed - got timeout\n";
4847                 }
4848             } else {
4849                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4850                 return;
4851             }
4852         } else {
4853             if ($force) {
4854                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4855                 kill 15, $pid;
4856             } else {
4857                 die "VM quit/powerdown failed\n";
4858             }
4859         }
4860
4861         # wait again
4862         $timeout = 10;
4863
4864         my $count = 0;
4865         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4866             $count++;
4867             sleep 1;
4868         }
4869
4870         if ($count >= $timeout) {
4871             warn "VM still running - terminating now with SIGKILL\n";
4872             kill 9, $pid;
4873             sleep 1;
4874         }
4875
4876         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4877    });
4878 }
4879
4880 sub vm_suspend {
4881     my ($vmid, $skiplock) = @_;
4882
4883     PVE::QemuConfig->lock_config($vmid, sub {
4884
4885         my $conf = PVE::QemuConfig->load_config($vmid);
4886
4887         PVE::QemuConfig->check_lock($conf)
4888             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4889
4890         vm_mon_cmd($vmid, "stop");
4891     });
4892 }
4893
4894 sub vm_resume {
4895     my ($vmid, $skiplock, $nocheck) = @_;
4896
4897     PVE::QemuConfig->lock_config($vmid, sub {
4898
4899         if (!$nocheck) {
4900
4901             my $conf = PVE::QemuConfig->load_config($vmid);
4902
4903             PVE::QemuConfig->check_lock($conf)
4904                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4905
4906             vm_mon_cmd($vmid, "cont");
4907
4908         } else {
4909             vm_mon_cmd_nocheck($vmid, "cont");
4910         }
4911     });
4912 }
4913
4914 sub vm_sendkey {
4915     my ($vmid, $skiplock, $key) = @_;
4916
4917     PVE::QemuConfig->lock_config($vmid, sub {
4918
4919         my $conf = PVE::QemuConfig->load_config($vmid);
4920
4921         # there is no qmp command, so we use the human monitor command
4922         vm_human_monitor_command($vmid, "sendkey $key");
4923     });
4924 }
4925
4926 sub vm_destroy {
4927     my ($storecfg, $vmid, $skiplock) = @_;
4928
4929     PVE::QemuConfig->lock_config($vmid, sub {
4930
4931         my $conf = PVE::QemuConfig->load_config($vmid);
4932
4933         if (!check_running($vmid)) {
4934             destroy_vm($storecfg, $vmid, undef, $skiplock);
4935         } else {
4936             die "VM $vmid is running - destroy failed\n";
4937         }
4938     });
4939 }
4940
4941 # pci helpers
4942
4943 sub file_write {
4944     my ($filename, $buf) = @_;
4945
4946     my $fh = IO::File->new($filename, "w");
4947     return undef if !$fh;
4948
4949     my $res = print $fh $buf;
4950
4951     $fh->close();
4952
4953     return $res;
4954 }
4955
4956 sub pci_device_info {
4957     my ($name) = @_;
4958
4959     my $res;
4960
4961     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4962     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4963
4964     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4965     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4966
4967     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4968     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4969
4970     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4971     return undef if !defined($product) || $product !~ s/^0x//;
4972
4973     $res = {
4974         name => $name,
4975         vendor => $vendor,
4976         product => $product,
4977         domain => $domain,
4978         bus => $bus,
4979         slot => $slot,
4980         func => $func,
4981         irq => $irq,
4982         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4983     };
4984
4985     return $res;
4986 }
4987
4988 sub pci_dev_reset {
4989     my ($dev) = @_;
4990
4991     my $name = $dev->{name};
4992
4993     my $fn = "$pcisysfs/devices/$name/reset";
4994
4995     return file_write($fn, "1");
4996 }
4997
4998 sub pci_dev_bind_to_vfio {
4999     my ($dev) = @_;
5000
5001     my $name = $dev->{name};
5002
5003     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5004
5005     if (!-d $vfio_basedir) {
5006         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5007     }
5008     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5009
5010     my $testdir = "$vfio_basedir/$name";
5011     return 1 if -d $testdir;
5012
5013     my $data = "$dev->{vendor} $dev->{product}";
5014     return undef if !file_write("$vfio_basedir/new_id", $data);
5015
5016     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5017     if (!file_write($fn, $name)) {
5018         return undef if -f $fn;
5019     }
5020
5021     $fn = "$vfio_basedir/bind";
5022     if (! -d $testdir) {
5023         return undef if !file_write($fn, $name);
5024     }
5025
5026     return -d $testdir;
5027 }
5028
5029 sub pci_dev_group_bind_to_vfio {
5030     my ($pciid) = @_;
5031
5032     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5033
5034     if (!-d $vfio_basedir) {
5035         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5036     }
5037     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5038
5039     # get IOMMU group devices
5040     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5041       my @devs = grep /^0000:/, readdir($D);
5042     closedir($D);
5043
5044     foreach my $pciid (@devs) {
5045         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5046
5047         # pci bridges, switches or root ports are not supported
5048         # they have a pci_bus subdirectory so skip them
5049         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5050
5051         my $info = pci_device_info($1);
5052         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5053     }
5054
5055     return 1;
5056 }
5057
5058 # vzdump restore implementaion
5059
5060 sub tar_archive_read_firstfile {
5061     my $archive = shift;
5062
5063     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5064
5065     # try to detect archive type first
5066     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5067         die "unable to open file '$archive'\n";
5068     my $firstfile = <$fh>;
5069     kill 15, $pid;
5070     close $fh;
5071
5072     die "ERROR: archive contaions no data\n" if !$firstfile;
5073     chomp $firstfile;
5074
5075     return $firstfile;
5076 }
5077
5078 sub tar_restore_cleanup {
5079     my ($storecfg, $statfile) = @_;
5080
5081     print STDERR "starting cleanup\n";
5082
5083     if (my $fd = IO::File->new($statfile, "r")) {
5084         while (defined(my $line = <$fd>)) {
5085             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5086                 my $volid = $2;
5087                 eval {
5088                     if ($volid =~ m|^/|) {
5089                         unlink $volid || die 'unlink failed\n';
5090                     } else {
5091                         PVE::Storage::vdisk_free($storecfg, $volid);
5092                     }
5093                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5094                 };
5095                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5096             } else {
5097                 print STDERR "unable to parse line in statfile - $line";
5098             }
5099         }
5100         $fd->close();
5101     }
5102 }
5103
5104 sub restore_archive {
5105     my ($archive, $vmid, $user, $opts) = @_;
5106
5107     my $format = $opts->{format};
5108     my $comp;
5109
5110     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5111         $format = 'tar' if !$format;
5112         $comp = 'gzip';
5113     } elsif ($archive =~ m/\.tar$/) {
5114         $format = 'tar' if !$format;
5115     } elsif ($archive =~ m/.tar.lzo$/) {
5116         $format = 'tar' if !$format;
5117         $comp = 'lzop';
5118     } elsif ($archive =~ m/\.vma$/) {
5119         $format = 'vma' if !$format;
5120     } elsif ($archive =~ m/\.vma\.gz$/) {
5121         $format = 'vma' if !$format;
5122         $comp = 'gzip';
5123     } elsif ($archive =~ m/\.vma\.lzo$/) {
5124         $format = 'vma' if !$format;
5125         $comp = 'lzop';
5126     } else {
5127         $format = 'vma' if !$format; # default
5128     }
5129
5130     # try to detect archive format
5131     if ($format eq 'tar') {
5132         return restore_tar_archive($archive, $vmid, $user, $opts);
5133     } else {
5134         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5135     }
5136 }
5137
5138 sub restore_update_config_line {
5139     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5140
5141     return if $line =~ m/^\#qmdump\#/;
5142     return if $line =~ m/^\#vzdump\#/;
5143     return if $line =~ m/^lock:/;
5144     return if $line =~ m/^unused\d+:/;
5145     return if $line =~ m/^parent:/;
5146     return if $line =~ m/^template:/; # restored VM is never a template
5147
5148     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5149     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5150         # try to convert old 1.X settings
5151         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5152         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5153             my ($model, $macaddr) = split(/\=/, $devconfig);
5154             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5155             my $net = {
5156                 model => $model,
5157                 bridge => "vmbr$ind",
5158                 macaddr => $macaddr,
5159             };
5160             my $netstr = print_net($net);
5161
5162             print $outfd "net$cookie->{netcount}: $netstr\n";
5163             $cookie->{netcount}++;
5164         }
5165     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5166         my ($id, $netstr) = ($1, $2);
5167         my $net = parse_net($netstr);
5168         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5169         $netstr = print_net($net);
5170         print $outfd "$id: $netstr\n";
5171     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5172         my $virtdev = $1;
5173         my $value = $3;
5174         my $di = parse_drive($virtdev, $value);
5175         if (defined($di->{backup}) && !$di->{backup}) {
5176             print $outfd "#$line";
5177         } elsif ($map->{$virtdev}) {
5178             delete $di->{format}; # format can change on restore
5179             $di->{file} = $map->{$virtdev};
5180             $value = print_drive($vmid, $di);
5181             print $outfd "$virtdev: $value\n";
5182         } else {
5183             print $outfd $line;
5184         }
5185     } else {
5186         print $outfd $line;
5187     }
5188 }
5189
5190 sub scan_volids {
5191     my ($cfg, $vmid) = @_;
5192
5193     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5194
5195     my $volid_hash = {};
5196     foreach my $storeid (keys %$info) {
5197         foreach my $item (@{$info->{$storeid}}) {
5198             next if !($item->{volid} && $item->{size});
5199             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5200             $volid_hash->{$item->{volid}} = $item;
5201         }
5202     }
5203
5204     return $volid_hash;
5205 }
5206
5207 sub is_volume_in_use {
5208     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5209
5210     my $path = PVE::Storage::path($storecfg, $volid);
5211
5212     my $scan_config = sub {
5213         my ($cref, $snapname) = @_;
5214
5215         foreach my $key (keys %$cref) {
5216             my $value = $cref->{$key};
5217             if (is_valid_drivename($key)) {
5218                 next if $skip_drive && $key eq $skip_drive;
5219                 my $drive = parse_drive($key, $value);
5220                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5221                 return 1 if $volid eq $drive->{file};
5222                 if ($drive->{file} =~ m!^/!) {
5223                     return 1 if $drive->{file} eq $path;
5224                 } else {
5225                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5226                     next if !$storeid;
5227                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5228                     next if !$scfg;
5229                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5230                 }
5231             }
5232         }
5233
5234         return 0;
5235     };
5236
5237     return 1 if &$scan_config($conf);
5238
5239     undef $skip_drive;
5240
5241     foreach my $snapname (keys %{$conf->{snapshots}}) {
5242         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5243     }
5244
5245     return 0;
5246 }
5247
5248 sub update_disksize {
5249     my ($vmid, $conf, $volid_hash) = @_;
5250
5251     my $changes;
5252
5253     my $used = {};
5254
5255     # Note: it is allowed to define multiple storages with same path (alias), so
5256     # we need to check both 'volid' and real 'path' (two different volid can point
5257     # to the same path).
5258
5259     my $usedpath = {};
5260
5261     # update size info
5262     foreach my $opt (keys %$conf) {
5263         if (is_valid_drivename($opt)) {
5264             my $drive = parse_drive($opt, $conf->{$opt});
5265             my $volid = $drive->{file};
5266             next if !$volid;
5267
5268             $used->{$volid} = 1;
5269             if ($volid_hash->{$volid} &&
5270                 (my $path = $volid_hash->{$volid}->{path})) {
5271                 $usedpath->{$path} = 1;
5272             }
5273
5274             next if drive_is_cdrom($drive);
5275             next if !$volid_hash->{$volid};
5276
5277             $drive->{size} = $volid_hash->{$volid}->{size};
5278             my $new = print_drive($vmid, $drive);
5279             if ($new ne $conf->{$opt}) {
5280                 $changes = 1;
5281                 $conf->{$opt} = $new;
5282             }
5283         }
5284     }
5285
5286     # remove 'unusedX' entry if volume is used
5287     foreach my $opt (keys %$conf) {
5288         next if $opt !~ m/^unused\d+$/;
5289         my $volid = $conf->{$opt};
5290         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5291         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5292             $changes = 1;
5293             delete $conf->{$opt};
5294         }
5295     }
5296
5297     foreach my $volid (sort keys %$volid_hash) {
5298         next if $volid =~ m/vm-$vmid-state-/;
5299         next if $used->{$volid};
5300         my $path = $volid_hash->{$volid}->{path};
5301         next if !$path; # just to be sure
5302         next if $usedpath->{$path};
5303         $changes = 1;
5304         PVE::QemuConfig->add_unused_volume($conf, $volid);
5305         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5306     }
5307
5308     return $changes;
5309 }
5310
5311 sub rescan {
5312     my ($vmid, $nolock) = @_;
5313
5314     my $cfg = PVE::Storage::config();
5315
5316     my $volid_hash = scan_volids($cfg, $vmid);
5317
5318     my $updatefn =  sub {
5319         my ($vmid) = @_;
5320
5321         my $conf = PVE::QemuConfig->load_config($vmid);
5322
5323         PVE::QemuConfig->check_lock($conf);
5324
5325         my $vm_volids = {};
5326         foreach my $volid (keys %$volid_hash) {
5327             my $info = $volid_hash->{$volid};
5328             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5329         }
5330
5331         my $changes = update_disksize($vmid, $conf, $vm_volids);
5332
5333         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5334     };
5335
5336     if (defined($vmid)) {
5337         if ($nolock) {
5338             &$updatefn($vmid);
5339         } else {
5340             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5341         }
5342     } else {
5343         my $vmlist = config_list();
5344         foreach my $vmid (keys %$vmlist) {
5345             if ($nolock) {
5346                 &$updatefn($vmid);
5347             } else {
5348                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5349             }
5350         }
5351     }
5352 }
5353
5354 sub restore_vma_archive {
5355     my ($archive, $vmid, $user, $opts, $comp) = @_;
5356
5357     my $input = $archive eq '-' ? "<&STDIN" : undef;
5358     my $readfrom = $archive;
5359
5360     my $uncomp = '';
5361     if ($comp) {
5362         $readfrom = '-';
5363         my $qarchive = PVE::Tools::shellquote($archive);
5364         if ($comp eq 'gzip') {
5365             $uncomp = "zcat $qarchive|";
5366         } elsif ($comp eq 'lzop') {
5367             $uncomp = "lzop -d -c $qarchive|";
5368         } else {
5369             die "unknown compression method '$comp'\n";
5370         }
5371
5372     }
5373
5374     my $tmpdir = "/var/tmp/vzdumptmp$$";
5375     rmtree $tmpdir;
5376
5377     # disable interrupts (always do cleanups)
5378     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5379         warn "got interrupt - ignored\n";
5380     };
5381
5382     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5383     POSIX::mkfifo($mapfifo, 0600);
5384     my $fifofh;
5385
5386     my $openfifo = sub {
5387         open($fifofh, '>', $mapfifo) || die $!;
5388     };
5389
5390     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5391
5392     my $oldtimeout;
5393     my $timeout = 5;
5394
5395     my $devinfo = {};
5396
5397     my $rpcenv = PVE::RPCEnvironment::get();
5398
5399     my $conffile = PVE::QemuConfig->config_file($vmid);
5400     my $tmpfn = "$conffile.$$.tmp";
5401
5402     # Note: $oldconf is undef if VM does not exists
5403     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5404     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5405
5406     my $print_devmap = sub {
5407         my $virtdev_hash = {};
5408
5409         my $cfgfn = "$tmpdir/qemu-server.conf";
5410
5411         # we can read the config - that is already extracted
5412         my $fh = IO::File->new($cfgfn, "r") ||
5413             "unable to read qemu-server.conf - $!\n";
5414
5415         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5416         if (-f $fwcfgfn) {
5417             my $pve_firewall_dir = '/etc/pve/firewall';
5418             mkdir $pve_firewall_dir; # make sure the dir exists
5419             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5420         }
5421
5422         while (defined(my $line = <$fh>)) {
5423             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5424                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5425                 die "archive does not contain data for drive '$virtdev'\n"
5426                     if !$devinfo->{$devname};
5427                 if (defined($opts->{storage})) {
5428                     $storeid = $opts->{storage} || 'local';
5429                 } elsif (!$storeid) {
5430                     $storeid = 'local';
5431                 }
5432                 $format = 'raw' if !$format;
5433                 $devinfo->{$devname}->{devname} = $devname;
5434                 $devinfo->{$devname}->{virtdev} = $virtdev;
5435                 $devinfo->{$devname}->{format} = $format;
5436                 $devinfo->{$devname}->{storeid} = $storeid;
5437
5438                 # check permission on storage
5439                 my $pool = $opts->{pool}; # todo: do we need that?
5440                 if ($user ne 'root@pam') {
5441                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5442                 }
5443
5444                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5445             }
5446         }
5447
5448         foreach my $devname (keys %$devinfo) {
5449             die "found no device mapping information for device '$devname'\n"
5450                 if !$devinfo->{$devname}->{virtdev};
5451         }
5452
5453         my $cfg = PVE::Storage::config();
5454
5455         # create empty/temp config
5456         if ($oldconf) {
5457             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5458             foreach_drive($oldconf, sub {
5459                 my ($ds, $drive) = @_;
5460
5461                 return if drive_is_cdrom($drive);
5462
5463                 my $volid = $drive->{file};
5464
5465                 return if !$volid || $volid =~ m|^/|;
5466
5467                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5468                 return if !$path || !$owner || ($owner != $vmid);
5469
5470                 # Note: only delete disk we want to restore
5471                 # other volumes will become unused
5472                 if ($virtdev_hash->{$ds}) {
5473                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5474                     if (my $err = $@) {
5475                         warn $err;
5476                     }
5477                 }
5478             });
5479
5480             # delete vmstate files
5481             # since after the restore we have no snapshots anymore
5482             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5483                 my $snap = $oldconf->{snapshots}->{$snapname};
5484                 if ($snap->{vmstate}) {
5485                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5486                     if (my $err = $@) {
5487                         warn $err;
5488                     }
5489                 }
5490             }
5491         }
5492
5493         my $map = {};
5494         foreach my $virtdev (sort keys %$virtdev_hash) {
5495             my $d = $virtdev_hash->{$virtdev};
5496             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5497             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5498
5499             # test if requested format is supported
5500             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5501             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5502             $d->{format} = $defFormat if !$supported;
5503
5504             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5505                                                   $d->{format}, undef, $alloc_size);
5506             print STDERR "new volume ID is '$volid'\n";
5507             $d->{volid} = $volid;
5508             my $path = PVE::Storage::path($cfg, $volid);
5509
5510             PVE::Storage::activate_volumes($cfg,[$volid]);
5511
5512             my $write_zeros = 1;
5513             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5514                 $write_zeros = 0;
5515             }
5516
5517             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5518
5519             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5520             $map->{$virtdev} = $volid;
5521         }
5522
5523         $fh->seek(0, 0) || die "seek failed - $!\n";
5524
5525         my $outfd = new IO::File ($tmpfn, "w") ||
5526             die "unable to write config for VM $vmid\n";
5527
5528         my $cookie = { netcount => 0 };
5529         while (defined(my $line = <$fh>)) {
5530             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5531         }
5532
5533         $fh->close();
5534         $outfd->close();
5535     };
5536
5537     eval {
5538         # enable interrupts
5539         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5540             die "interrupted by signal\n";
5541         };
5542         local $SIG{ALRM} = sub { die "got timeout\n"; };
5543
5544         $oldtimeout = alarm($timeout);
5545
5546         my $parser = sub {
5547             my $line = shift;
5548
5549             print "$line\n";
5550
5551             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5552                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5553                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5554             } elsif ($line =~ m/^CTIME: /) {
5555                 # we correctly received the vma config, so we can disable
5556                 # the timeout now for disk allocation (set to 10 minutes, so
5557                 # that we always timeout if something goes wrong)
5558                 alarm(600);
5559                 &$print_devmap();
5560                 print $fifofh "done\n";
5561                 my $tmp = $oldtimeout || 0;
5562                 $oldtimeout = undef;
5563                 alarm($tmp);
5564                 close($fifofh);
5565             }
5566         };
5567
5568         print "restore vma archive: $cmd\n";
5569         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5570     };
5571     my $err = $@;
5572
5573     alarm($oldtimeout) if $oldtimeout;
5574
5575     my $vollist = [];
5576     foreach my $devname (keys %$devinfo) {
5577         my $volid = $devinfo->{$devname}->{volid};
5578         push @$vollist, $volid if $volid;
5579     }
5580
5581     my $cfg = PVE::Storage::config();
5582     PVE::Storage::deactivate_volumes($cfg, $vollist);
5583
5584     unlink $mapfifo;
5585
5586     if ($err) {
5587         rmtree $tmpdir;
5588         unlink $tmpfn;
5589
5590         foreach my $devname (keys %$devinfo) {
5591             my $volid = $devinfo->{$devname}->{volid};
5592             next if !$volid;
5593             eval {
5594                 if ($volid =~ m|^/|) {
5595                     unlink $volid || die 'unlink failed\n';
5596                 } else {
5597                     PVE::Storage::vdisk_free($cfg, $volid);
5598                 }
5599                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5600             };
5601             print STDERR "unable to cleanup '$volid' - $@" if $@;
5602         }
5603         die $err;
5604     }
5605
5606     rmtree $tmpdir;
5607
5608     rename($tmpfn, $conffile) ||
5609         die "unable to commit configuration file '$conffile'\n";
5610
5611     PVE::Cluster::cfs_update(); # make sure we read new file
5612
5613     eval { rescan($vmid, 1); };
5614     warn $@ if $@;
5615 }
5616
5617 sub restore_tar_archive {
5618     my ($archive, $vmid, $user, $opts) = @_;
5619
5620     if ($archive ne '-') {
5621         my $firstfile = tar_archive_read_firstfile($archive);
5622         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5623             if $firstfile ne 'qemu-server.conf';
5624     }
5625
5626     my $storecfg = PVE::Storage::config();
5627
5628     # destroy existing data - keep empty config
5629     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5630     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5631
5632     my $tocmd = "/usr/lib/qemu-server/qmextract";
5633
5634     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5635     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5636     $tocmd .= ' --prealloc' if $opts->{prealloc};
5637     $tocmd .= ' --info' if $opts->{info};
5638
5639     # tar option "xf" does not autodetect compression when read from STDIN,
5640     # so we pipe to zcat
5641     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5642         PVE::Tools::shellquote("--to-command=$tocmd");
5643
5644     my $tmpdir = "/var/tmp/vzdumptmp$$";
5645     mkpath $tmpdir;
5646
5647     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5648     local $ENV{VZDUMP_VMID} = $vmid;
5649     local $ENV{VZDUMP_USER} = $user;
5650
5651     my $conffile = PVE::QemuConfig->config_file($vmid);
5652     my $tmpfn = "$conffile.$$.tmp";
5653
5654     # disable interrupts (always do cleanups)
5655     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5656         print STDERR "got interrupt - ignored\n";
5657     };
5658
5659     eval {
5660         # enable interrupts
5661         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5662             die "interrupted by signal\n";
5663         };
5664
5665         if ($archive eq '-') {
5666             print "extracting archive from STDIN\n";
5667             run_command($cmd, input => "<&STDIN");
5668         } else {
5669             print "extracting archive '$archive'\n";
5670             run_command($cmd);
5671         }
5672
5673         return if $opts->{info};
5674
5675         # read new mapping
5676         my $map = {};
5677         my $statfile = "$tmpdir/qmrestore.stat";
5678         if (my $fd = IO::File->new($statfile, "r")) {
5679             while (defined (my $line = <$fd>)) {
5680                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5681                     $map->{$1} = $2 if $1;
5682                 } else {
5683                     print STDERR "unable to parse line in statfile - $line\n";
5684                 }
5685             }
5686             $fd->close();
5687         }
5688
5689         my $confsrc = "$tmpdir/qemu-server.conf";
5690
5691         my $srcfd = new IO::File($confsrc, "r") ||
5692             die "unable to open file '$confsrc'\n";
5693
5694         my $outfd = new IO::File ($tmpfn, "w") ||
5695             die "unable to write config for VM $vmid\n";
5696
5697         my $cookie = { netcount => 0 };
5698         while (defined (my $line = <$srcfd>)) {
5699             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5700         }
5701
5702         $srcfd->close();
5703         $outfd->close();
5704     };
5705     my $err = $@;
5706
5707     if ($err) {
5708
5709         unlink $tmpfn;
5710
5711         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5712
5713         die $err;
5714     }
5715
5716     rmtree $tmpdir;
5717
5718     rename $tmpfn, $conffile ||
5719         die "unable to commit configuration file '$conffile'\n";
5720
5721     PVE::Cluster::cfs_update(); # make sure we read new file
5722
5723     eval { rescan($vmid, 1); };
5724     warn $@ if $@;
5725 };
5726
5727 sub foreach_writable_storage {
5728     my ($conf, $func) = @_;
5729
5730     my $sidhash = {};
5731
5732     foreach my $ds (keys %$conf) {
5733         next if !is_valid_drivename($ds);
5734
5735         my $drive = parse_drive($ds, $conf->{$ds});
5736         next if !$drive;
5737         next if drive_is_cdrom($drive);
5738
5739         my $volid = $drive->{file};
5740
5741         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5742         $sidhash->{$sid} = $sid if $sid;
5743     }
5744
5745     foreach my $sid (sort keys %$sidhash) {
5746         &$func($sid);
5747     }
5748 }
5749
5750 sub do_snapshots_with_qemu {
5751     my ($storecfg, $volid) = @_;
5752
5753     my $storage_name = PVE::Storage::parse_volume_id($volid);
5754
5755     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5756         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5757         return 1;
5758     }
5759
5760     if ($volid =~ m/\.(qcow2|qed)$/){
5761         return 1;
5762     }
5763
5764     return undef;
5765 }
5766
5767 sub qga_check_running {
5768     my ($vmid) = @_;
5769
5770     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5771     if ($@) {
5772         warn "Qemu Guest Agent are not running - $@";
5773         return 0;
5774     }
5775     return 1;
5776 }
5777
5778 sub template_create {
5779     my ($vmid, $conf, $disk) = @_;
5780
5781     my $storecfg = PVE::Storage::config();
5782
5783     foreach_drive($conf, sub {
5784         my ($ds, $drive) = @_;
5785
5786         return if drive_is_cdrom($drive);
5787         return if $disk && $ds ne $disk;
5788
5789         my $volid = $drive->{file};
5790         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5791
5792         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5793         $drive->{file} = $voliddst;
5794         $conf->{$ds} = print_drive($vmid, $drive);
5795         PVE::QemuConfig->write_config($vmid, $conf);
5796     });
5797 }
5798
5799 sub qemu_img_convert {
5800     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5801
5802     my $storecfg = PVE::Storage::config();
5803     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5804     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5805
5806     if ($src_storeid && $dst_storeid) {
5807
5808         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5809
5810         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5811         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5812
5813         my $src_format = qemu_img_format($src_scfg, $src_volname);
5814         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5815
5816         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5817         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5818
5819         my $cmd = [];
5820         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5821         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5822         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5823         if ($is_zero_initialized) {
5824             push @$cmd, "zeroinit:$dst_path";
5825         } else {
5826             push @$cmd, $dst_path;
5827         }
5828
5829         my $parser = sub {
5830             my $line = shift;
5831             if($line =~ m/\((\S+)\/100\%\)/){
5832                 my $percent = $1;
5833                 my $transferred = int($size * $percent / 100);
5834                 my $remaining = $size - $transferred;
5835
5836                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5837             }
5838
5839         };
5840
5841         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5842         my $err = $@;
5843         die "copy failed: $err" if $err;
5844     }
5845 }
5846
5847 sub qemu_img_format {
5848     my ($scfg, $volname) = @_;
5849
5850     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5851         return $1;
5852     } else {
5853         return "raw";
5854     }
5855 }
5856
5857 sub qemu_drive_mirror {
5858     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5859
5860     my $storecfg = PVE::Storage::config();
5861     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5862
5863     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5864
5865     my $format = qemu_img_format($dst_scfg, $dst_volname);
5866
5867     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5868
5869     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5870
5871     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5872     $opts->{format} = $format if $format;
5873
5874     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5875
5876     my $finish_job = sub {
5877         while (1) {
5878             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5879             my $stat = @$stats[0];
5880             last if !$stat;
5881             sleep 1;
5882         }
5883     };
5884
5885     eval {
5886     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5887         while (1) {
5888             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5889             my $stat = @$stats[0];
5890             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5891             die "error job is not mirroring" if $stat->{type} ne "mirror";
5892
5893             my $busy = $stat->{busy};
5894             my $ready = $stat->{ready};
5895
5896             if (my $total = $stat->{len}) {
5897                 my $transferred = $stat->{offset} || 0;
5898                 my $remaining = $total - $transferred;
5899                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5900
5901                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5902             }
5903
5904
5905             if ($stat->{ready} eq 'true') {
5906
5907                 last if $vmiddst != $vmid;
5908
5909                 # try to switch the disk if source and destination are on the same guest
5910                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5911                 if (!$@) {
5912                     &$finish_job();
5913                     last;
5914                 }
5915                 die $@ if $@ !~ m/cannot be completed/;
5916             }
5917             sleep 1;
5918         }
5919
5920
5921     };
5922     my $err = $@;
5923
5924     my $cancel_job = sub {
5925         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5926         &$finish_job();
5927     };
5928
5929     if ($err) {
5930         eval { &$cancel_job(); };
5931         die "mirroring error: $err";
5932     }
5933
5934     if ($vmiddst != $vmid) {
5935         # if we clone a disk for a new target vm, we don't switch the disk
5936         &$cancel_job(); # so we call block-job-cancel
5937     }
5938 }
5939
5940 sub clone_disk {
5941     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5942         $newvmid, $storage, $format, $full, $newvollist) = @_;
5943
5944     my $newvolid;
5945
5946     if (!$full) {
5947         print "create linked clone of drive $drivename ($drive->{file})\n";
5948         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5949         push @$newvollist, $newvolid;
5950     } else {
5951         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5952         $storeid = $storage if $storage;
5953
5954         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5955         if (!$format) {
5956             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5957             $format = qemu_img_format($scfg, $volname);
5958         }
5959
5960         # test if requested format is supported - else use default
5961         my $supported = grep { $_ eq $format } @$validFormats;
5962         $format = $defFormat if !$supported;
5963
5964         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5965
5966         print "create full clone of drive $drivename ($drive->{file})\n";
5967         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5968         push @$newvollist, $newvolid;
5969
5970         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
5971
5972         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5973         if (!$running || $snapname) {
5974             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5975         } else {
5976
5977             my $kvmver = get_running_qemu_version ($vmid);
5978             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
5979                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
5980                     if $drive->{iothread};
5981             }
5982
5983             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5984         }
5985     }
5986
5987     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5988
5989     my $disk = $drive;
5990     $disk->{format} = undef;
5991     $disk->{file} = $newvolid;
5992     $disk->{size} = $size;
5993
5994     return $disk;
5995 }
5996
5997 # this only works if VM is running
5998 sub get_current_qemu_machine {
5999     my ($vmid) = @_;
6000
6001     my $cmd = { execute => 'query-machines', arguments => {} };
6002     my $res = vm_qmp_command($vmid, $cmd);
6003
6004     my ($current, $default);
6005     foreach my $e (@$res) {
6006         $default = $e->{name} if $e->{'is-default'};
6007         $current = $e->{name} if $e->{'is-current'};
6008     }
6009
6010     # fallback to the default machine if current is not supported by qemu
6011     return $current || $default || 'pc';
6012 }
6013
6014 sub get_running_qemu_version {
6015     my ($vmid) = @_;
6016     my $cmd = { execute => 'query-version', arguments => {} };
6017     my $res = vm_qmp_command($vmid, $cmd);
6018     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6019 }
6020
6021 sub qemu_machine_feature_enabled {
6022     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6023
6024     my $current_major;
6025     my $current_minor;
6026
6027     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6028
6029         $current_major = $3;
6030         $current_minor = $4;
6031
6032     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6033
6034         $current_major = $1;
6035         $current_minor = $2;
6036     }
6037
6038     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6039
6040
6041 }
6042
6043 sub qemu_machine_pxe {
6044     my ($vmid, $conf, $machine) = @_;
6045
6046     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6047
6048     foreach my $opt (keys %$conf) {
6049         next if $opt !~ m/^net(\d+)$/;
6050         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6051         next if !$net;
6052         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6053         return $machine.".pxe" if $romfile =~ m/pxe/;
6054         last;
6055     }
6056
6057     return $machine;
6058 }
6059
6060 sub qemu_use_old_bios_files {
6061     my ($machine_type) = @_;
6062
6063     return if !$machine_type;
6064
6065     my $use_old_bios_files = undef;
6066
6067     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6068         $machine_type = $1;
6069         $use_old_bios_files = 1;
6070     } else {
6071         my $kvmver = kvm_user_version();
6072         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6073         # load new efi bios files on migration. So this hack is required to allow
6074         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6075         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6076         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6077     }
6078
6079     return ($use_old_bios_files, $machine_type);
6080 }
6081
6082 sub lspci {
6083
6084     my $devices = {};
6085
6086     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6087             my (undef, $id, $function) = @_;
6088             my $res = { id => $id, function => $function};
6089             push @{$devices->{$id}}, $res;
6090     });
6091
6092     return $devices;
6093 }
6094
6095 sub vm_iothreads_list {
6096     my ($vmid) = @_;
6097
6098     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6099
6100     my $iothreads = {};
6101     foreach my $iothread (@$res) {
6102         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6103     }
6104
6105     return $iothreads;
6106 }
6107
6108 sub scsihw_infos {
6109     my ($conf, $drive) = @_;
6110
6111     my $maxdev = 0;
6112
6113     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6114         $maxdev = 7;
6115     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6116         $maxdev = 1;
6117     } else {
6118         $maxdev = 256;
6119     }
6120
6121     my $controller = int($drive->{index} / $maxdev);
6122     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6123
6124     return ($maxdev, $controller, $controller_prefix);
6125 }
6126
6127 # bash completion helper
6128
6129 sub complete_backup_archives {
6130     my ($cmdname, $pname, $cvalue) = @_;
6131
6132     my $cfg = PVE::Storage::config();
6133
6134     my $storeid;
6135
6136     if ($cvalue =~ m/^([^:]+):/) {
6137         $storeid = $1;
6138     }
6139
6140     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6141
6142     my $res = [];
6143     foreach my $id (keys %$data) {
6144         foreach my $item (@{$data->{$id}}) {
6145             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6146             push @$res, $item->{volid} if defined($item->{volid});
6147         }
6148     }
6149
6150     return $res;
6151 }
6152
6153 my $complete_vmid_full = sub {
6154     my ($running) = @_;
6155
6156     my $idlist = vmstatus();
6157
6158     my $res = [];
6159
6160     foreach my $id (keys %$idlist) {
6161         my $d = $idlist->{$id};
6162         if (defined($running)) {
6163             next if $d->{template};
6164             next if $running && $d->{status} ne 'running';
6165             next if !$running && $d->{status} eq 'running';
6166         }
6167         push @$res, $id;
6168
6169     }
6170     return $res;
6171 };
6172
6173 sub complete_vmid {
6174     return &$complete_vmid_full();
6175 }
6176
6177 sub complete_vmid_stopped {
6178     return &$complete_vmid_full(0);
6179 }
6180
6181 sub complete_vmid_running {
6182     return &$complete_vmid_full(1);
6183 }
6184
6185 sub complete_storage {
6186
6187     my $cfg = PVE::Storage::config();
6188     my $ids = $cfg->{ids};
6189
6190     my $res = [];
6191     foreach my $sid (keys %$ids) {
6192         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6193         next if !$ids->{$sid}->{content}->{images};
6194         push @$res, $sid;
6195     }
6196
6197     return $res;
6198 }
6199
6200 1;